@ultimat3/cli 29.4.0 → 30.1.0

package/src/cmd-db-spec.ts

@@ -12,13 +12,15 @@ export const DB_SUBCOMMANDS = [
   'studio',
   'branch',
   'backfill',
+  'diagnostics',
+  'recount',
 ] as const;
 
 export const dbSpec: CommandSpec = {
   name: 'db',
-  summary: 'gen, migrate, reset, seed, studio, branch, backfill',
+  summary: 'gen, migrate, reset, seed, studio, branch, backfill, diagnostics, recount',
   usage:
-    'x db gen "add publish_at" | migrate | reset | seed [<name>] [--tier reference|dev] [--dry-run] | studio | branch list | branch create <name> | branch delete <name> | backfill [<name>|--all] [--write] [--force] | backfill --pending | backfill --list [--name n] [--status s] [--limit n]',
+    'x db gen "add publish_at" | migrate | reset | seed [<name>] [--tier reference|dev] [--dry-run] | seed --generate [--rows n] [--seed n] [--dry-run] | studio | branch list | branch create <name> | branch delete <name> | backfill [<name>|--all] [--write] [--force] | backfill --pending | backfill --list [--name n] [--status s] [--limit n] | diagnostics [--limit n] | recount <entity>.<column> [--write]',
   requiresApp: true,
   subcommands: DB_SUBCOMMANDS,
   // Declared from the constant `runBranchCommand` validates against, never a second literal: it
@@ -46,6 +48,26 @@ export const dbSpec: CommandSpec = {
       summary: 'seed: report what each seed would write, and write nothing',
       subcommands: ['seed'],
     },
+    // Rows for every table off the entity declarations, with no seed written: the seed graph
+    // `x exercise` runs on. Fixture data, so `dev` only; production is refused whatever the tier.
+    {
+      name: 'generate',
+      type: 'boolean',
+      summary: 'seed: generated dev rows for every entity, parents first; never in production',
+      subcommands: ['seed'],
+    },
+    {
+      name: 'rows',
+      type: 'string',
+      summary: 'seed: with --generate, rows per table (per tenant when tenant-scoped; default 3)',
+      subcommands: ['seed'],
+    },
+    {
+      name: 'seed',
+      type: 'string',
+      summary: 'seed: with --generate, the integer the graph is drawn from — same seed, same rows',
+      subcommands: ['seed'],
+    },
     {
       name: 'list',
       type: 'boolean',
@@ -67,8 +89,8 @@ export const dbSpec: CommandSpec = {
     {
       name: 'write',
       type: 'boolean',
-      summary: 'backfill: enqueue the pass; dry run without it',
-      subcommands: ['backfill'],
+      summary: 'backfill, recount: enqueue the pass, or set drifted counters; dry run without it',
+      subcommands: ['backfill', 'recount'],
     },
     {
       name: 'force',
@@ -85,8 +107,9 @@ export const dbSpec: CommandSpec = {
     {
       name: 'limit',
       type: 'string',
-      summary: 'backfill: max ledger rows to return',
-      subcommands: ['backfill'],
+      summary:
+        'backfill, diagnostics: max ledger rows, or rows per diagnostics section (default 20)',
+      subcommands: ['backfill', 'diagnostics'],
     },
     // Declared because `X_MIGRATION_IRREVERSIBLE`'s own fix line names it. A `fix:` is copied
     // and run verbatim, so a flag the parser refuses would make the error unfollowable.
@@ -95,5 +118,12 @@ export const dbSpec: CommandSpec = {
       type: 'boolean',
       summary: 'let x db gen emit a drop whose down cannot restore the rows',
     },
+    // Declared because `X_MIGRATION_ROLLING_UNSAFE`'s own fix line names it.
+    {
+      name: 'rolling-unsafe',
+      type: 'boolean',
+      summary:
+        'let x db gen drop a column never retired, for an app that stops before it starts: one instance, or strategy: Recreate',
+    },
   ],
 };

package/src/cmd-db.ts

@@ -1,6 +1,7 @@
-// `x db gen|migrate|reset|seed|studio|branch|backfill` — everything that touches the database. One
-// subcommand per line and no fall-through: a word this file does not know is refused, never
-// re-read as an argument to the last branch. `branch` itself is `cmd-db-branch.ts`.
+// `x db gen|migrate|reset|seed|studio|branch|backfill|diagnostics|recount` — everything that touches the
+// database. One subcommand per line and no fall-through: a word this file does not know is refused,
+// never re-read as an argument to the last branch. `branch` is `cmd-db-branch.ts`, `diagnostics`
+// is `db-diagnostics.ts`.
 //
 // Every subcommand here runs `@ultimat3/db`'s own engine, which is the engine `ROLE=migrate` runs
 // (`serve.ts`): one `x_migrations` ledger, one checksum rule, one advisory lock, from a laptop to
@@ -17,9 +18,11 @@ import { postgresDriver } from '@ultimat3/entity';
 import { requireAppRoot } from './app-root';
 import { runBackfillCommand } from './cmd-db-backfill';
 import { runBranchCommand } from './cmd-db-branch';
+import { runRecountCommand } from './cmd-db-recount';
 import { DB_SUBCOMMANDS, dbSpec } from './cmd-db-spec';
 import { plannedSubcommand } from './cmd-planned';
 import type { CliCommand, CommandContext } from './command';
+import { runDiagnostics } from './db-diagnostics';
 import { stepFinding } from './db-finding';
 import { generateAppMigration, unrenderedJson, unrenderedLines } from './db-generate';
 import { liveObjectDrift } from './db-object-drift';
@@ -40,6 +43,7 @@ import {
   seedTotals,
   selectSeeds,
 } from './db-seed';
+import { runGeneratedSeed } from './db-seed-generate';
 import { DevAlreadyRunningError, liveDevLock } from './dev-lock';
 import { MissingSubcommandError, UnknownCommandError } from './errors';
 import { withJobDriver } from './jobs-driver';
@@ -71,6 +75,8 @@ export const dbCommand: CliCommand = {
     if (sub === 'studio') throw plannedSubcommand('db', 'studio');
     if (sub === 'backfill') return runBackfillCommand(ctx, root);
     if (sub === 'branch') return runBranchCommand(ctx, root);
+    if (sub === 'diagnostics') return runDiagnostics(ctx, root);
+    if (sub === 'recount') return runRecountCommand(ctx, root);
 
     // Never a fall-through. This used to end `return runBranch(ctx, root, argument ?? 'preview')`,
     // so ANY subcommand the parser had not already refused was reinterpreted as a branch NAME and
@@ -102,6 +108,7 @@ async function runGen(ctx: CommandContext, root: string, name: string): Promise<
     generated = await generateAppMigration(root, {
       name,
       allowDestructive: flagBool(ctx.args, 'allow-destructive'),
+      rollingUnsafe: flagBool(ctx.args, 'rolling-unsafe'),
     });
   } catch (error) {
     return {
@@ -297,6 +304,7 @@ async function runReset(ctx: CommandContext, root: string): Promise<CommandResul
  * `handle.enqueue()` a seeded write triggers. A second boot path would be a second answer.
  */
 async function runSeed(ctx: CommandContext, root: string): Promise<CommandResult> {
+  if (flagBool(ctx.args, 'generate')) return runGeneratedSeed(ctx, root);
   const environment = resolveEnvironment({ env: ctx.env });
   const requested = parseSeedTierFlag(
     flagString(ctx.args, 'tier') ?? ctx.env['ULTIMATE_SEED_TIER'],

package/src/cmd-deploy-compose.ts

@@ -1,6 +1,6 @@
-// The compose half of `x deploy`'s rollout: which serving roles can start their new container
-// BEFORE the old one stops, read off the app's compose file. The sequence that does it is
-// `cmd-deploy-start-first.ts`.
+// The compose half of `x deploy`, read off the app's compose file: which serving roles can start
+// their new container BEFORE the old one stops (the sequence is `cmd-deploy-start-first.ts`), and
+// what the file demands of `.env.production` — read by the deploy's pre-flight and `x env init`.
 // `docker compose up -d <role>` alone is stop-first — it recreates, so the role serves nothing
 // between the old container's exit and the new one's healthcheck.
 
@@ -101,3 +101,61 @@ export async function readComposeFile(path: string): Promise<string | undefined>
   const file = Bun.file(path);
   return (await file.exists()) ? file.text() : undefined;
 }
+
+/**
+ * The reference `x build --target docker` tags and `x deploy` deploys when neither is given one
+ * (`cmd-build.ts` spells the same literal). `x env init` writes it as `IMAGE`, so a compose command
+ * run by hand reaches the image the build made rather than the compose file's own default.
+ */
+export const DEFAULT_IMAGE = 'ultimate-app:dev';
+
+/**
+ * Every variable the file interpolates with `${VAR:?…}` or `${VAR?…}`, in first-use order — the
+ * ones compose refuses to parse without. Read off the text, not the parse: YAML keeps the
+ * interpolation inside a string, wherever in the document it sits. Comment lines are skipped.
+ */
+export function composeRequiredVars(text: string): readonly string[] {
+  // A comment line is never interpolated, and the scaffold's header spells `${VAR:?…}` in prose.
+  const live = text
+    .split('\n')
+    .filter((line) => !line.trimStart().startsWith('#'))
+    .join('\n');
+  const names = [...live.matchAll(/\$\{([A-Za-z_][A-Za-z0-9_]*):?\?/g)].map((m) => m[1] ?? '');
+  return [...new Set(names)];
+}
+
+/** The `db` service's role and database, which a `DATABASE_URL` for this file must name. */
+export interface ComposeDatabase {
+  readonly user: string;
+  readonly database: string;
+}
+
+/** One `environment:` entry, from either of compose's two spellings (map or `KEY=value` list). */
+function environmentValue(environment: unknown, key: string): string | undefined {
+  if (isRecord(environment)) {
+    const value = environment[key];
+    return typeof value === 'string' || typeof value === 'number' ? String(value) : undefined;
+  }
+  if (!Array.isArray(environment)) return undefined;
+  const entry = environment.find((e) => typeof e === 'string' && e.startsWith(`${key}=`));
+  return typeof entry === 'string' ? entry.slice(key.length + 1) : undefined;
+}
+
+/**
+ * The `db` service's Postgres role and database: the image's defaults (`postgres`, and a database
+ * named after the role) unless `POSTGRES_USER` / `POSTGRES_DB` say otherwise. `undefined` when the
+ * file runs no `db` service, so the database is the operator's and no URL can be derived.
+ */
+export function composeDatabase(text: string | undefined): ComposeDatabase | undefined {
+  let parsed: unknown;
+  try {
+    parsed = text === undefined ? undefined : Bun.YAML.parse(text);
+  } catch {
+    return undefined;
+  }
+  const services = isRecord(parsed) ? parsed['services'] : undefined;
+  const db = isRecord(services) ? services['db'] : undefined;
+  if (!isRecord(db)) return undefined;
+  const user = environmentValue(db['environment'], 'POSTGRES_USER') ?? 'postgres';
+  return { user, database: environmentValue(db['environment'], 'POSTGRES_DB') ?? user };
+}

package/src/cmd-deploy-drain.ts

@@ -1,10 +1,12 @@
 // What a helm deploy tells the chart about the drain: `app.config.ts`'s `drain` section — the ONE
 // drain budget since 25.0.0 deleted `http.drainTimeoutMs` — in the seconds `docker/helm` derives
 // each role's terminationGracePeriodSeconds from. Passed on every upgrade, so a raised
-// budget can never ship against a grace period sized for 25 s.
+// budget can never ship against a grace period sized for 25 s, and nor can the retire beside it.
 
 import type { DrainConfig } from '@ultimat3/core';
 import { isUltimateError, READINESS_GRACE_DEFAULT_MS } from '@ultimat3/core';
+import { loadAppConfig } from './app-config-load';
+import { helmWorkerTargetOverrides } from './cmd-deploy-helm';
 import { loadDrainConfig } from './serve-drain';
 
 const seconds = (ms: number): number => Math.ceil(ms / 1000);
@@ -29,6 +31,10 @@ export function helmDrainOverrides(drain: Partial<DrainConfig> | undefined): rea
   if (drain?.workerDeadlineMs !== undefined) {
     out.push('--set', `drain.workerDeadlineSeconds=${seconds(drain.workerDeadlineMs)}`);
   }
+  // Sent at 0 as well: `workerRetireMs: 0` must reach a chart whose default is on.
+  if (drain?.workerRetireMs !== undefined) {
+    out.push('--set', `roles.worker.retireSeconds=${seconds(drain.workerRetireMs)}`);
+  }
   if (drain === undefined) return out;
   const grace = Math.max(drain.readinessGraceMs ?? 0, READINESS_GRACE_DEFAULT_MS);
   out.push('--set', `drain.readinessGraceSeconds=${seconds(grace)}`);
@@ -43,11 +49,13 @@ export function helmDrainOverrides(drain: Partial<DrainConfig> | undefined): rea
  */
 export async function readHelmDrainOverrides(root: string): Promise<readonly string[]> {
   let drain: Partial<DrainConfig> | undefined;
+  let maxInFlight: number | undefined;
   try {
     drain = await loadDrainConfig(root);
+    maxInFlight = (await loadAppConfig(root))?.jobs?.maxInFlight;
   } catch (error) {
     if (!isUltimateError(error) || error.code !== 'X_CONFIG_INVALID') throw error;
     return [];
   }
-  return helmDrainOverrides(drain);
+  return [...helmDrainOverrides(drain), ...helmWorkerTargetOverrides(maxInFlight)];
 }

package/src/cmd-deploy-helm.ts

@@ -104,6 +104,17 @@ export function helmUpgradeArgs(
   ];
 }
 
+/**
+ * One worker pod's capacity is what an AverageValue target on `queue_demand` must be: the HPA asks
+ * for `ceil(demand / target)` pods, so a target above `jobs.maxInFlight` leaves demand no pod can
+ * hold. Undeclared, the chart's value stands: nothing here knows the size of a pod.
+ */
+export function helmWorkerTargetOverrides(maxInFlight: number | undefined): readonly string[] {
+  return maxInFlight === undefined
+    ? []
+    : ['--set', `roles.worker.autoscaling.targetAverageValue=${maxInFlight}`];
+}
+
 /** What `--json` reports about the rollout helm waited on. */
 export interface Rollout {
   readonly release: string;

package/src/cmd-deploy-preflight.ts

@@ -0,0 +1,118 @@
+// What `x deploy --method compose` checks before its first step, and what it reads after a failed
+// one. Before: `.env.production` exists and sets every `${VAR:?}` the compose file demands —
+// without it each step died as `role "migrate" step exited 1`. After: the role's last log line.
+
+// why: Bun ships no path-join primitive; the env file is app-root-relative.
+import { join } from 'node:path';
+import type { EnvSchema } from '@ultimat3/core';
+import { ERROR_DOCS_URL } from '@ultimat3/core';
+import { composeRequiredVars } from './cmd-deploy-compose';
+import { requiredAtBoot } from './env-production';
+import type { ExecOptions, Runner } from './exec';
+import type { Finding } from './output';
+import { parseDotenv } from './root-env';
+import { PROD_ENV_FILE } from './templates/scaffold-container-compose';
+
+type EnvTable = Readonly<Record<string, string | undefined>>;
+
+const missingFinding = (cause: string): Finding => ({
+  code: 'X_DEPLOY_ENV_MISSING',
+  cause,
+  fix: 'x env init --production --json',
+  docs: ERROR_DOCS_URL,
+  at: PROD_ENV_FILE,
+});
+
+/**
+ * The refusal, or `undefined` when every step can parse the compose file and every role can boot.
+ * A compose variable counts as set from the shell too — compose reads it there first; a key the
+ * app's `envSchema` requires reaches the containers through `env_file:` alone, so only the file
+ * counts for it. Empty is as absent as none: `${VAR:?}` and the boot's env check refuse both.
+ */
+export async function composeEnvPreflight(
+  root: string,
+  compose: string | undefined,
+  shell: EnvTable,
+  schema: EnvSchema = {},
+): Promise<Finding | undefined> {
+  const file = Bun.file(join(root, PROD_ENV_FILE));
+  if (!(await file.exists())) {
+    return missingFinding(
+      `${PROD_ENV_FILE} does not exist, and every compose step reads it (--env-file and env_file:)`,
+    );
+  }
+  const values = parseDotenv(await file.text());
+  const interpolated = (compose === undefined ? [] : composeRequiredVars(compose)).filter(
+    (key) => (shell[key] ?? values.get(key) ?? '') === '',
+  );
+  const declared = Object.entries(schema)
+    .filter(([key, decl]) => requiredAtBoot(decl) && (values.get(key) ?? '') === '')
+    .map(([key]) => key)
+    .filter((key) => !interpolated.includes(key));
+  const missing = [...interpolated, ...declared];
+  if (missing.length === 0) return undefined;
+  return {
+    ...missingFinding(
+      `${PROD_ENV_FILE} leaves ${missing.join(', ')} unset or empty, and ${missing.length === 1 ? 'it is' : 'each is'} required: by docker/docker-compose.prod.yml (\${VAR:?}) or by app.config.ts's envSchema, which every role checks at boot`,
+    ),
+    // The file exists, so the generator would refuse it: the edit is the fix.
+    fix: `NODE_ENV=production x env check --json   # after setting ${missing.join(', ')} in ${PROD_ENV_FILE}`,
+  };
+}
+
+/** A rendered `UltimateError`: `X_CODE: title — cause`. A JSON log's `"code":"X_…"` is not one. */
+const RENDERED_CODE = /\bX_[A-Z][A-Z0-9_]*: /;
+
+/** Bun's trailer after an uncaught error, the last line of every crash and never its reason. */
+const BUN_TRAILER = /^Bun v\d+\.\d+/;
+
+/** The longest line a finding carries: the reason fits, a stack-sized blob does not. */
+const LINE_CEILING = 500;
+
+/**
+ * The line that says why a process failed: the last rendered `X_*` refusal, else the last
+ * non-empty line that is not Bun's version trailer. stderr first, where a refusal is written.
+ */
+export function lastLine(output: { readonly stdout: string; readonly stderr: string }): string {
+  for (const text of [output.stderr, output.stdout]) {
+    const lines = text
+      .split(/\r?\n/)
+      .map((entry) => entry.trim())
+      .filter((entry) => entry !== '');
+    const line =
+      lines.findLast((entry) => RENDERED_CODE.test(entry)) ??
+      lines.findLast((entry) => !BUN_TRAILER.test(entry));
+    if (line !== undefined) return line.slice(0, LINE_CEILING);
+  }
+  return '';
+}
+
+/**
+ * A serving role's last log line, read after its step failed: a crash-looping container says why
+ * in its log, and `up -d` itself only says that it did not become healthy. Empty when compose
+ * cannot read the log either; the finding then carries the rerun command alone.
+ */
+export async function roleLastLogLine(input: {
+  readonly base: readonly string[];
+  readonly role: string;
+  readonly runner: Runner;
+  readonly options: ExecOptions;
+}): Promise<string> {
+  const command = [
+    ...input.base,
+    'logs',
+    '--no-color',
+    '--no-log-prefix',
+    '--tail',
+    '50',
+    input.role,
+  ];
+  try {
+    const result = await input.runner(command, input.options);
+    return result.ok ? lastLine(result) : '';
+  } catch {
+    // The log is a courtesy on a failure already being reported; a runner that cannot spawn
+    // compose here must not replace that report with its own.
+    return '';
+  }
+}

package/src/cmd-deploy-spec.ts

@@ -7,24 +7,76 @@ export const deploySpec: CommandSpec = {
   name: 'deploy',
   summary: 'run the container deploy plan: migrate first, then the serving roles',
   usage:
-    'x deploy --image repo/app:tag [--method compose|helm] [--release name] [--namespace ns] [--timeout 15m] [--dry-run] [--json]',
+    'x deploy --image repo/app:tag [--method compose|helm] [--release name] [--namespace ns] [--timeout 15m] [--dry-run] [--json] | x deploy verify --url <origin> [--build <id>] [--window 60s] [--interval 5s] [--timeout 10m] [--idle-ok <queue,…>] [--json]',
   requiresApp: true,
+  // `up` is the parser's name for the bare form, so `x deploy --image …` stays the deploy. The usage
+  // and every doc keep that one spelling.
+  subcommands: ['up', 'verify'],
+  defaultSubcommand: 'up',
   flags: [
-    { name: 'image', type: 'string', summary: 'image reference to deploy' },
-    { name: 'method', type: 'string', summary: 'compose | helm', default: 'compose' },
-    { name: 'dry-run', type: 'boolean', summary: 'print the plan, run nothing' },
+    { name: 'image', type: 'string', summary: 'image reference to deploy', subcommands: ['up'] },
+    {
+      name: 'method',
+      type: 'string',
+      summary: 'compose | helm',
+      default: 'compose',
+      subcommands: ['up'],
+    },
+    {
+      name: 'dry-run',
+      type: 'boolean',
+      summary: 'print the plan, run nothing',
+      subcommands: ['up'],
+    },
     {
       name: 'release',
       type: 'string',
       summary: 'helm release name (default: app.config.ts name)',
+      subcommands: ['up'],
+    },
+    {
+      name: 'namespace',
+      type: 'string',
+      summary: 'helm namespace (default: the kube context)',
+      subcommands: ['up'],
     },
-    { name: 'namespace', type: 'string', summary: 'helm namespace (default: the kube context)' },
     {
       name: 'timeout',
       type: 'string',
       // No `default:` — the parser would then fill it on every compose deploy too, and the refusal
       // of a helm-only flag there could not tell an operator's value from its own.
-      summary: 'how long helm waits for the migrate hook and the rollout (default 15m)',
+      summary:
+        'how long helm waits for the migrate hook and the rollout (default 15m); verify: the whole wait (default 10m)',
+    },
+    {
+      name: 'url',
+      type: 'string',
+      summary: 'the origin to sample: /readyz behind the load balancer',
+      subcommands: ['verify'],
+    },
+    {
+      name: 'build',
+      type: 'string',
+      summary: "the build that must answer (default: x.manifest.json's buildId)",
+      subcommands: ['verify'],
+    },
+    {
+      name: 'window',
+      type: 'string',
+      summary: 'how long consecutive good samples must span (default 60s); 0: answered once',
+      subcommands: ['verify'],
+    },
+    {
+      name: 'interval',
+      type: 'string',
+      summary: 'the gap between samples (default 5s)',
+      subcommands: ['verify'],
+    },
+    {
+      name: 'idle-ok',
+      type: 'string',
+      summary: 'queues a scale-to-zero pool may leave unserved while their depth is 0',
+      subcommands: ['verify'],
     },
     // `--critical` was here and is gone. It parsed, it was echoed into the plan JSON as
     // `critical: <bool>`, and no file in `packages/` read that field — so the flag changed