@starci/hfs 4.11.8 → 4.11.12

package/scaffold/add-queue-lite.mjs

@@ -84,12 +84,14 @@ const DATABASE_ERROR = 'be/src/modules/platform/database/errors/database.error.t
 
 /** Skeleton files the lite starter leaves out and the queue capabilities use, each from its skeleton template. */
 const SKELETON_FILES = Object.freeze([
+  ['be/src/modules/platform/database/database.client.ts', 'skeleton-lite/src/modules/platform/database/database.client.ts'],
+  ['be/src/modules/platform/database/database.client.spec.ts', 'skeleton-lite/src/modules/platform/database/database.client.spec.ts'],
   ['be/src/modules/platform/database/database.sql.ts', 'skeleton-lite/src/modules/platform/database/database.sql.ts'],
   ['be/src/modules/platform/primitives/sequence.policy.ts', 'skeleton/src/modules/platform/primitives/sequence.policy.ts'],
   ['be/src/modules/platform/primitives/sequence.policy.spec.ts', 'skeleton/src/modules/platform/primitives/sequence.policy.spec.ts'],
 ]);
 
-/** The sql tag of the outbox and claim statements (with its identifier error) and the in-order relay the queue adapters walk their rows with. */
+/** The database-function call and the sql tag of the outbox and claim statements (with its identifier error) and the in-order relay the queue adapters walk their rows with. */
 function skeletonFiles(repoRoot) {
   const created = SKELETON_FILES.filter(([relative]) => !fs.existsSync(path.join(repoRoot, ...relative.split('/'))))
     .map(([relative, source]) => ({ relative, text: render(template(`be/${source}`), {}) }));
@@ -364,7 +366,10 @@ export function planLiteQueue({ repoRoot, repo, noun, patterns, now }) {
         fs.writeFileSync(target, entry.text);
       }
       for (const patch of patches) fs.writeFileSync(patch.file, patch.text);
-      const exported = registerLiteExports({ root: repoRoot, generator: 'queue' });
+      const exported = [
+        ...registerLiteExports({ root: repoRoot, generator: 'queue' }),
+        ...(patterns.includes('fenced-job') ? registerLiteExports({ root: repoRoot, generator: 'jobs' }) : []),
+      ];
       const sealed = sealRedisMembers(repoRoot, stack.environments);
       const changed = patches.map((patch) => path.relative(repoRoot, patch.file).split(path.sep).join('/'));
       return [...new Set([...created.map((entry) => entry.relative), ...changed, ...exported, ...sealed])];

package/scaffold/add-table.mjs

@@ -472,9 +472,14 @@ function writeTableFiles({ root, planned, edit }, created) {
 
 }
 
+/** The lite sql helpers of platform/database (requireOwnedEntity, the bounds and the sql tag). */
+const DATABASE_SQL_FILE = "be/src/modules/platform/database/database.sql.ts";
+
 function tableRegistrations({ root, repo, fe, edit }) {
   const errorFile = "be/src/modules/platform/database/errors/database.error.ts";
   if (repo.edition === "lite" && fs.existsSync(at(root, errorFile))) edit(errorFile, withSqlIdentifierError);
+  // requireOwnedEntity lives in the sql helpers the lite starter leaves out; the first table writes them.
+  if (repo.edition === "lite") edit(DATABASE_SQL_FILE, (text) => text || render(fs.readFileSync(path.join(TEMPLATES_DIR, "be", "skeleton-lite", "src", "modules", "platform", "database", "database.sql.ts"), "utf8"), {}));
     const registrations = [
       ...(repo.edition === "lite" ? ["table"] : []),
       ...(fe ? ["api", "graphql"] : []),

package/scaffold/app.mjs

@@ -353,13 +353,25 @@ const CLI_APP_DIR = 'apps/cli/';
 const declaresCliApp = app => app.sides.be.apps.some(entry => entry.kind === 'cli' && entry.name === 'cli');
 
 const LITE_APP_REUSED = new Set(['.editorconfig', '.gitattributes', '.nvmrc']);
+/**
+ * The lite be files a generator writes when it first needs them, never the starter: the cli's runners, the in-order relay, and the
+ * raw SQL helpers of platform/database (the sql tag, and the database-function call statement with its spec), which only the table,
+ * queue, jobs and webhook capabilities call; a fresh app reads and writes through entities alone (R334).
+ */
 const LITE_DEFERRED_GENERATOR_FILES = new Set([
+  'src/modules/platform/database/database.client.ts',
+  'src/modules/platform/database/database.client.spec.ts',
+  'src/modules/platform/database/database.sql.ts',
   'src/modules/platform/database/migration-runner.service.ts',
   'src/modules/platform/database/seed-runner.service.ts',
   'src/modules/platform/primitives/sequence.policy.ts',
 ]);
-/** The TypeORM schema files of a platform capability: lite's schema authority is db/migrations (be.persistence lite overlay). */
-const LITE_TYPEORM_SCHEMA = /^src\/modules\/platform\/[^/]+\/persistence\/(?:connection\.ts$|entities\/|migrations\/)/;
+/**
+ * The persistence folder of a full platform capability (TypeORM migrations, connection arrays, `sql`-tagged statements and rows):
+ * lite's schema authority is db/migrations and its code reads and writes through entities (be.persistence lite overlay, R334), so a
+ * lite platform capability takes its persistence/ from the lite overlay alone.
+ */
+const FULL_PLATFORM_PERSISTENCE = /^src\/modules\/platform\/[^/]+\/persistence\//;
 const liteBaseSource = (rel) => !rel.startsWith('apps/cli/')
     && !rel.startsWith('src/features/cli/')
     && !rel.startsWith('src/modules/domain/note/')
@@ -367,7 +379,7 @@ const liteBaseSource = (rel) => !rel.startsWith('apps/cli/')
     && rel !== 'src/modules/platform/database/migrate-connections.client.ts'
     && rel !== 'src/modules/platform/database/seed-connections.client.ts'
     && !LITE_DEFERRED_GENERATOR_FILES.has(rel)
-    && !LITE_TYPEORM_SCHEMA.test(rel)
+    && !FULL_PLATFORM_PERSISTENCE.test(rel)
     && !rel.startsWith('src/tests/');
 /** The be files the lite overlay replaces: a base unit spec of such a file tests the full variant, so lite carries its own. */
 const liteOverlaid = (() => {

package/scaffold/lite-exports.mjs

@@ -51,6 +51,22 @@ const REGISTRATIONS = Object.freeze({
       names: ["sql"],
       beforeSource: "./errors/database.error",
     },
+    {
+      // The outbox write and the relay call the queue functions of their migration (R334).
+      file: "be/src/modules/platform/database/index.ts",
+      source: "./database.client",
+      names: ["callDbFunction"],
+      beforeSource: "./primary.decorators",
+    },
+  ],
+  // The jobs capability (add job, the fenced-job pattern): its tick takes a transaction-scoped lock before it claims the minute (R335).
+  jobs: [
+    {
+      file: "be/src/modules/platform/database/index.ts",
+      source: "./database.client",
+      names: ["advisoryXactLock", "callDbFunction"],
+      beforeSource: "./primary.decorators",
+    },
   ],
   webhook: [
     {
@@ -59,6 +75,13 @@ const REGISTRATIONS = Object.freeze({
       names: ["BATCH_ROWS", "sql"],
       beforeSource: "./errors/database.error",
     },
+    {
+      // The inbox claim and the lite event bus call the functions of their migration (R334).
+      file: "be/src/modules/platform/database/index.ts",
+      source: "./database.client",
+      names: ["callDbFunction"],
+      beforeSource: "./primary.decorators",
+    },
     {
       // PRIMARY_ENTITY_MANAGER: the token the inbox service's unit spec provides its EntityManager double under.
       file: "be/src/modules/platform/database/index.ts",

package/src/main.mjs

@@ -135,6 +135,7 @@ starci app secret remove <env> <slug> [--execute] [--force-nonempty] [--cwd <dir
 starci app secret materialize <env> [--cwd <dir>]
 starci app secret list <env> [--cwd <dir>]
 starci app secret verify <env> [--cwd <dir>]
+starci app secret server-key <env> [--rotate | --backup github | --restore github] [--execute] [--cwd <dir>]
 starci app ci secret set <NAME> [--env E | --dependabot] [--repo owner/name] [--cwd <dir>]
 starci app ci var set <NAME> [--env E] [--value v] [--repo owner/name] [--cwd <dir>]
 starci app ci var sync [--execute] [--repo owner/name] [--cwd <dir>]

package/stack/age-install.mjs

@@ -6,7 +6,8 @@ const AGE_VERSION = 'v1.2.1';
 const AGE_URL = `https://github.com/FiloSottile/age/releases/download/${AGE_VERSION}/age-${AGE_VERSION}-linux-amd64.tar.gz`;
 /** sha256 of the official linux/amd64 archive of AGE_VERSION. */
 const AGE_SHA256 = '7df45a6cc87d4da11cc03a539a7470c15b1041ab2b396af088fe9990f7c79d50';
-const INSTALL = { dnf: 'sudo -n dnf install -y age', yum: 'sudo -n yum install -y age', apk: 'sudo -n apk add age', 'apt-get': 'sudo -n apt-get install -y age' };
+/** The install command of each package manager installAge detects; a refusal names them for a host it cannot install on. */
+export const AGE_INSTALL = { dnf: 'sudo -n dnf install -y age', yum: 'sudo -n yum install -y age', apk: 'sudo -n apk add age', 'apt-get': 'sudo -n apt-get install -y age' };
 
 const STATIC_INSTALL = [
   'set -e',
@@ -26,8 +27,8 @@ export function installAge(run, report) {
   if (present(run)) return;
   const manager = packageManager(run);
   let failure = 'no package manager (dnf, yum, apk, apt-get) found';
-  if (Object.hasOwn(INSTALL, manager)) {
-    const packaged = run(INSTALL[manager]);
+  if (Object.hasOwn(AGE_INSTALL, manager)) {
+    const packaged = run(AGE_INSTALL[manager]);
     if (packaged.status === 0 && present(run)) { report.note(`installed age with ${manager}`); return; }
     failure = `${manager} could not install age: ${firstLine(packaged.stderr)}`;
   }

package/stack/context.mjs

@@ -17,6 +17,34 @@ const realSpawn = (command, args, options = {}) => {
   return { status: run.status, stdout: run.stdout ?? '', stderr: run.stderr ?? '', error: run.error };
 };
 
+/** Bytes of a stream's stderr a pipe keeps, for the first line of a failure. */
+const STDERR_KEPT = 4096;
+
+/**
+ * Streams `source`'s stdout into `sink`'s stdin, neither through a shell, and resolves { source, sink, stderr } (exit codes and
+ * the first stderr line of a failure). The bytes pass through the OS pipe only: they are never collected, logged or written to disk.
+ */
+const realPipe = (source, sink) => new Promise((resolve) => {
+  const options = (end, stdio) => ({ env: end.env, cwd: end.cwd, stdio, windowsHide: true });
+  const from = spawn(source.command, source.args, options(source, ['ignore', 'pipe', 'pipe']));
+  const to = spawn(sink.command, sink.args, options(sink, ['pipe', 'ignore', 'pipe']));
+  const codes = {};
+  let stderr = '';
+  const keep = (chunk) => { if (stderr.length < STDERR_KEPT) stderr += String(chunk).slice(0, STDERR_KEPT - stderr.length); };
+  from.stderr.on('data', keep);
+  to.stderr.on('data', keep);
+  to.stdin.on('error', () => {});
+  from.stdout.pipe(to.stdin);
+  const settle = (name) => (code) => {
+    codes[name] = code ?? 1;
+    if ('source' in codes && 'sink' in codes) resolve({ ...codes, stderr: stderr.trim().split(/\r?\n/u)[0] ?? '' });
+  };
+  from.on('error', () => { from.stdout.unpipe(to.stdin); to.stdin.end(); settle('source')(127); });
+  to.on('error', () => settle('sink')(127));
+  from.on('close', settle('source'));
+  to.on('close', settle('sink'));
+});
+
 const realSleep = (ms) => new Promise((done) => { setTimeout(done, ms); });
 
 /**
@@ -28,7 +56,8 @@ const realSleep = (ms) => new Promise((done) => { setTimeout(done, ms); });
  *   sleep, now     await sleep(ms); now() -> epoch ms
  *   stdout, stderr text writers
  *   fetch          the HTTP seam of verify (url, init) -> Response
- *   sops           the secret seam (decrypt(file, format) -> map; decryptBinary(file) -> text); `io.sops` or, on first use, the app's own sops/age tools
+ *   pipe           (source, sink) -> Promise<{ source, sink, stderr }>: one process's stdout streamed into another's stdin (realPipe)
+ *   sops           the secret seam (decrypt(file, format) -> map; decryptBinary(file) -> text); `io.sops`, else on first use `io.openSops()` or the app's own sops/age tools
  */
 export function stackContext(io = {}) {
   const cwd = io.cwd ?? process.cwd();
@@ -46,9 +75,10 @@ export function stackContext(io = {}) {
     stdout: io.stdout ?? ((text) => process.stdout.write(text)),
     stderr: io.stderr ?? ((text) => process.stderr.write(text)),
     fetch: io.fetch ?? globalThis.fetch,
+    pipe: io.pipe ?? realPipe,
     /** The secret seam: the app's own sops/age tools, created on first use so verbs that open no custody never need them. */
     get sops() {
-      sops ??= defaultSops(env, cwd);
+      sops ??= io.openSops?.() ?? defaultSops(env, cwd);
       return sops;
     },
   });

package/stack/remote.mjs

@@ -1,5 +1,6 @@
 // remote.mjs - how the stack verbs reach the Swarm manager: the ssh runner, remote command quoting, the CI transport setup.
-//   sshRunner(ctx, config, { sshCmd })  -> run(command, { input }) -> { status, stdout, stderr }   (`.cleanup()`, `.notes`)
+//   sshRunner(ctx, config, { sshCmd })  -> run(command, { input }) -> { status, stdout, stderr }   (`.cleanup()`, `.notes`, `.argv(command)`:
+//                                          the { command, args, env } a caller spawns itself to stream the remote stdout)
 //   transports, in this order: --ssh-cmd "ssh vps" (no shell) -> the owner-machine session opened from the sealed members
 //   (only when config.remote.sealed is declared and STARCI_STACK_SEALED_SSH=1) -> the ssh alias config.remote.host (CI after `connect`,
 //   or the operator's own ssh config).
@@ -53,7 +54,8 @@ function makeRunner(ctx, argv, { cleanup = noop, notes = [], connection } = {})
     if (result.error) return { status: 127, stdout: '', stderr: `cannot run ${command}: ${result.error.message}` };
     return { status: result.status ?? 1, stdout: result.stdout, stderr: result.stderr };
   };
-  return Object.assign(run, { cleanup, notes });
+  const argvOf = (remote) => (connection ? connection.remoteArgv(lfOnly(remote)) : { command, args: [...head, lfOnly(remote)], env: ctx.env });
+  return Object.assign(run, { cleanup, notes, argv: argvOf });
 }
 
 function sshCommandArgv(sshCmd) {

package/stack/server-key.mjs

@@ -0,0 +1,296 @@
+// server-key.mjs - `starci app secret server-key <env> [--rotate | --backup github | --restore github] [--execute]`: the deploy
+// server's own age identity, made ON the server and never leaving it, and the environment sealed to it (ruling
+// app-secrets-one-standard, contexts/project/stacks.md PROJECT-STACKS-5).
+//   (none)     the server keeps SERVER_AGE_IDENTITY when it exists, else age-keygen makes it there (umask 077, mode 600, owner
+//              root); only `age-keygen -y` (the public recipient) comes back. Locally the recipient joins the environment's rule
+//              beside the owner key, is recorded as deploy.remote.serverRecipient, and every member is wrapped again (sops
+//              updatekeys: the owner identity opens the data key only, no value is decrypted).
+//   --rotate   a new identity is staged at <identity>.next, the members are wrapped to it, the server swaps it in keeping the old
+//              file as <identity>.retired-<UTC stamp> until the owner deletes it, then the old recipient leaves the rule.
+//   --backup github   the identity streams from the server's ssh stdout into `gh secret set SOPS_AGE_KEY --env <env>` of the
+//              app's origin repository: the protected GitHub Environment <env> holds it as SOPS_AGE_KEY, the secret every CI job
+//              bound to <env> (environment: <env>) reads to open that environment's members; never printed, held or written locally.
+//   --restore github  dispatches the managed .github/workflows/stack-key-restore.yml, which writes that secret back to the server
+//              only when its public key is the recorded deploy.remote.serverRecipient.
+// Both GitHub actions warn when the repository itself holds a SOPS_AGE_KEY secret (names only): a job bound to no environment
+// reads it, and the owner key never belongs in GitHub.
+// The server is reached only through the deploy's own transport (remote.mjs withRunner): the same host, user and key source.
+// Every effect is a dry run until --execute.
+import path from 'node:path';
+import { parseYaml } from '../runtime/engine/yaml.mjs';
+import { readFileSync } from 'node:fs';
+import { configuredRecipients, environmentRecipients } from '../runtime/scripts/hfs/secret-init.mjs';
+import { recordServerRecipient } from '../runtime/scripts/hfs/secret-standard.mjs';
+import { sealedFiles } from '../runtime/scripts/hfs/secret-store.mjs';
+import { github } from '../ci/github.mjs';
+import { AGE_INSTALL, installAge } from './age-install.mjs';
+import { STACKS_DIR, STACKS_FILE, loadDeployConfig } from './config.mjs';
+import { firstLine } from './context.mjs';
+import { failure, refusal } from './errors.mjs';
+import { shellQuote, withRunner } from './remote.mjs';
+
+/**
+ * The deploy server's age identity: sops' default key file of root, so a decrypt on the server reads it with nothing set, and
+ * SOPS_AGE_KEY_FILE names it where one is set. The one owner of this path; the restore workflow is rendered from it.
+ */
+export const SERVER_AGE_IDENTITY = '/root/.config/sops/age/keys.txt';
+
+/** The managed workflow `--restore github` dispatches (templates/app/ci-workflows*). */
+export const RESTORE_WORKFLOW = 'stack-key-restore.yml';
+
+const RECIPIENT = /^age1[a-z0-9]+$/u;
+/** A command run as root: directly when the deploy user is root, else over passwordless sudo, the script read from stdin. */
+const AS_ROOT = 'if [ "$(id -u)" -eq 0 ]; then sh -s; else sudo -n sh -s; fi';
+
+/**
+ * The GitHub Environment secret that holds an environment's server identity. It carries the name the deploy workflows read
+ * (secrets.SOPS_AGE_KEY), scoped to the Environment <env>: for a job bound to <env> it overrides a repository secret of that name.
+ */
+export const BACKUP_SECRET = 'SOPS_AGE_KEY';
+
+/** The shell that keeps `file` when it is an ordinary file, else makes it with age-keygen (umask 077), then prints only its public key. */
+export function ensureIdentityScript(file) {
+  const quoted = shellQuote(file);
+  return [
+    'set -eu',
+    'umask 077',
+    `f=${quoted}`,
+    'if [ -e "$f" ] || [ -L "$f" ]; then',
+    '  if [ -L "$f" ] || [ ! -f "$f" ]; then echo "the identity path is not an ordinary file" >&2; exit 4; fi',
+    '  echo "identity kept" >&2',
+    'else',
+    '  mkdir -p "$(dirname "$f")"',
+    '  age-keygen -o "$f" >/dev/null 2>&1',
+    '  echo "identity created" >&2',
+    'fi',
+    'chown root:root "$f"',
+    'chmod 600 "$f"',
+    'age-keygen -y "$f"',
+  ].join('\n');
+}
+
+/** The shell that prints only the public key of `file`, or exits 5 when there is none. */
+export function publicKeyScript(file) {
+  return ['set -eu', `f=${shellQuote(file)}`, '[ -f "$f" ] || exit 5', 'age-keygen -y "$f"'].join('\n');
+}
+
+/** The shell that swaps the staged `<file>.next` in, keeping the old identity as `retired`, then prints only the new public key. */
+export function swapIdentityScript(file, retired) {
+  return [
+    'set -eu',
+    'umask 077',
+    `f=${shellQuote(file)}`,
+    `r=${shellQuote(retired)}`,
+    'n="$f.next"',
+    'if [ ! -f "$n" ] || [ -L "$n" ]; then echo "no staged identity" >&2; exit 4; fi',
+    'if [ -e "$r" ]; then echo "the retired name is taken" >&2; exit 4; fi',
+    'mv "$f" "$r"',
+    'mv "$n" "$f"',
+    'chmod 600 "$f" "$r"',
+    'age-keygen -y "$f"',
+  ].join('\n');
+}
+
+/** The one remote command that streams `file` to stdout as root; only --backup github runs it, into the stdin of gh. */
+export const streamIdentityCommand = (file) =>
+  `if [ "$(id -u)" -eq 0 ]; then cat -- ${shellQuote(file)}; else sudo -n cat -- ${shellQuote(file)}; fi`;
+
+/** The single public recipient a server script printed; anything else is refused without echoing a byte of it. */
+function recipientOf(result, what) {
+  const text = String(result.stdout ?? '').trim();
+  if (result.status !== 0) throw failure('STACK_SERVER_KEY_REMOTE', `${what} failed on the server (exit ${result.status}): ${firstLine(result.stderr)}`);
+  if (!RECIPIENT.test(text)) throw failure('STACK_SERVER_KEY_OUTPUT', `${what} printed something other than one public age recipient; none of it is shown`);
+  return text;
+}
+
+/** The raw `environments.<env>.deploy.remote` block: the deploy's own transport, which this verb never invents. */
+function declaredRemote(repoRoot, env) {
+  const declaration = parseYaml(readFileSync(path.join(repoRoot, STACKS_DIR, STACKS_FILE), 'utf8'));
+  const remote = declaration?.environments?.[env]?.deploy?.remote;
+  if (!remote || typeof remote !== 'object' || (remote.host === undefined && remote.sealed === undefined))
+    throw refusal('STACK_SERVER_KEY_NO_REMOTE', `environments.${env}.deploy.remote of ${STACKS_FILE} declares no transport: add host (the ssh alias the deploy uses) or sealed (ops, sshKey, knownHosts); server-key reaches the server only the way the deploy does`);
+  return remote;
+}
+
+/** Ensures age-keygen on the server: stack prepare's installer owns the install (with --execute); without it the exact commands. */
+function ensureTool(run, execute, lines) {
+  if (run('command -v age-keygen').status === 0) return true;
+  const install = `install age on the server (one of: ${Object.values(AGE_INSTALL).join('; ')})`;
+  if (!execute) { lines.push(`dry run: age-keygen is missing on the server; --execute installs it with stack prepare's installer, or ${install}`); return false; }
+  const problems = [];
+  installAge(run, { note: (text) => lines.push(text), bad: (text) => problems.push(text) });
+  if (problems.length || run('command -v age-keygen').status !== 0)
+    throw refusal('STACK_SERVER_KEY_NO_AGE', `age-keygen is not on the server and could not be installed (${problems.join('; ') || 'still missing'}): ${install}`);
+  return true;
+}
+
+/** Wraps every member of the environment again to its rule (sops updatekeys); names only. */
+function reseal(config, seam, lines) {
+  const members = sealedFiles(config.secretsDir);
+  for (const name of members) seam.updateKeys(path.join(config.secretsDir, name));
+  lines.push(members.length ? `wrapped ${members.length} member${members.length === 1 ? '' : 's'} of ${config.env} again: ${members.join(', ')}` : `${config.env} has no sealed member yet`);
+}
+
+/** Puts `add` on the environment's rule and takes `remove` off it, every other recipient (the owner key) kept, then reseals. */
+function sealTo(ctx, config, { add = [], remove = [] }, lines) {
+  try {
+    environmentRecipients(config.repoRoot, config.env, { add, remove, owner: () => { throw new Error('owner is not named here'); }, stdout: (text) => lines.push(text.trimEnd()) });
+  } catch (error) {
+    throw refusal('STACK_SERVER_KEY_RULE', error.message);
+  }
+  reseal(config, ctx.sops, lines);
+}
+
+/** The environment's rule must already name a recipient other than the server's (the owner key), which every seal keeps. */
+function requireOwnerRecipient(config, server = []) {
+  let rule = [];
+  try { rule = configuredRecipients(config.repoRoot, path.join(config.secretsDir, 'member.enc.yaml')); } catch { rule = []; }
+  if (!rule.some((recipient) => !server.includes(recipient)))
+    throw refusal('STACK_SERVER_KEY_NO_OWNER', `the ${config.env} rule of .sops.yaml names no owner key, so the server key would be its only recipient: starci app secret recipients ${config.env} --add owner`);
+}
+
+/** The members need the owner identity to be wrapped again: refused before the server is touched. */
+function requireOwnerIdentity(ctx, config) {
+  const [first] = sealedFiles(config.secretsDir);
+  if (first) ctx.sops.requireIdentity?.(path.join(config.secretsDir, first));
+}
+
+const asRoot = (run, script) => run(AS_ROOT, { input: script });
+
+const retiredName = (ctx) => `${SERVER_AGE_IDENTITY}.retired-${new Date(ctx.now()).toISOString().replaceAll('-', '').replaceAll(':', '').slice(0, 15)}Z`;
+
+async function ensureKey(ctx, config, { execute }, deps) {
+  return withRunner(ctx, config, {}, deps, async (run, notes) => {
+    const lines = [...notes];
+    const tool = ensureTool(run, execute, lines);
+    if (!execute) {
+      const current = tool ? asRoot(run, publicKeyScript(SERVER_AGE_IDENTITY)) : { status: 5 };
+      if (current.status === 5) lines.push(`dry run: would create ${SERVER_AGE_IDENTITY} on the server (age-keygen, umask 077, mode 600, owner root) and read back only its public key`);
+      else lines.push(`dry run: the server holds ${SERVER_AGE_IDENTITY} (${recipientOf(current, 'age-keygen -y')}); it is kept`);
+      lines.push(`dry run: would add that recipient to the ${config.env} rule of .sops.yaml beside the owner key, record it as deploy.remote.serverRecipient and wrap every member of ${config.env} again (--execute)`);
+      return { ok: true, lines, data: { env: config.env, identity: SERVER_AGE_IDENTITY, execute } };
+    }
+    requireOwnerIdentity(ctx, config);
+    requireOwnerRecipient(config, [config.remote.serverRecipient].filter(Boolean));
+    const made = asRoot(run, ensureIdentityScript(SERVER_AGE_IDENTITY));
+    const recipient = recipientOf(made, 'age-keygen');
+    lines.push(`${String(made.stderr).includes('identity created') ? 'created' : 'kept'} ${SERVER_AGE_IDENTITY} on the server; public recipient ${recipient}`);
+    sealTo(ctx, config, { add: [recipient] }, lines);
+    recordServerRecipient(config.repoRoot, config.env, recipient);
+    lines.push(`recorded environments.${config.env}.deploy.remote.serverRecipient: ${recipient}`);
+    return { ok: true, lines, data: { env: config.env, identity: SERVER_AGE_IDENTITY, recipient, execute } };
+  });
+}
+
+async function rotateKey(ctx, config, { execute }, deps) {
+  return withRunner(ctx, config, {}, deps, async (run, notes) => {
+    const lines = [...notes];
+    if (!ensureTool(run, execute, lines)) return { ok: true, lines, data: { env: config.env, identity: SERVER_AGE_IDENTITY, execute } };
+    const current = asRoot(run, publicKeyScript(SERVER_AGE_IDENTITY));
+    if (current.status === 5) throw refusal('STACK_SERVER_KEY_NONE', `the server holds no ${SERVER_AGE_IDENTITY} to rotate: starci app secret server-key ${config.env} --execute`);
+    const old = recipientOf(current, 'age-keygen -y');
+    const retired = retiredName(ctx);
+    if (!execute) {
+      lines.push(`dry run: would stage a new identity at ${SERVER_AGE_IDENTITY}.next, wrap every member of ${config.env} to it beside ${old} and the owner key, swap it in keeping the old file as ${retired}, then remove ${old} from the rule (--execute)`);
+      return { ok: true, lines, data: { env: config.env, identity: SERVER_AGE_IDENTITY, previous: old, execute } };
+    }
+    requireOwnerIdentity(ctx, config);
+    requireOwnerRecipient(config, [old]);
+    const next = recipientOf(asRoot(run, ensureIdentityScript(`${SERVER_AGE_IDENTITY}.next`)), 'age-keygen');
+    lines.push(`staged ${SERVER_AGE_IDENTITY}.next on the server; public recipient ${next}`);
+    sealTo(ctx, config, { add: [next] }, lines);
+    recordServerRecipient(config.repoRoot, config.env, next);
+    const swapped = recipientOf(asRoot(run, swapIdentityScript(SERVER_AGE_IDENTITY, retired)), 'the identity swap');
+    if (swapped !== next) throw failure('STACK_SERVER_KEY_SWAP', `the server's identity reads ${swapped} after the swap, not the staged ${next}`);
+    lines.push(`swapped the new identity in; the old one is kept on the server as ${retired} until the owner deletes it`);
+    sealTo(ctx, config, { remove: old === next ? [] : [old] }, lines);
+    lines.push(`recorded environments.${config.env}.deploy.remote.serverRecipient: ${next}; ${old} left the ${config.env} rule`);
+    return { ok: true, lines, data: { env: config.env, identity: SERVER_AGE_IDENTITY, recipient: next, previous: old, retired, execute } };
+  });
+}
+
+/**
+ * The GitHub Environment that holds the backup, refused unless it exists in the app's origin repository and is protected: required
+ * reviewers, or deployment branches limited to main. Names what is missing.
+ */
+export function protectedEnvironment(gh, env) {
+  let repo;
+  try { repo = gh.repository(); } catch { throw refusal('STACK_SERVER_KEY_GITHUB', 'the app\'s origin repository cannot be read with gh (gh repo view); authenticate gh in the app checkout'); }
+  let environment;
+  try { environment = gh.json(['api', `repos/${repo}/environments/${env}`]); } catch {
+    throw refusal('STACK_SERVER_KEY_GITHUB_ENVIRONMENT', `the GitHub Environment ${env} of ${repo} does not exist or cannot be read; create it with required reviewers or deployment branches limited to main`);
+  }
+  const reviewers = (environment?.protection_rules ?? []).some((rule) => rule?.type === 'required_reviewers' && (rule.reviewers ?? []).length > 0);
+  let mainOnly = false;
+  if (environment?.deployment_branch_policy?.custom_branch_policies === true) {
+    const policies = gh.json(['api', `repos/${repo}/environments/${env}/deployment-branch-policies`])?.branch_policies ?? [];
+    mainOnly = policies.length > 0 && policies.every((policy) => policy?.name === 'main' && (policy.type ?? 'branch') === 'branch');
+  }
+  if (!reviewers && !mainOnly)
+    throw refusal('STACK_SERVER_KEY_UNPROTECTED', `the GitHub Environment ${env} of ${repo} has no protection rule: it has no required reviewers and its deployment branches are not limited to main; add either before it holds the server identity`);
+  return repo;
+}
+
+/**
+ * The warning lines about a repository-level SOPS_AGE_KEY of `repo`, read by name only (gh api repos/<repo>/actions/secrets): a job
+ * bound to no Environment, or to one without its own SOPS_AGE_KEY, reads that one. Empty when the repository holds none.
+ */
+export function repositoryKeyWarnings(gh, repo) {
+  let names;
+  try {
+    names = (gh.json(['api', `repos/${repo}/actions/secrets?per_page=100`])?.secrets ?? []).map((secret) => secret?.name);
+  } catch {
+    return [`warning: the repository secret names of ${repo} cannot be read (gh api repos/${repo}/actions/secrets needs admin access); make sure the repository holds no ${BACKUP_SECRET} of its own`];
+  }
+  if (!names.includes(BACKUP_SECRET)) return [];
+  return [`warning: ${repo} holds a repository secret ${BACKUP_SECRET}; a job bound to no Environment, or to one without its own ${BACKUP_SECRET}, reads it. Once every environment holds its server key (server-key <env> --backup github --execute), delete it: gh secret delete ${BACKUP_SECRET} --repo ${repo}. The owner key never belongs in GitHub.`];
+}
+
+async function backupKey(ctx, config, { execute }, deps) {
+  const gh = deps.github ?? github({ cwd: config.repoRoot, env: ctx.env });
+  const repo = protectedEnvironment(gh, config.env);
+  const warnings = repositoryKeyWarnings(gh, repo);
+  const sinkArgs = ['secret', 'set', BACKUP_SECRET, '--env', config.env, '--repo', repo];
+  const plan = `stream ${SERVER_AGE_IDENTITY} from the server's ssh stdout into gh ${sinkArgs.join(' ')} (the ${config.env} Environment's ${BACKUP_SECRET}, which every job bound to ${config.env} reads; never printed, held or written locally)`;
+  const data = { env: config.env, repo, secret: BACKUP_SECRET, repositorySecret: warnings.length > 0, execute };
+  if (!execute) return { ok: true, lines: [`dry run: would ${plan} (--execute)`, ...warnings], data };
+  return withRunner(ctx, config, {}, deps, async (run, notes) => {
+    const source = run.argv(streamIdentityCommand(SERVER_AGE_IDENTITY));
+    const sink = deps.ghCommand ?? { command: 'gh', args: [] };
+    const piped = await ctx.pipe(source, { command: sink.command, args: [...sink.args, ...sinkArgs], env: ctx.env, cwd: config.repoRoot });
+    if (piped.source !== 0) throw failure('STACK_SERVER_KEY_BACKUP', `the server could not stream ${SERVER_AGE_IDENTITY} (exit ${piped.source}); nothing was stored`);
+    if (piped.sink !== 0) throw failure('STACK_SERVER_KEY_BACKUP', `gh secret set ${BACKUP_SECRET} failed (exit ${piped.sink}): ${piped.stderr}`);
+    return { ok: true, lines: [...notes, `stored the server identity of ${config.env} as the secret ${BACKUP_SECRET} of the GitHub Environment ${config.env} of ${repo}`, ...warnings], data };
+  });
+}
+
+async function restoreKey(ctx, config, { execute }, deps) {
+  const recipient = config.remote.serverRecipient;
+  if (!recipient)
+    throw refusal('STACK_SERVER_KEY_NO_RECIPIENT', `environments.${config.env}.deploy.remote records no serverRecipient, so the restore cannot tell the server key from another identity: starci app secret server-key ${config.env} --execute records it`);
+  const gh = deps.github ?? github({ cwd: config.repoRoot, env: ctx.env });
+  const repo = protectedEnvironment(gh, config.env);
+  const warnings = repositoryKeyWarnings(gh, repo);
+  const args = ['workflow', 'run', RESTORE_WORKFLOW, '--repo', repo, '-f', `environment=${config.env}`, '-f', `recipient=${recipient}`];
+  const data = { env: config.env, repo, recipient, repositorySecret: warnings.length > 0, execute };
+  if (!execute) return { ok: true, lines: [`dry run: would run gh ${args.join(' ')}; the workflow writes the ${config.env} Environment's ${BACKUP_SECRET} to ${SERVER_AGE_IDENTITY} on the server when none is there and its public key is ${recipient} (--execute)`, ...warnings], data };
+  try { gh.run(args); } catch { throw failure('STACK_SERVER_KEY_RESTORE', `gh could not dispatch ${RESTORE_WORKFLOW} in ${repo}; is the managed workflow rendered (starci app sync --write) and pushed?`); }
+  return { ok: true, lines: [`dispatched ${RESTORE_WORKFLOW} in ${repo} for ${config.env}; follow it with gh run list --workflow ${RESTORE_WORKFLOW} --repo ${repo}`, ...warnings], data };
+}
+
+/**
+ * Runs one server-key action. `options`: { execute, rotate, backup, restore };
+ * `deps` (specs): { ssh (an injected runner), github (the gh port of ci/github.mjs), ghCommand ({ command, args } the pipe's sink) }.
+ */
+export async function serverKey(ctx, env, options, deps = {}) {
+  const config = loadDeployConfig({ repoRoot: ctx.cwd, env });
+  declaredRemote(ctx.cwd, env);
+  for (const [flag, value] of [['backup', options.backup], ['restore', options.restore]])
+    if (value !== undefined && value !== 'github') throw refusal('STACK_USAGE', `--${flag} takes github`);
+  if ([options.rotate === true, options.backup !== undefined, options.restore !== undefined].filter(Boolean).length > 1)
+    throw refusal('STACK_USAGE', '--rotate, --backup and --restore are separate actions; pass one');
+  if (options.backup) return backupKey(ctx, config, options, deps);
+  if (options.restore) return restoreKey(ctx, config, options, deps);
+  if (options.rotate) return rotateKey(ctx, config, options, deps);
+  return ensureKey(ctx, config, options, deps);
+}

package/stack/ssh-session.mjs

@@ -136,6 +136,12 @@ export function createSshSession(ctx, { argv, env = ctx.env, controlPath, socket
     return ctx.spawn(command, [...controls, ...head, remoteCommand], { ...options, env });
   }
 
+  /** The argv `remote` would spawn, for a caller that streams the command's stdout itself ({ command, args, env }). */
+  function remoteArgv(remoteCommand) {
+    if (!windows) start();
+    return { command, args: [...controls, ...head, remoteCommand], env };
+  }
+
   function dockerHost(alias) {
     start();
     return windows ? host : `ssh://${alias}`;
@@ -169,7 +175,7 @@ export function createSshSession(ctx, { argv, env = ctx.env, controlPath, socket
     }
   }
 
-  return { remote, dockerHost, close };
+  return { remote, remoteArgv, dockerHost, close };
 }
 
 /** A short private directory for an invocation whose SSH credentials already live in the operator or CI HOME. */

package/sync/index.mjs

@@ -41,6 +41,7 @@ import { LIST } from '../runtime/scripts/lib/loosening-config.mjs';
 import { readTruths } from '../runtime/scripts/lib/loosening-state.mjs';
 import { parseYaml as bundledParseYaml } from '../runtime/engine/yaml.mjs';
 import { bootstrapSection } from '../runtime/scripts/hfs/rules/database-constants.mjs';
+import { SERVER_AGE_IDENTITY } from '../stack/server-key.mjs';
 
 export const TEMPLATES_DIR = path.join(import.meta.dirname, '..', 'templates');
 const NODE_MAJOR = 22;
@@ -412,6 +413,8 @@ export function variables(app, scope, presets, sonarKey, manifest = loadSlotMani
     // The typed lint reads the types the fe packages declare (their dist, turbo.json: lint dependsOn ^build), so the packages are built first.
     lintPrefix: packages ? 'npm run codegen --silent && turbo run build --filter=./fe/packages/* && ' : 'npm run codegen --silent && ',
     nodeMajor: String(NODE_MAJOR),
+    // The deploy server's age identity the restore workflow writes (stack/server-key.mjs owns the path).
+    serverAgeIdentity: SERVER_AGE_IDENTITY,
     sonarKey: sonarKey ?? app.project,
     sonarHostLine: sonarHostLine(quality.sonar),
     sonarExclusions: sonarScope(presets).exclusions.join(','),

package/templates/app/ci-workflows/github/workflows/images.yml

@@ -1,6 +1,8 @@
 {{header}}
 # Builds the image of every app on its release tag, so a Dockerfile that no longer builds fails that release.
 # It never pushes: publishing an image is the product's decision, not this file's.
+# When an environment declares images, the job is bound to that protected GitHub environment and opens its sealed build
+# arguments with that environment's SOPS_AGE_KEY (its server identity, starci app secret server-key <env> --backup github).
 name: images
 
 on:

package/templates/app/ci-workflows/github/workflows/stack-deploy.yml

@@ -7,7 +7,9 @@
 # operation rollback  redeploys a stored release; "release" is 16 lowercase hex characters, empty means the previous accepted release
 # The environment input names a protected GitHub environment (required reviewers) that holds:
 #   vars    VPS_HOST, VPS_USER (the swarm manager); a variable named in deploy.passEnv, when the stack interpolates one
-#   secrets SOPS_AGE_KEY (the age identity of .sops.yaml), VPS_SSH_KEY (CI-only deploy key), VPS_KNOWN_HOSTS (pinned host keys)
+#   secrets SOPS_AGE_KEY (that environment's server identity, stored there by starci app secret server-key <env> --backup github;
+#           both jobs are bound to the environment, so they read it and never a repository secret of that name), VPS_SSH_KEY
+#           (CI-only deploy key), VPS_KNOWN_HOSTS (pinned host keys)
 # Nothing else is a secret: images are pushed with the workflow token, which also reads the CI verdict of the tag's commit a production deploy
 # waits for (requireCi deploy.prod). Dispatch inputs reach scripts only through env.
 name: deploy stack