@starci/hfs 4.12.0 → 4.13.0

package/stack/journal.mjs

@@ -0,0 +1,182 @@
+// journal.mjs - the record of every deploy attempt on the manager, and the lock that serialises attempts.
+// A deploy (CI, dispatch or the owner's urgent one), a rollback and an accept each run as one ATTEMPT:
+//   1. the journal is opened: /srv/<stack>/deploys/<attempt>.json (the document, rewritten at every phase boundary, so a cancelled job still leaves its
+//      last phase) and one line in the append-only /srv/<stack>/deploys.jsonl; a journal that cannot be written refuses the attempt before anything changed;
+//   2. the deploy lock /srv/<stack>/deploy.lock is taken (a phase of the journal, `lock`): the wait is bounded and a held lock is a refusal
+//      `STACK_DEPLOY_LOCKED` with nothing changed; the lock is refreshed at every phase start and released last;
+//   3. the body runs, every phase through `phase()`: start, end and verdict are written, a thrown step is recorded failed and rethrown;
+//   4. the outcome (accepted, pending, rolled-back, rollback-failed, failed, refused, already-current) closes the document.
+// The document holds names, digests, times and verdicts, never a secret value. After the first write a failing journal never fails the deploy: the failure is
+// a line of the result (`warnings`).
+import { randomBytes } from 'node:crypto';
+import { firstLine } from './context.mjs';
+import { StackError, refusal } from './errors.mjs';
+const REASON_MAX = 500;
+const SAFE = /[^A-Za-z0-9._/:@ -]/gu;
+const isoOf = (ctx) => new Date(ctx.now()).toISOString();
+const compact = (iso) => iso.replaceAll(/[-:]|\.\d+/gu, '');
+const clean = (value, max) => value.replaceAll(SAFE, '?').slice(0, max);
+/** A new attempt id: the UTC time and 8 random hex characters. */
+export const newAttemptId = (ctx) => `${compact(isoOf(ctx))}-${randomBytes(4).toString('hex')}`;
+/** The CI facts of the runner environment: the tag of a tag run and the run's URL. Nothing else of the environment is read. */
+function ciFacts(ctx, run) {
+    const { GITHUB_REF_TYPE: refType, GITHUB_REF_NAME: ref, GITHUB_SERVER_URL: server, GITHUB_REPOSITORY: repository } = ctx.env;
+    const tag = refType === 'tag' && ref ? clean(ref, 100) : null;
+    if (!run)
+        return { tag, ci: null };
+    const url = server && repository && /^https:\/\/[A-Za-z0-9.-]+$/u.test(server) && /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/u.test(repository) ? `${server}/${repository}/actions/runs/${run}` : null;
+    return { tag, ci: { run, url } };
+}
+/**
+ * Opens the journal of an attempt and returns its recorder. Writes the first document and the index line; a failure of either is the refusal
+ * `STACK_DEPLOY_JOURNAL` (the manager cannot keep a record, so nothing is changed).
+ */
+export async function openAttempt(ctx, config, store, meta) {
+    const id = newAttemptId(ctx);
+    const journal = store.journal(id);
+    const facts = ciFacts(ctx, meta.ciRun);
+    const document = {
+        schema: 'starci/stack-deploy-attempt@1', attempt: id, kind: meta.kind, stack: config.stack, env: config.env, startedAt: isoOf(ctx), endedAt: null,
+        trigger: meta.trigger, sourceSha: meta.sourceSha ?? null, tag: facts.tag, ci: facts.ci,
+        urgent: meta.urgent ? { role: clean(meta.urgent.role, 40), reason: meta.urgent.reason.slice(0, REASON_MAX), tag: meta.urgent.tag ?? null } : null,
+        releaseId: null, previous: null, phases: [], backup: null, migrations: [], observed: {}, outcome: 'running', error: null,
+    };
+    const warnings = [];
+    let beat = async () => { };
+    const save = async () => {
+        try {
+            await journal.write(document);
+        }
+        catch (error) {
+            warnings.push(`journal: could not write ${id}: ${firstLine(error.message)}`);
+        }
+    };
+    try {
+        await journal.write(document);
+        await journal.index({ attempt: id, at: document.startedAt, event: 'start', kind: meta.kind, trigger: meta.trigger, sourceSha: document.sourceSha });
+    }
+    catch (error) {
+        throw refusal('STACK_DEPLOY_JOURNAL', `the manager cannot keep the deploy journal (${firstLine(error.message)}); nothing was changed`);
+    }
+    const recorder = {
+        id, warnings,
+        beforePhase: (next) => { beat = next; },
+        note(found) {
+            if (found.releaseId !== undefined)
+                document.releaseId = found.releaseId;
+            if (found.previous !== undefined)
+                document.previous = found.previous;
+            if (found.backup !== undefined)
+                document.backup = found.backup;
+            if (found.migrations !== undefined)
+                document.migrations = found.migrations;
+            if (found.urgent)
+                document.urgent = { role: clean(found.urgent.role, 40), reason: found.urgent.reason.slice(0, REASON_MAX), tag: found.urgent.tag ?? null };
+        },
+        async phase(name, action, passed = () => true) {
+            const entry = { phase: name, startedAt: isoOf(ctx), endedAt: null, verdict: 'running' };
+            document.phases.push(entry);
+            await beat();
+            await save();
+            try {
+                const value = await action();
+                entry.verdict = passed(value) ? 'ok' : 'failed';
+                return value;
+            }
+            catch (error) {
+                entry.verdict = 'failed';
+                throw error;
+            }
+            finally {
+                entry.endedAt = isoOf(ctx);
+                await save();
+            }
+        },
+        snapshot(moment, services) { document.observed[moment] = services; },
+        async finish(outcome, error = null) {
+            document.outcome = outcome;
+            document.endedAt = isoOf(ctx);
+            document.error = error;
+            await save();
+            try {
+                await journal.index({ attempt: id, at: document.endedAt, event: 'end', outcome, releaseId: document.releaseId });
+            }
+            catch (indexError) {
+                warnings.push(`journal: could not index ${id}: ${firstLine(indexError.message)}`);
+            }
+        },
+    };
+    return recorder;
+}
+/** The default wait for a held deploy lock (the design's ten minutes). */
+export const LOCK_WAIT_SECONDS = 600;
+const LOCK_POLL_SECONDS = 10;
+/** Takes the deploy lock, waiting while another deploy holds it; `STACK_DEPLOY_LOCKED` when the wait ends. */
+export async function holdLock(ctx, store, trigger, wait = {}) {
+    const token = randomBytes(16).toString('hex');
+    const deadline = ctx.now() + (wait.waitSeconds ?? LOCK_WAIT_SECONDS) * 1000;
+    for (;;) {
+        const attempt = await store.tryLock(token, trigger);
+        if (attempt.acquired)
+            return { token, refresh: () => store.refreshLock(token), release: () => store.unlock(token) };
+        if (ctx.now() >= deadline)
+            throw refusal('STACK_DEPLOY_LOCKED', `another deploy (${attempt.holder}) has held the deploy lock for ${attempt.ageSeconds}s without finishing; nothing was changed. If its job is gone, remove the deploy.lock directory under the releases base on the manager`);
+        await ctx.sleep((wait.pollSeconds ?? LOCK_POLL_SECONDS) * 1000);
+    }
+}
+/** How a finished result reads as an outcome. */
+export function outcomeOf(result, kind) {
+    const data = (result.data ?? {});
+    if (data.alreadyCurrent)
+        return 'already-current';
+    if (result.ok)
+        return data.pendingAccept ? 'pending' : 'accepted';
+    if (kind !== 'deploy')
+        return 'failed';
+    const changed = (data.steps ?? []).some((entry) => entry.step === 'deploy' || entry.step === 'deploy-hold');
+    if (!changed)
+        return 'refused';
+    return data.rolledBack ? 'rolled-back' : 'rollback-failed';
+}
+/** A thrown value as the { code, message } of a journal document. */
+function errorOf(error) {
+    if (error instanceof StackError)
+        return { code: error.code, message: firstLine(error.message).slice(0, 300) };
+    return { code: 'UNEXPECTED', message: firstLine(error?.message).slice(0, 300) };
+}
+/**
+ * Runs `body` as one recorded attempt under the deploy lock: journal opened, lock taken, `body(recorder)` run, outcome written, lock released. The result
+ * gains `data.attempt` (the attempt id) and the journal's warnings as lines; a throw is recorded as `refused` and rethrown.
+ */
+export async function recordedAttempt(ctx, config, store, meta, wait, body) {
+    const recorder = await openAttempt(ctx, config, store, meta);
+    let hold = null;
+    try {
+        hold = await recorder.phase('lock', () => holdLock(ctx, store, meta.trigger, wait));
+        const owned = hold;
+        recorder.beforePhase(async () => {
+            try {
+                if (!await owned.refresh())
+                    recorder.warnings.push('journal: the deploy lock is no longer ours (taken over as stale)');
+            }
+            catch {
+                recorder.warnings.push('journal: could not refresh the deploy lock');
+            }
+        });
+        const result = await body(recorder);
+        await recorder.finish(outcomeOf(result, meta.kind), result.refused ? { code: result.refused.code, message: result.refused.message.slice(0, 300) } : null);
+        return { ...result, lines: [...result.lines, ...recorder.warnings], data: { ...result.data, attempt: recorder.id } };
+    }
+    catch (error) {
+        await recorder.finish('refused', errorOf(error));
+        throw error;
+    }
+    finally {
+        if (hold) {
+            try {
+                await hold.release();
+            }
+            catch { /* the lock is taken over as stale after its TTL; the deploy's own verdict stands */ }
+        }
+    }
+}

package/stack/manager-backup.d.mts

@@ -0,0 +1,54 @@
+import type { BackupReceipt } from './db-phases.mjs';
+import type { DeployConfig } from './config.mjs';
+import type { StackContext } from './context.mjs';
+import type { Runner } from './remote.mjs';
+/** What was measured on an archive: whether it exists, its size, whether it starts with the age header and its sha256. */
+export interface ArchiveFacts {
+    exists: boolean;
+    size: number;
+    header: boolean;
+    sha256: string;
+}
+/** A dump the manager made: where the archive and its receipt are, the receipt and the measurements taken after it was written. */
+export interface ManagerBackup {
+    receiptPath: string;
+    receipt: BackupReceipt;
+    archive: string;
+    facts: ArchiveFacts;
+}
+/** The parts of the archive, in tar order. */
+export declare const BACKUP_PARTS: readonly string[];
+/** The folder of the database backups on the manager. */
+export declare function backupFolder(config: Pick<DeployConfig, 'backup' | 'stack'>): string;
+/** The measurements of a local archive (an external database's backup, made where the deploy runs). */
+export declare function localFacts(archive: string): ArchiveFacts;
+/** What the dump script is given. */
+export interface DumpInput {
+    dir: string;
+    file: string;
+    container: string;
+    url: string;
+    recipients: readonly string[];
+}
+/** The one-line shell script that dumps, tars and encrypts; its stdout is: the archive sha256, its byte size, then the row counts JSON. */
+export declare function dumpScript({ dir, file, container, url, recipients }: DumpInput): string;
+/** The script that measures an archive: `exists`, `bytes`, `sha256` and `header` lines. */
+export declare function measureScript(archive: string): string;
+/** The root script of the test decrypt: the manager's own identity opens the archive in a private scratch directory and tar lists it. */
+export declare function decryptScript(archive: string): string;
+/**
+ * Dumps the database of `container` on the manager and writes the receipt beside the archive. Returns the receipt and what the manager measures on
+ * the archive afterwards (a second reading, so the receipt is judged against the file, not against the script that wrote it).
+ */
+export declare function managerBackup(_ctx: StackContext, config: DeployConfig, { run, container, url, recipients, at }: {
+    run: Runner;
+    container: string;
+    url: string;
+    recipients: string[];
+    at: string;
+}): Promise<ManagerBackup>;
+/** Opens the archive on the manager with the manager's own identity and lists its parts. Throws when the identity cannot open it. */
+export declare function managerDecrypt(_ctx: StackContext, { run, archive }: {
+    run: Runner;
+    archive: string;
+}): Promise<string[]>;

package/stack/manager-backup.mjs

@@ -0,0 +1,136 @@
+// manager-backup.mjs - the database backup of a deploy whose database runs in the stack (database.target: stack): the dump is made ON the
+// manager from the database container (pg_dumpall roles, pg_dump schema and data, per-table row counts), tarred and encrypted with age to the
+// .sops.yaml recipients plus the server's own, and kept on the manager beside the volume backups (deploy.backup.dir, else /var/backups/<stack>/db).
+// No plaintext leaves a root-private scratch directory that a trap removes, no database URL or password exists on the machine that runs the
+// deploy (the container reads its own POSTGRES_PASSWORD_FILE), and the receipt is written next to the archive. The archive stays on the manager;
+// copying it to another failure domain remains the owner's job. Every remote command is composed only from validated names and quoted values.
+import fs from 'node:fs';
+import { createHash } from 'node:crypto';
+import { OWN_PASSWORD_SHELL, DB_IDENT } from '../db/postgres.mjs';
+import { RECEIPT_SCHEMA } from '../db/backup-receipt.mjs';
+import { failure, refusal } from './errors.mjs';
+import { shellQuote } from './remote.mjs';
+import { AS_ROOT, SERVER_AGE_IDENTITY } from './server-key.mjs';
+/** The parts of the archive, in tar order. */
+export const BACKUP_PARTS = Object.freeze(['roles.sql', 'schema.sql', 'data.sql', 'rowcounts.json']);
+const BASE = /^\/[A-Za-z0-9._/-]+$/u;
+const CONTAINER = /^[0-9a-f]{12,64}$/u;
+const RECIPIENT = /^age1[0-9a-z]{50,}$/u;
+const AGE_HEADER = 'age-encryption.org/v1';
+/** Row counts of every ordinary table but the catalogs' (auth included), one JSON array, computed inside the database. */
+const ROWCOUNTS_SQL = "select coalesce(json_agg(json_build_object('schema', schemaname, 'table', tablename, 'count', (xpath('/row/c/text()', query_to_xml(format('select count(*) as c from %I.%I', schemaname, tablename), false, true, '')))[1]::text) order by schemaname, tablename), '[]'::json) from pg_catalog.pg_tables where schemaname not in ('pg_catalog', 'information_schema')";
+/** The folder of the database backups on the manager. */
+export function backupFolder(config) {
+    const base = config.backup?.dir ?? `/var/backups/${config.stack}`;
+    const dir = `${base}/db`;
+    if (!BASE.test(dir) || dir.split('/').includes('..'))
+        throw refusal('STACK_DEPLOY_BACKUP_DIR', `the backup folder ${dir} must be an absolute path without ..`);
+    return dir;
+}
+/** The measurements of a local archive (an external database's backup, made where the deploy runs). */
+export function localFacts(archive) {
+    if (!fs.existsSync(archive))
+        return { exists: false, size: 0, header: false, sha256: '' };
+    const bytes = fs.readFileSync(archive);
+    return { exists: true, size: bytes.length, header: bytes.subarray(0, AGE_HEADER.length + 1).toString('ascii') === `${AGE_HEADER}\n`, sha256: createHash('sha256').update(bytes).digest('hex') };
+}
+/** The one-line shell script that dumps, tars and encrypts; its stdout is: the archive sha256, its byte size, then the row counts JSON. */
+export function dumpScript({ dir, file, container, url, recipients }) {
+    if (!CONTAINER.test(container))
+        throw refusal('STACK_DEPLOY_BACKUP_FAILED', 'the database container id is not a plain id');
+    if (recipients.length === 0 || recipients.some((recipient) => !RECIPIENT.test(recipient)))
+        throw refusal('STACK_DEPLOY_BACKUP_FAILED', 'the backup needs at least one age recipient, each a plain age1... key');
+    const final = `${dir}/${file}`;
+    const partial = `${final}.partial`;
+    const [archive, tmp] = [shellQuote(final), shellQuote(partial)];
+    const dx = (...args) => `dx ${args.join(' ')}`;
+    const dump = [
+        dx('pg_dumpall', '--roles-only', '--no-role-passwords', '--dbname', shellQuote(url), '>', '"$S/roles.sql"'),
+        dx('pg_dump', '--schema-only', '--dbname', shellQuote(url), '>', '"$S/schema.sql"'),
+        dx('pg_dump', '--data-only', '--dbname', shellQuote(url), '>', '"$S/data.sql"'),
+        dx('psql', '-X', '-q', '-A', '-t', '-d', shellQuote(url), '-c', shellQuote(ROWCOUNTS_SQL), '>', '"$S/rowcounts.json"'),
+        `tar -cf "$S/backup.tar" -C "$S" ${BACKUP_PARTS.join(' ')}`,
+        `age ${recipients.map((recipient) => `-r ${shellQuote(recipient)}`).join(' ')} -o ${tmp} "$S/backup.tar"`,
+        `mv ${tmp} ${archive}`,
+    ].join(' && ');
+    return [
+        'set -o pipefail', 'umask 077', `mkdir -p ${shellQuote(dir)}`,
+        `S=$(mktemp -d ${shellQuote(`${dir}/.plain.XXXXXX`)})`, "trap 'rm -rf \"$S\"' EXIT",
+        `PW=${shellQuote(OWN_PASSWORD_SHELL)}`,
+        `dx() { docker exec ${container} sh -c "$PW" sh "$@" </dev/null; }`,
+        `{ ${dump}; } || { rm -f ${tmp}; exit 1; }`,
+        `sha256sum ${archive} | cut -d' ' -f1`, `stat -c %s ${archive}`, 'cat "$S/rowcounts.json"',
+    ].join('; ');
+}
+/** The script that measures an archive: `exists`, `bytes`, `sha256` and `header` lines. */
+export function measureScript(archive) {
+    const file = shellQuote(archive);
+    return [
+        `[ -f ${file} ] || { echo exists=0; exit 0; }`, 'echo exists=1', `echo "bytes=$(stat -c %s ${file})"`, `echo "sha256=$(sha256sum ${file} | cut -d' ' -f1)"`,
+        `if [ "$(head -c ${AGE_HEADER.length} ${file})" = ${shellQuote(AGE_HEADER)} ]; then echo header=1; else echo header=0; fi`,
+    ].join('; ');
+}
+/** The root script of the test decrypt: the manager's own identity opens the archive in a private scratch directory and tar lists it. */
+export function decryptScript(archive) {
+    return [
+        'set -e', 'umask 077', 'PATH="$PATH:/usr/local/bin:/usr/bin:/bin"', 't=$(mktemp -d)', "trap 'rm -rf \"$t\"' EXIT",
+        `age -d -i ${shellQuote(SERVER_AGE_IDENTITY)} -o "$t/backup.tar" ${shellQuote(archive)}`, 'tar -tf "$t/backup.tar"',
+    ].join('\n');
+}
+const sh = (script) => `bash -c ${shellQuote(script)}`;
+const stamp = (at) => at.replaceAll(/[-:.]/gu, '');
+const projectName = (config) => config.project.replaceAll(/[^A-Za-z0-9_-]/gu, '-') || 'database';
+/** The rows of rowcounts.json as the receipt's summary: table count and total rows. */
+function summarize(text) {
+    let counts;
+    try {
+        counts = JSON.parse(text.trim() || '[]');
+    }
+    catch {
+        throw failure('STACK_DEPLOY_BACKUP_FAILED', 'the row counts of the dump are unreadable; output withheld');
+    }
+    if (!Array.isArray(counts))
+        throw failure('STACK_DEPLOY_BACKUP_FAILED', 'the row counts of the dump are not a list');
+    return { tables: counts.length, rows: counts.reduce((sum, row) => sum + BigInt(/^\d+$/u.test(String(row?.count)) ? String(row.count) : '0'), 0n).toString() };
+}
+/**
+ * Dumps the database of `container` on the manager and writes the receipt beside the archive. Returns the receipt and what the manager measures on
+ * the archive afterwards (a second reading, so the receipt is judged against the file, not against the script that wrote it).
+ */
+export async function managerBackup(_ctx, config, { run, container, url, recipients, at }) {
+    if (!DB_IDENT.test(new URL(url).username))
+        throw refusal('STACK_DEPLOY_BACKUP_FAILED', 'the database role is not a plain identifier');
+    const dir = backupFolder(config);
+    const name = `${projectName(config)}-${stamp(at)}`;
+    const file = `${name}.tar.age`;
+    const archive = `${dir}/${file}`;
+    const receiptPath = `${dir}/${name}.receipt.json`;
+    const dumped = run(sh(dumpScript({ dir, file, container, url, recipients })));
+    if (dumped.status !== 0)
+        throw failure('STACK_DEPLOY_BACKUP_FAILED', 'the dump on the manager failed; output withheld');
+    const [sha256 = '', bytes = '', ...rest] = dumped.stdout.split('\n');
+    if (!/^[a-f0-9]{64}$/u.test(sha256.trim()) || !/^\d+$/u.test(bytes.trim()))
+        throw failure('STACK_DEPLOY_BACKUP_FAILED', 'the dump on the manager printed no archive size or hash');
+    const receipt = {
+        schema: RECEIPT_SCHEMA, project: projectName(config), target: 'container', ref: `${config.stack}`, at, file, bytes: Number(bytes.trim()), sha256: sha256.trim(),
+        parts: [...BACKUP_PARTS], rowcounts: summarize(rest.join('\n')), schemaOnly: false,
+    };
+    const written = run(sh(`umask 077; cat > ${shellQuote(receiptPath)}`), { input: `${JSON.stringify(receipt, null, 2)}\n` });
+    if (written.status !== 0)
+        throw failure('STACK_DEPLOY_BACKUP_FAILED', 'the receipt could not be written on the manager');
+    const measured = run(sh(measureScript(archive)));
+    if (measured.status !== 0)
+        throw failure('STACK_DEPLOY_BACKUP_FAILED', 'the archive could not be measured on the manager');
+    const reading = Object.fromEntries(measured.stdout.split('\n').map((line) => line.trim().split('=')).filter((pair) => pair.length === 2));
+    const facts = { exists: reading.exists === '1', size: Number(reading.bytes ?? 0), header: reading.header === '1', sha256: reading.sha256 ?? '' };
+    return { receiptPath, receipt, archive, facts };
+}
+/** Opens the archive on the manager with the manager's own identity and lists its parts. Throws when the identity cannot open it. */
+export async function managerDecrypt(_ctx, { run, archive }) {
+    if (!BASE.test(archive))
+        throw refusal('STACK_DEPLOY_BACKUP_DECRYPT_FAILED', 'the archive path is not a plain absolute path');
+    const result = run(AS_ROOT, { input: `${decryptScript(archive)}\n` });
+    if (result.status !== 0)
+        throw failure('STACK_DEPLOY_BACKUP_DECRYPT_FAILED', `the manager's identity could not open the archive (is the server recipient in .sops.yaml?); output withheld`);
+    return result.stdout.split('\n').map((line) => line.replace(/^\.\//u, '').trim()).filter(Boolean);
+}

package/stack/release-store.d.mts

@@ -10,6 +10,26 @@ export interface ReleaseEntry {
     current: boolean;
     previous: boolean;
 }
+/** What a deploy lock is taken for: the trigger of the attempt (a closed set, so no free text reaches the remote shell). */
+export type LockTrigger = 'ci' | 'dispatch' | 'urgent';
+/** The answer of one try at the deploy lock: taken, or who holds it (a trigger word) and for how long it has gone unrefreshed. */
+export type LockAttempt = {
+    acquired: true;
+} | {
+    acquired: false;
+    holder: string;
+    ageSeconds: number;
+};
+/** The journal files of one attempt: its document (rewritten at every phase boundary) and the append-only index line. */
+export interface AttemptJournal {
+    write(document: object): Promise<void>;
+    index(entry: object): Promise<void>;
+}
+/** A stored release read into memory: the stack bytes and the parsed release record. */
+export interface StoredRelease {
+    stackYaml: string;
+    release: Record<string, unknown>;
+}
 /** The release record protocol on the manager (see createReleaseStore). */
 export interface ReleaseStore {
     store(dir: string, id: string): Promise<void>;
@@ -18,7 +38,31 @@ export interface ReleaseStore {
     previous(): Promise<string>;
     fetch(id: string, targetDir: string): Promise<string>;
     list(): Promise<ReleaseEntry[]>;
+    /** A stored release read into memory (both files); throws when it is not stored. */
+    read(id: string): Promise<StoredRelease>;
+    /** One try at /srv/<stack>/deploy.lock for the holder `token` (32 hex): a stale lock (unrefreshed for LOCK_TTL_SECONDS) is taken over. */
+    tryLock(token: string, trigger: LockTrigger): Promise<LockAttempt>;
+    /** Marks the lock as alive; false when the holder no longer owns it. */
+    refreshLock(token: string): Promise<boolean>;
+    /** Releases the lock when `token` still owns it; false when it did not. */
+    unlock(token: string): Promise<boolean>;
+    /** The journal of one attempt under /srv/<stack>/deploys. */
+    journal(attempt: string): AttemptJournal;
 }
+/** A lock nobody refreshed for this long (its holder died without releasing it) is taken over by the next try. Phases refresh it. */
+export declare const LOCK_TTL_SECONDS = 3600;
+/** An attempt id: UTC compact time and 8 random hex. */
+export declare const ATTEMPT_ID: RegExp;
+/**
+ * The remote script of one try at the deploy lock: `mkdir` is the atomic claim, the owner file (token, trigger, epoch) is replaced by rename, a lock
+ * unrefreshed for `ttl` seconds is moved aside (one rename wins) and claimed anew. Prints `acquired` or `busy <trigger> <age>`. Composed only from
+ * a validated base, token and trigger.
+ */
+export declare const lockAcquireScript: (base: string, token: string, trigger: string, ttl?: number) => string;
+/** The remote script that marks the lock alive when `token` owns it: prints `held` or `lost`. */
+export declare const lockRefreshScript: (base: string, token: string) => string;
+/** The remote script that removes the lock when `token` owns it: prints `released` or `lost`. */
+export declare const lockReleaseScript: (base: string, token: string) => string;
 /** Removes every carriage return: a snippet from a Windows checkout must never reach a remote shell with CRLF. */
 export declare const lfOnly: (text: unknown) => string;
 /**

package/stack/release-store.mjs

@@ -9,6 +9,48 @@ import { firstLine, lines as splitLines } from './context.mjs';
 import { isReleaseId } from './digest.mjs';
 import { failure, refusal } from './errors.mjs';
 import { releaseListScript, releasePointerScript } from './remote.mjs';
+/** A lock nobody refreshed for this long (its holder died without releasing it) is taken over by the next try. Phases refresh it. */
+export const LOCK_TTL_SECONDS = 3600;
+const TOKEN = /^[0-9a-f]{32}$/u;
+const TRIGGERS = ['ci', 'dispatch', 'urgent'];
+/** An attempt id: UTC compact time and 8 random hex. */
+export const ATTEMPT_ID = /^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{8}$/u;
+const checkedToken = (token) => {
+    if (typeof token !== 'string' || !TOKEN.test(token))
+        throw refusal('STACK_LOCK_TOKEN', 'a lock token is exactly 32 lowercase hex characters');
+    return token;
+};
+const checkedTrigger = (trigger) => {
+    if (typeof trigger !== 'string' || !TRIGGERS.includes(trigger))
+        throw refusal('STACK_LOCK_TRIGGER', 'a deploy trigger is ci, dispatch or urgent');
+    return trigger; // TRIGGERS holds exactly the LockTrigger members
+};
+const checkedAttempt = (attempt) => {
+    if (typeof attempt !== 'string' || !ATTEMPT_ID.test(attempt))
+        throw refusal('STACK_ATTEMPT_ID', 'an attempt id is a UTC compact time and 8 lowercase hex characters');
+    return attempt;
+};
+const OWNER_READ = (lock) => `if [ -r "${lock}/owner" ]; then read -r held_token held_trigger held_stamp < "${lock}/owner"; else held_token=none; held_trigger=unknown; held_stamp=$(stat -c %Y "${lock}" 2>/dev/null) || held_stamp=0; fi; case "$held_stamp" in ''|*[!0-9]*) held_stamp=0;; esac`;
+/**
+ * The remote script of one try at the deploy lock: `mkdir` is the atomic claim, the owner file (token, trigger, epoch) is replaced by rename, a lock
+ * unrefreshed for `ttl` seconds is moved aside (one rename wins) and claimed anew. Prints `acquired` or `busy <trigger> <age>`. Composed only from
+ * a validated base, token and trigger.
+ */
+export const lockAcquireScript = (base, token, trigger, ttl = LOCK_TTL_SECONDS) => {
+    const lock = `${base}/deploy.lock`;
+    return [
+        `b="${base}"; l="${lock}"; now=$(date +%s); mkdir -p "$b" || exit 1`,
+        `claim() { printf '%s %s %s\\n' '${checkedToken(token)}' '${checkedTrigger(trigger)}' "$now" > "$l/owner.new" && mv -f "$l/owner.new" "$l/owner" && echo acquired && exit 0; exit 1; }`,
+        'if mkdir "$l" 2>/dev/null; then claim; fi',
+        OWNER_READ('$l'),
+        `age=$((now - held_stamp)); if [ "$age" -gt ${Math.trunc(ttl)} ]; then if mv "$l" "$l.stale.${token}" 2>/dev/null; then rm -rf "$l.stale.${token}"; if mkdir "$l" 2>/dev/null; then claim; fi; fi; fi`,
+        'case "$held_trigger" in ci|dispatch|urgent) ;; *) held_trigger=unknown;; esac; echo "busy $held_trigger $age"',
+    ].join('; ');
+};
+/** The remote script that marks the lock alive when `token` owns it: prints `held` or `lost`. */
+export const lockRefreshScript = (base, token) => `l="${base}/deploy.lock"; now=$(date +%s); if [ -r "$l/owner" ]; then read -r t g s < "$l/owner"; if [ "$t" = '${checkedToken(token)}' ]; then printf '%s %s %s\\n' "$t" "$g" "$now" > "$l/owner.new" && mv -f "$l/owner.new" "$l/owner" && echo held; else echo lost; fi; else echo lost; fi`;
+/** The remote script that removes the lock when `token` owns it: prints `released` or `lost`. */
+export const lockReleaseScript = (base, token) => `l="${base}/deploy.lock"; if [ -r "$l/owner" ]; then read -r t g s < "$l/owner"; if [ "$t" = '${checkedToken(token)}' ]; then rm -rf "$l" && echo released; else echo lost; fi; else echo lost; fi`;
 const BASE = /^\/[A-Za-z0-9._/-]+$/u;
 const FILES = ['stack.yaml', 'release.json'];
 /** Removes every carriage return: a snippet from a Windows checkout must never reach a remote shell with CRLF. */
@@ -50,6 +92,15 @@ export function createReleaseStore(config, ssh) {
         const id = out.trim();
         return isReleaseId(id) ? id : '';
     }
+    /** Both files of a stored release as [name, text] pairs, framed as base64 lines. */
+    async function readFiles(id) {
+        const source = dirOf(id);
+        const out = await run(`set -e; for f in stack.yaml release.json; do test -r "${source}/$f"; base64 < "${source}/$f" | tr -d '\\n'; printf '\\n'; done`);
+        const frames = out.split('\n');
+        if (frames.length !== 3 || frames[2] !== '' || frames.slice(0, 2).some((frame) => !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(frame)))
+            throw failure('STACK_RELEASE_SSH', 'invalid release response framing');
+        return FILES.map((name, index) => [name, Buffer.from(frames[index], 'base64').toString('utf8')]);
+    }
     return {
         /** Uploads stack.yaml and release.json of a rendered release as text; read-only afterwards. */
         async store(dir, id) {
@@ -69,17 +120,49 @@ export function createReleaseStore(config, ssh) {
         previous: () => pointer('previous'),
         /** Downloads a stored release (both files) into `targetDir` for a rollback or an audit. */
         async fetch(id, targetDir) {
-            const source = dirOf(id);
-            const out = await run(`set -e; for f in stack.yaml release.json; do test -r "${source}/$f"; base64 < "${source}/$f" | tr -d '\\n'; printf '\\n'; done`);
-            const frames = out.split('\n');
-            if (frames.length !== 3 || frames[2] !== '' || frames.slice(0, 2).some((frame) => !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(frame)))
-                throw failure('STACK_RELEASE_SSH', 'invalid release response framing');
-            const files = FILES.map((name, index) => [name, Buffer.from(frames[index], 'base64').toString('utf8')]);
+            const files = await readFiles(id);
             fs.mkdirSync(targetDir, { recursive: true });
             for (const [name, text] of files)
                 fs.writeFileSync(path.join(targetDir, name), text);
             return targetDir;
         },
+        /** A stored release in memory: the stack bytes and the parsed release record. */
+        async read(id) {
+            const [stack, release] = await readFiles(id);
+            let record;
+            try {
+                record = JSON.parse(release[1]);
+            }
+            catch {
+                throw failure('STACK_RELEASE_STORE', `release ${id} has a release.json that is not JSON`);
+            }
+            return { stackYaml: stack[1], release: record !== null && typeof record === 'object' && !Array.isArray(record) ? record : {} };
+        },
+        async tryLock(token, trigger) {
+            const answer = (await run(lockAcquireScript(base, token, trigger))).trim();
+            if (answer === 'acquired')
+                return { acquired: true };
+            const busy = /^busy ([a-z]+) (-?[0-9]+)$/u.exec(answer);
+            if (!busy)
+                throw failure('STACK_RELEASE_SSH', 'the deploy lock script answered something else than acquired or busy');
+            return { acquired: false, holder: busy[1], ageSeconds: Number(busy[2]) };
+        },
+        async refreshLock(token) { return (await run(lockRefreshScript(base, token))).trim() === 'held'; },
+        async unlock(token) { return (await run(lockReleaseScript(base, token))).trim() === 'released'; },
+        journal(attempt) {
+            const id = checkedAttempt(attempt);
+            const file = `${base}/deploys/${id}.json`;
+            const encoded = (value) => `${Buffer.from(JSON.stringify(value), 'utf8').toString('base64')}
+`;
+            return {
+                async write(document) {
+                    await run(`set -e; d="${base}/deploys"; mkdir -p "$d"; IFS= read -r doc; printf '%s' "$doc" | base64 -d > "$d/.${id}.tmp"; mv -f "$d/.${id}.tmp" "${file}"`, encoded(document));
+                },
+                async index(entry) {
+                    await run(`set -e; mkdir -p "${base}/deploys"; IFS= read -r line; printf '%s\\n' "$(printf '%s' "$line" | base64 -d)" >> "${base}/deploys.jsonl"`, encoded(entry));
+                },
+            };
+        },
         /** The stored releases, newest first: [{ id, current, previous }]. */
         async list() {
             const response = await run(`${releasePointerScript(`"${base}"`, 'current')}; ${releasePointerScript(`"${base}"`, 'previous')}; ${releaseListScript(`"${base}"`)}`);

package/stack/remote.d.mts

@@ -73,6 +73,8 @@ export declare function openSealedSession(ctx: StackContext, config: DeployConfi
 export declare const ownerSshRunner: (ctx: StackContext, config: DeployConfig, options?: SshOptions) => Runner;
 /** Opens the runner (`deps.ssh` wins in specs), calls `body(run, notes)` and always cleans the session up. */
 export declare function withRunner<T>(ctx: StackContext, config: DeployConfig, options: SshOptions, deps: RunnerDeps, body: (run: Runner, notes: string[]) => T | Promise<T>): Promise<T>;
+/** The sealed member that holds the CI transport of an environment: VPS_SSH_KEY (the CI-only deploy key), VPS_KNOWN_HOSTS, VPS_HOST and VPS_USER. */
+export declare const CI_MEMBER = "vps-ci";
 /** CI transport: pinned key, known_hosts, ssh config block and DOCKER_HOST=ssh://<alias> in $GITHUB_ENV. */
 export declare function connectCi(ctx: StackContext, config: DeployConfig, { execute }?: {
     execute?: boolean;