@hyperfixation/cli 0.1.5 → 0.1.7

package/dist/deploy-app.js

@@ -0,0 +1,81 @@
+import { deployCommit } from "./cloud-steps/deploy.js";
+import { spawnStepExec, StepFailed } from "./cloud-steps/index.js";
+import { gitAuthEnv } from "./cloud-steps/repo.js";
+import { loadOperatorConfig, requireOperatorConfig } from "./config.js";
+import { deriveNames } from "./names.js";
+import { CoolifyClient } from "./providers/coolify.js";
+import { openAppState } from "./state.js";
+/** What a commit looks like once `git` has resolved it; `/api/status` reports the same form. */
+const FULL_SHA = /^[0-9a-f]{40}$/;
+/**
+ * `hf deploy <name>` — point the app at a commit and wait until it says it is serving it.
+ *
+ * The same path as `hf new`'s tenth step, and the reason there is a command at all: auto-deploy
+ * is off, so a merge to main changes nothing on the box until this runs. Everything it needs is
+ * in the state cache the provisioning run wrote; nothing here is interactive.
+ */
+export async function deployApp(options) {
+    const env = options.env ?? process.env;
+    const config = options.config ?? (await loadOperatorConfig({ env }));
+    const names = deriveNames(options.app);
+    const required = requireOperatorConfig(config, ["HF_COOLIFY_URL", "HF_COOLIFY_TOKEN", "HF_BASE_DOMAIN", "HF_GITHUB_TOKEN"], { env });
+    const store = await openAppState(names.given, { dir: options.stateDir, env });
+    const { coolify, statusTokens, repo } = store.state;
+    const appUuid = coolify?.appUuid;
+    const readToken = statusTokens?.read;
+    if (appUuid === undefined || readToken === undefined) {
+        const missing = appUuid === undefined ? "Coolify application uuid" : "read status token";
+        throw new StepFailed(`${store.file} has no ${missing}: hf new has not finished provisioning ${names.given}`);
+    }
+    const sha = options.sha === undefined
+        ? await mainSha(options, repo, required.HF_GITHUB_TOKEN)
+        : options.sha;
+    if (!FULL_SHA.test(sha)) {
+        throw new StepFailed(`${sha} is not a commit sha: --sha takes the full forty hex characters, because that is ` +
+            "what /api/status reports back");
+    }
+    const fqdn = `${names.given}.${required.HF_BASE_DOMAIN}`;
+    await deployCommit({
+        name: names.given,
+        appUuid,
+        fqdn,
+        sha,
+        readToken,
+        coolify: new CoolifyClient({
+            url: required.HF_COOLIFY_URL,
+            token: required.HF_COOLIFY_TOKEN,
+            fetch: options.fetch,
+        }),
+        io: options.io,
+        now: options.now ?? (() => Date.now()),
+        sleep: options.sleep ?? (async (ms) => await new Promise((resolve) => setTimeout(resolve, ms))),
+        fetch: options.fetch,
+    });
+    await store.patch({ lastDeployedSha: sha });
+    return { app: names.given, sha, url: `https://${fqdn}` };
+}
+/**
+ * `main`'s sha on the app's repository, read with `git ls-remote` and no checkout.
+ *
+ * The remote rather than a local clone: the operator running this has just merged a pull request,
+ * and whatever is in a directory on the laptop is not what the box would build.
+ */
+async function mainSha(options, repo, token) {
+    if (repo === undefined) {
+        throw new StepFailed(`no owner/name repository in ${options.app}'s state cache: nothing can be asked what main ` +
+            "is. Pass --sha <sha>.");
+    }
+    const url = `https://github.com/${repo}.git`;
+    const exec = options.exec ?? spawnStepExec;
+    const outcome = await exec("git", ["ls-remote", url, "refs/heads/main"], {
+        cwd: process.cwd(),
+        capture: true,
+        env: gitAuthEnv(token),
+    });
+    const sha = /^([0-9a-f]{40})\s/.exec(outcome.stdout.trim())?.[1];
+    if (outcome.code !== 0 || sha === undefined) {
+        throw new StepFailed(`git ls-remote ${url} refs/heads/main named no commit: the repository may be gone, the ` +
+            "branch unborn, or HF_GITHUB_TOKEN unable to read it. Pass --sha <sha>.");
+    }
+    return sha;
+}

package/dist/doctor.d.ts

@@ -1,15 +1,23 @@
 import { type OperatorConfig } from "./config.js";
+import { type Database } from "./database.js";
 import type { FetchLike } from "./providers/http.js";
 import { type Runner } from "./runner.js";
 /** A restore check older than this is a warning: E5 is meant to run weekly, not once. */
 export declare const RESTORE_CHECK_MAX_AGE_DAYS = 7;
+/** The `CONNECTION LIMIT` every application role is created with, in `provisionRoles`. */
+export declare const APPLICATION_ROLE_CONNECTION_LIMIT = 25;
+/** Past this share of `max_connections`, the next app to deploy is the one that cannot connect. */
+export declare const CONNECTIONS_WARN_FRACTION = 0.8;
 /** The branch prefix Phase 4's core bumps open their pull requests on. */
 export declare const CORE_BUMP_BRANCH_PREFIX = "core-bump/";
 export type Severity = "ok" | "warn" | "fail";
 export interface DoctorFinding {
     /** The app as the state cache names it. */
     app: string;
-    /** `state`, `status`, `runs`, `version`, `budget`, `E006`, `restore-check`, `core-bump`. */
+    /**
+     * `state`, `status`, `runs`, `version`, `budget`, `E006`, `connections`, `lock`,
+     * `restore-check`, `core-bump`.
+     */
     check: string;
     severity: Severity;
     message: string;
@@ -39,6 +47,11 @@ export interface DoctorOptions {
     now?: () => Date;
     /** How E006 is read. Defaults to the tunnel to `HF_SSH_HOST` as `postgres`. */
     privileges?: PrivilegeCheck;
+    /**
+     * Where the connection counts and the worker locks are read: the whole cluster, as the admin
+     * E006 already goes in as. Defaults to the same tunnel to `HF_SSH_HOST`.
+     */
+    database?: () => Promise<Database>;
     env?: NodeJS.ProcessEnv;
 }
 /**
@@ -70,3 +83,11 @@ export declare function tunnelPrivilegeCheck(runner: Runner, options?: {
  * privilege query is a second thing to keep in step with the grants the migrator makes.
  */
 export declare function checkAppRolePrivileges(adminUrl: string, role: string): Promise<void>;
+/**
+ * The worker's advisory lock in one app's database: exactly one, under the key the worker takes.
+ *
+ * `pg_try_advisory_lock(bigint)` splits its key across `classid` and `objid`, so the key is
+ * reassembled rather than compared whole — and masked rather than only shifted, because
+ * `hashtext` answers `int4` and a negative hash widens to a bigint of sign bits.
+ */
+export declare function workerLockSql(appName: string): string;

package/dist/doctor.js

@@ -6,10 +6,15 @@ import { DEFAULT_PG_ADMIN_USER, loadOperatorConfig, pgAdminUser, postgresContain
 import { openDatabase } from "./database.js";
 import { deriveNames } from "./names.js";
 import { GithubClient } from "./providers/github.js";
+import { quoteLiteral } from "./roles.js";
 import { createSshRunner } from "./runner.js";
 import { openAppState, stateDir } from "./state.js";
 /** A restore check older than this is a warning: E5 is meant to run weekly, not once. */
 export const RESTORE_CHECK_MAX_AGE_DAYS = 7;
+/** The `CONNECTION LIMIT` every application role is created with, in `provisionRoles`. */
+export const APPLICATION_ROLE_CONNECTION_LIMIT = 25;
+/** Past this share of `max_connections`, the next app to deploy is the one that cannot connect. */
+export const CONNECTIONS_WARN_FRACTION = 0.8;
 /** The branch prefix Phase 4's core bumps open their pull requests on. */
 export const CORE_BUMP_BRANCH_PREFIX = "core-bump/";
 /**
@@ -27,6 +32,7 @@ export async function doctor(options = {}) {
     const config = options.config ?? (await loadOperatorConfig({ env }));
     const required = requireOperatorConfig(config, ["HF_BASE_DOMAIN", "HF_GITHUB_TOKEN"], { env });
     const dir = options.stateDir ?? stateDir(env);
+    const cluster = lazyDatabase(options.database ?? defaultDatabase(config, env));
     const context = {
         dir,
         env,
@@ -35,11 +41,20 @@ export async function doctor(options = {}) {
         fetch: options.fetch ?? ((input, init) => globalThis.fetch(input, init)),
         now: options.now ?? (() => new Date()),
         privileges: options.privileges ?? defaultPrivilegeCheck(config, env),
+        database: cluster.get,
+        // One snapshot for the whole run: the counts are the box's, not any one app's, and an app
+        // whose line is read a second later has not moved the cluster.
+        backends: once(async () => await readBackends(await cluster.get())),
     };
     const names = options.name === undefined ? await stateNames(dir) : [options.name];
     const findings = [];
-    for (const name of names)
-        findings.push(...(await doctorApp(context, name)));
+    try {
+        for (const name of names)
+            findings.push(...(await doctorApp(context, name)));
+    }
+    finally {
+        await cluster.close();
+    }
     return { findings, ok: findings.every((finding) => finding.severity === "ok") };
 }
 const MARKER = { ok: "OK  ", warn: "WARN", fail: "FAIL" };
@@ -109,6 +124,30 @@ function defaultPrivilegeCheck(config, env) {
         adminUser: pgAdminUser(config),
     });
 }
+function defaultDatabase(config, env) {
+    const { HF_SSH_HOST } = requireOperatorConfig(config, ["HF_SSH_HOST"], { env });
+    const runner = createSshRunner({ host: HF_SSH_HOST });
+    return async () => await openDatabase(runner, {
+        admin: { user: pgAdminUser(config) },
+        containers: postgresContainers(config),
+    });
+}
+/** One cluster connection for the whole run: opened when a check first needs it, closed once. */
+function lazyDatabase(open) {
+    let pending;
+    return {
+        get: () => (pending ??= open()),
+        close: async () => {
+            // A run whose every app failed before the first query never opened one, and an open that
+            // failed is already a finding.
+            await pending?.then(async (db) => await db.close(), () => undefined);
+        },
+    };
+}
+function once(read) {
+    let pending;
+    return () => (pending ??= read());
+}
 async function doctorApp(context, name) {
     const findings = [];
     const add = (check, severity, message) => {
@@ -142,12 +181,19 @@ async function doctorApp(context, name) {
         }
     }
     const report = await statusFindings(context, name, state, add);
-    versionFinding(report?.applicationVersion, mainSha, mainShaProblem, add);
+    versionFinding(name, report?.applicationVersion, mainSha, mainShaProblem, add);
     if (report?.budget !== undefined) {
         budgetFinding(report.budget.current, "current", add);
         budgetFinding(report.budget.previous, "previous", add);
     }
     await privilegeFindings(context, name, add);
+    // A name `deriveNames` refuses has already failed E006 on that same message; the cluster checks
+    // have no names to run under and say nothing more.
+    const names = tryNames(name);
+    if (names !== undefined) {
+        await connectionFindings(context, names.applicationRole, add);
+        await lockFindings(context, names, add);
+    }
     restoreCheckFindings(context, state, add);
     if (repo !== undefined)
         await bumpFindings(context, repo, add);
@@ -232,7 +278,14 @@ function amount(value) {
     const parsed = Number(value);
     return Number.isFinite(parsed) ? parsed : undefined;
 }
-function versionFinding(deployed, mainSha, mainShaProblem, add) {
+/**
+ * What the app answers against what its repository's main holds.
+ *
+ * The mismatch names `hf deploy` because nothing else closes it: Coolify's push auto-deploy is
+ * disabled on every application `hf new` creates, so a merged pull request sits unpublished until
+ * an operator says so.
+ */
+function versionFinding(name, deployed, mainSha, mainShaProblem, add) {
     if (mainShaProblem !== undefined) {
         add("version", "fail", mainShaProblem);
         return;
@@ -245,7 +298,7 @@ function versionFinding(deployed, mainSha, mainShaProblem, add) {
     }
     add("version", deployed === mainSha ? "ok" : "warn", deployed === mainSha
         ? `applicationVersion ${short(mainSha)} is main`
-        : `applicationVersion ${short(deployed)} is not main ${short(mainSha)}`);
+        : `applicationVersion ${short(deployed)} is not main ${short(mainSha)} — run hf deploy ${name}`);
 }
 function budgetFinding(period, which, add) {
     if (period === null) {
@@ -287,6 +340,99 @@ async function privilegeFindings(context, name, add) {
         add("E006", "fail", flatten(error.message));
     }
 }
+/** Every backend belonging to an app role — `hf_<app>` and its `_migrator` and `_ro`. */
+const BACKENDS_SQL = "SELECT usename, count(*) FROM pg_stat_activity WHERE usename LIKE 'hf\\_%' GROUP BY usename";
+async function readBackends(db) {
+    const setting = (await db.query("SHOW max_connections")).rows[0]?.[0];
+    const byRole = new Map();
+    for (const row of (await db.query(BACKENDS_SQL)).rows) {
+        if (row[0] !== undefined)
+            byRole.set(row[0], Number(row[1]));
+    }
+    let total = 0;
+    for (const count of byRole.values())
+        total += count;
+    return { max: setting === undefined ? undefined : numeric(Number(setting)), total, byRole };
+}
+/**
+ * What the box's connection slots are spent on, and what this app has of them.
+ *
+ * Box-wide rather than per-app because that is where it runs out: every app on the box draws on
+ * one `max_connections`, and the app that then cannot connect is whichever one deploys next.
+ */
+async function connectionFindings(context, role, add) {
+    let backends;
+    try {
+        backends = await context.backends();
+    }
+    catch (error) {
+        add("connections", "fail", flatten(error.message));
+        return;
+    }
+    const limit = String(APPLICATION_ROLE_CONNECTION_LIMIT);
+    const line = `${role} ${String(backends.byRole.get(role) ?? 0)}/${limit}, box ` +
+        `${String(backends.total)}/${backends.max === undefined ? UNKNOWN : String(backends.max)} ` +
+        "on hf_ roles";
+    const crowded = backends.max !== undefined && backends.total > backends.max * CONNECTIONS_WARN_FRACTION;
+    add("connections", crowded ? "warn" : "ok", crowded
+        ? `${line} — over ${String(CONNECTIONS_WARN_FRACTION * 100)}% of max_connections`
+        : line);
+}
+/**
+ * The worker's advisory lock in one app's database: exactly one, under the key the worker takes.
+ *
+ * `pg_try_advisory_lock(bigint)` splits its key across `classid` and `objid`, so the key is
+ * reassembled rather than compared whole — and masked rather than only shifted, because
+ * `hashtext` answers `int4` and a negative hash widens to a bigint of sign bits.
+ */
+export function workerLockSql(appName) {
+    return (`WITH k AS (SELECT hashtext('hf-worker:' || ${quoteLiteral(appName)})::bigint AS value) ` +
+        "SELECT count(l.pid), count(l.pid) FILTER (WHERE " +
+        "l.classid = ((k.value >> 32) & 4294967295)::oid AND " +
+        "l.objid = (k.value & 4294967295)::oid) " +
+        "FROM k LEFT JOIN pg_locks AS l ON l.locktype = 'advisory' AND " +
+        "l.database = (SELECT oid FROM pg_database WHERE datname = current_database())");
+}
+async function lockFindings(context, names, add) {
+    let row;
+    try {
+        const db = await context.database();
+        row = (await db.query(workerLockSql(names.appName), { database: names.databaseName })).rows[0];
+    }
+    catch (error) {
+        add("lock", "fail", flatten(error.message));
+        return;
+    }
+    const key = `hf-worker:${names.appName}`;
+    const held = Number(row?.[0]);
+    const matching = Number(row?.[1]);
+    if (!Number.isFinite(held)) {
+        add("lock", "fail", `pg_locks in ${names.databaseName} answered no count`);
+        return;
+    }
+    if (held === 0) {
+        add("lock", "fail", `no advisory lock in ${names.databaseName}: no worker holds ${key}`);
+        return;
+    }
+    if (held !== 1) {
+        add("lock", "fail", `${String(held)} advisory locks in ${names.databaseName}; one worker per app holds one`);
+        return;
+    }
+    if (matching !== 1) {
+        add("lock", "fail", `the one advisory lock in ${names.databaseName} is not hashtext('${key}'): ` +
+            "something other than the worker holds it");
+        return;
+    }
+    add("lock", "ok", `one worker holds ${key} in ${names.databaseName}`);
+}
+function tryNames(name) {
+    try {
+        return deriveNames(name);
+    }
+    catch {
+        return undefined;
+    }
+}
 function restoreCheckFindings(context, state, add) {
     const last = state.lastRestoreCheckAt;
     if (last === undefined) {

package/dist/index.d.ts

@@ -17,5 +17,5 @@ export { run, CommandFailed, type RunOptions } from "./spawn.js";
 export { createLocalRunner, createSshRunner, shellQuote, sshExecArgv, sshTunnelArgv, RunnerError, DEFAULT_TUNNEL_READY_TIMEOUT_MS, type ExecOptions, type ExecResult, type LocalRunner, type LocalRunnerOptions, type Runner, type SshRunnerOptions, type Tunnel, } from "./runner.js";
 export { findPostgresContainer, openDatabase, openDatabaseUrl, redactPasswords, DatabaseTransportError, DEFAULT_POSTGRES_PORT, type AdminCredentials, type Database, type DatabaseTransport, type OpenDatabaseOptions, type QueryOptions, type QueryResult, } from "./database.js";
 export { createLocalDirectoryBackupSource, createS3BackupSource, BackupSourceError, COOLIFY_BACKUP_DIR, type BackupDump, type BackupSource, type BackupSourceKind, type LocalDirectoryBackupSourceOptions, } from "./backup-source.js";
-export { formatRestoreCheck, pgRestoreArgv, pgRestoreInContainerArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, type RestoreCheckAppOptions, type RestoreCheckOptions, type RestoreCheckResult, type RestoreCheckRow, type RestoreVerdict, } from "./restore-check.js";
+export { APPEND_ONLY_TABLES, formatRestoreCheck, pgRestoreArgv, pgRestoreInContainerArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, type RestoreCheckAppOptions, type RestoreCheckOptions, type RestoreCheckResult, type RestoreCheckRow, type RestoreVerdict, } from "./restore-check.js";
 export { provisionDatabase, ProvisionDatabaseError, REQUIRED_EXTENSIONS, type ProvisionDatabaseOptions, type ProvisionDatabaseResult, } from "./provision-database.js";

package/dist/index.js

@@ -17,5 +17,5 @@ export { run, CommandFailed } from "./spawn.js";
 export { createLocalRunner, createSshRunner, shellQuote, sshExecArgv, sshTunnelArgv, RunnerError, DEFAULT_TUNNEL_READY_TIMEOUT_MS, } from "./runner.js";
 export { findPostgresContainer, openDatabase, openDatabaseUrl, redactPasswords, DatabaseTransportError, DEFAULT_POSTGRES_PORT, } from "./database.js";
 export { createLocalDirectoryBackupSource, createS3BackupSource, BackupSourceError, COOLIFY_BACKUP_DIR, } from "./backup-source.js";
-export { formatRestoreCheck, pgRestoreArgv, pgRestoreInContainerArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, } from "./restore-check.js";
+export { APPEND_ONLY_TABLES, formatRestoreCheck, pgRestoreArgv, pgRestoreInContainerArgv, restoreCheck, restoreCheckApp, RestoreCheckError, SCRATCH_SUFFIX, STALE_DUMP_HOURS, } from "./restore-check.js";
 export { provisionDatabase, ProvisionDatabaseError, REQUIRED_EXTENSIONS, } from "./provision-database.js";

package/dist/new.d.ts

@@ -14,6 +14,23 @@ export declare const EXCLUDED_ENTRIES: readonly string[];
 export declare class TemplateError extends Error {
     constructor(message: string);
 }
+/**
+ * Where a cloud `hf new` fetches the template before it becomes the app.
+ *
+ * Beside the target rather than under `os.tmpdir()`, so the rename is a rename and not a second
+ * copy across filesystems, and dot-prefixed so a half-fetched tree does not look like an app.
+ */
+export declare function templateTempDir(dir: string): string;
+/**
+ * The two refusals a directory in the way earns, worded once because both halves of `hf new`
+ * raise them.
+ *
+ * Each names the absolute path and the single move that clears it: "already exists" alone leaves
+ * the operator to work out which of the app directory and the dot-prefixed scratch beside it is
+ * meant, and they are one keystroke apart.
+ */
+export declare function targetInTheWay(dir: string): TemplateError;
+export declare function scratchInTheWay(scratch: string): TemplateError;
 export interface NewAppOptions {
     /** The name as typed; becomes the directory and, underscored, both placeholders. */
     name: string;
@@ -29,6 +46,11 @@ export interface NewAppOptions {
     local: boolean;
     /** The bootstrap admin's address, written to `.env` as `HF_BOOTSTRAP_EMAIL`. Skips the prompt. */
     email?: string;
+    /**
+     * The app's monthly LLM budget, written to `.env` as `HF_BOOTSTRAP_BUDGET_USD`. `hf up` seeds
+     * the app state from there instead of falling back to its dev default.
+     */
+    budgetUsd?: string;
     /** Overrides the real interactive prompt; for tests and other callers with their own stdin. */
     promptEmail?: () => Promise<string>;
 }
@@ -41,6 +63,8 @@ export interface NewAppResult extends AppNames {
     wroteEnv: boolean;
     /** True when `HF_BOOTSTRAP_EMAIL` was written to `.env`, from `--email` or the prompt. */
     wroteBootstrapEmail: boolean;
+    /** True when `HF_BOOTSTRAP_BUDGET_USD` was written to `.env`, from `--budget-usd`. */
+    wroteBootstrapBudget: boolean;
 }
 /**
  * Copies the template checkout and substitutes the two placeholders across it.

package/dist/new.js

@@ -46,6 +46,31 @@ export class TemplateError extends Error {
         this.name = "TemplateError";
     }
 }
+/**
+ * Where a cloud `hf new` fetches the template before it becomes the app.
+ *
+ * Beside the target rather than under `os.tmpdir()`, so the rename is a rename and not a second
+ * copy across filesystems, and dot-prefixed so a half-fetched tree does not look like an app.
+ */
+export function templateTempDir(dir) {
+    return path.join(path.dirname(dir), `.${path.basename(dir)}.hf-new`);
+}
+/**
+ * The two refusals a directory in the way earns, worded once because both halves of `hf new`
+ * raise them.
+ *
+ * Each names the absolute path and the single move that clears it: "already exists" alone leaves
+ * the operator to work out which of the app directory and the dot-prefixed scratch beside it is
+ * meant, and they are one keystroke apart.
+ */
+export function targetInTheWay(dir) {
+    return new TemplateError(`${dir} already exists; hf new will not write into it. Move it away (or delete it) and rerun — ` +
+        "the template is fetched into a directory of its own.");
+}
+export function scratchInTheWay(scratch) {
+    return new TemplateError(`${scratch} is a leftover hf new scratch directory and hf has no record of creating it. ` +
+        "Remove it and rerun; the template is fetched into it fresh.");
+}
 /**
  * Copies the template checkout and substitutes the two placeholders across it.
  *
@@ -64,7 +89,13 @@ export async function newApp(options) {
     await assertTemplateSource(source);
     const dir = path.resolve(options.into ?? process.cwd(), names.given);
     if (await exists(dir)) {
-        throw new TemplateError(`${dir} already exists; hf new will not write into it`);
+        throw targetInTheWay(dir);
+    }
+    // A scratch directory here is an interrupted cloud run for this same name: copying an app over
+    // the top of it would leave that run's rerun to judge a directory neither flow made.
+    const scratch = templateTempDir(dir);
+    if (await exists(scratch)) {
+        throw scratchInTheWay(scratch);
     }
     await cp(source, dir, {
         recursive: true,
@@ -75,6 +106,7 @@ export async function newApp(options) {
     const example = path.join(dir, ".env.example");
     const wroteEnv = await exists(example);
     let wroteBootstrapEmail = false;
+    let wroteBootstrapBudget = false;
     if (wroteEnv) {
         let contents = await readFile(example, "utf8");
         const email = options.email ?? (await (options.promptEmail ?? promptForBootstrapEmail)());
@@ -82,9 +114,13 @@ export async function newApp(options) {
             contents = `${contents.trimEnd()}\nHF_BOOTSTRAP_EMAIL=${email}\n`;
             wroteBootstrapEmail = true;
         }
+        if (options.budgetUsd !== undefined && options.budgetUsd !== "") {
+            contents = `${contents.trimEnd()}\nHF_BOOTSTRAP_BUDGET_USD=${options.budgetUsd}\n`;
+            wroteBootstrapBudget = true;
+        }
         await writeFile(path.join(dir, ".env"), contents);
     }
-    return { ...names, dir, substituted, wroteEnv, wroteBootstrapEmail };
+    return { ...names, dir, substituted, wroteEnv, wroteBootstrapEmail, wroteBootstrapBudget };
 }
 /** The one-time prompt: the address `hf up` later hands `hf bootstrap` via `.env`. */
 async function promptForBootstrapEmail() {

package/dist/providers/coolify.d.ts

@@ -80,6 +80,23 @@ export interface CoolifyEnvironmentVariable {
     is_literal?: boolean;
     is_multiline?: boolean;
     is_shown_once?: boolean;
+    /**
+     * Whether the builder interpolates the variable, and whether the containers get it.
+     *
+     * Both default true in Coolify's UI and neither is in the document's *request* schemas —
+     * only in its `EnvironmentVariable` model — but the box accepts and stores them (verified
+     * against 4.3.21). `SOURCE_COMMIT` needs both: it is the compose image tag and build arg as
+     * well as the `HF_BUILD_SHA` three services read.
+     */
+    is_buildtime?: boolean;
+    is_runtime?: boolean;
+}
+/** One entry of `GET /applications/{uuid}/envs`, narrowed to what an upsert has to match on. */
+export interface CoolifyEnvEntry {
+    uuid: string;
+    key: string;
+    value?: string;
+    is_preview?: boolean;
 }
 export interface CoolifyDeploymentRequest {
     deployments: {
@@ -171,6 +188,13 @@ export declare class CoolifyClient {
         tag?: string;
     }): Promise<CoolifyApplicationSummary[]>;
     updateEnvsBulk(appUuid: string, data: readonly CoolifyEnvironmentVariable[]): Promise<unknown>;
+    /** Every environment variable on the application, preview and non-preview alike. */
+    listEnvs(appUuid: string): Promise<CoolifyEnvEntry[]>;
+    createEnv(appUuid: string, variable: CoolifyEnvironmentVariable): Promise<{
+        uuid: string;
+    }>;
+    /** Keyed by `key` — and by `is_preview`, which is why an upsert sends the entry's own flag. */
+    updateEnv(appUuid: string, variable: CoolifyEnvironmentVariable): Promise<unknown>;
     deploy(uuid: string, options?: {
         force?: boolean;
     }): Promise<CoolifyDeploymentRequest>;

package/dist/providers/coolify.js

@@ -55,6 +55,27 @@ export class CoolifyClient {
             secrets: data.map((variable) => variable.value),
         });
     }
+    /** Every environment variable on the application, preview and non-preview alike. */
+    async listEnvs(appUuid) {
+        return await this.request({ method: "GET", path: `/applications/${segment(appUuid)}/envs` });
+    }
+    async createEnv(appUuid, variable) {
+        return await this.request({
+            method: "POST",
+            path: `/applications/${segment(appUuid)}/envs`,
+            body: variable,
+            secrets: [variable.value],
+        });
+    }
+    /** Keyed by `key` — and by `is_preview`, which is why an upsert sends the entry's own flag. */
+    async updateEnv(appUuid, variable) {
+        return await this.request({
+            method: "PATCH",
+            path: `/applications/${segment(appUuid)}/envs`,
+            body: variable,
+            secrets: [variable.value],
+        });
+    }
     async deploy(uuid, options = {}) {
         return await this.request({
             method: "POST",

package/dist/restore-check.d.ts

@@ -4,8 +4,21 @@ import { type Runner } from "./runner.js";
 import { type AppStateStore } from "./state.js";
 /** Appended to `hf_<app>` for the database the dump is restored into and then dropped. */
 export declare const SCRATCH_SUFFIX = "_restore_check";
-/** Older than this and the dump gets a warning line; it never changes the exit code. */
-export declare const STALE_DUMP_HOURS = 36;
+/** Older than this and the dump gets a WARN line, which — as in `hf doctor` — exits 1. */
+export declare const STALE_DUMP_HOURS = 24;
+/**
+ * Tables a live row is only ever added to, so a live count above the dump's is the app working,
+ * not lost data.
+ *
+ * X1 found this the hard way: against a 1.3-hour-old dump of a running app, 9 of 24 tables
+ * "mismatched" purely from churn since the dump, and the same check against a fresh dump matched
+ * all 24. A table earns a place here only when no code path deletes from it and none updates it
+ * in a way that lowers its count — checked against `packages/db/src/schema` and every statement
+ * in `core`, `workflows`, `auth` and `admin`. Anything else, including a table whose rows merely
+ * look permanent, stays exact: a false `ok` here hides exactly the data loss this command exists
+ * to catch.
+ */
+export declare const APPEND_ONLY_TABLES: readonly string[];
 export type RestoreVerdict = "ok" | "mismatch" | "live only" | "restored only";
 export interface RestoreCheckRow {
     table: string;
@@ -13,6 +26,14 @@ export interface RestoreCheckRow {
     live?: number;
     /** Absent when the table is not in the restored dump. */
     restored?: number;
+    /**
+     * Rows the live side gained since the dump, on an `APPEND_ONLY_TABLES` table.
+     *
+     * Drift is not its own verdict because it is not its own outcome: the restore held everything
+     * the dump had, which is `ok`. The column prints it as `ok (drift +N)` so the operator can see
+     * why two counts differ without having to decide whether it mattered.
+     */
+    drift?: number;
     verdict: RestoreVerdict;
 }
 export interface RestoreCheckResult {
@@ -21,11 +42,15 @@ export interface RestoreCheckResult {
     scratchDatabase: string;
     dump: BackupDump;
     dumpAgeHours: number;
-    /** The dump is older than `STALE_DUMP_HOURS`. Informational. */
+    /** The dump is older than `STALE_DUMP_HOURS`; on its own enough to exit 1. */
     dumpStale: boolean;
+    /** Exact matching was asked for, so no table was allowed to drift. */
+    strict: boolean;
     /** One row per table, `hf_*` or carrying `normalized_name`, sorted by name. */
     rows: readonly RestoreCheckRow[];
-    /** Every row's verdict is `ok`. The command's exit code is `ok ? 0 : 1`. */
+    /** Every row's verdict is `ok`, drift included. What `lastRestoreCheckAt` is written on. */
+    matched: boolean;
+    /** `matched` and the dump is not stale. The command's exit code is `ok ? 0 : 1`. */
     ok: boolean;
 }
 export declare class RestoreCheckError extends Error {
@@ -57,6 +82,8 @@ export interface RestoreCheckOptions {
     restoreAdminUrl?: string;
     /** The `pg_restore` binary inside the container. */
     pgRestorePath?: string;
+    /** Compare every table exactly, `APPEND_ONLY_TABLES` included. */
+    strict?: boolean;
     now?: Date;
 }
 /**
@@ -104,6 +131,8 @@ export interface RestoreCheckAppOptions {
     backupDir?: string;
     /** Read the dump from Hetzner object storage instead. Not implemented; see `backup-source`. */
     fromS3?: boolean;
+    /** Compare every table exactly, `APPEND_ONLY_TABLES` included. */
+    strict?: boolean;
     env?: NodeJS.ProcessEnv;
 }
 /**