@theford/runly 0.0.0-stage → 0.20.0

package/dist/core/exec.js

@@ -0,0 +1,84 @@
+/**
+ * Bounded shell execution for repo-declared commands.
+ *
+ * Every datastore and appliance command comes from someone's runly.yml, so it
+ * can hang: an appliance `start:` that forgets `-d` stays in the foreground, a
+ * `psql` against a half-up server blocks on connect. Unbounded, that wedges the
+ * bind forever — and because the environment stays in the daemon's `busy` set,
+ * the sweeper will not expire its lease and teardown refuses it, so the
+ * environment is unreclaimable until the daemon is killed.
+ *
+ * So every such command is spawned as its own process group with a hard
+ * deadline, and the timeout kills the GROUP: killing only `sh` would leave the
+ * real client (psql, docker) running and still holding whatever it holds.
+ * This mirrors the engine's check runner, which learned the same lesson.
+ */
+import { spawn } from 'node:child_process';
+/** Repo-declared commands are IO-bound setup steps, not builds. */
+export const DEFAULT_CMD_TIMEOUT_S = 300;
+/** Build/exec/check-grade work gets a longer leash than setup steps. */
+export const LONG_CMD_TIMEOUT_S = 600;
+/**
+ * The deadline for a repo-declared command. Callers pass the default that fits
+ * their site's class (setup steps 300s, or an upkeep rule's own `timeout`;
+ * build/exec-grade work 600s); BACKLOT_CMD_TIMEOUT_S overrides ALL of them —
+ * the escape hatch for a repo whose legitimate commands outrun the class default.
+ */
+export function cmdTimeoutS(fallback = DEFAULT_CMD_TIMEOUT_S) {
+    const e = process.env.BACKLOT_CMD_TIMEOUT_S;
+    if (e === undefined || e === '')
+        return fallback;
+    const n = Number(e);
+    return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+export function runBounded(cmd, cwd, timeoutS = DEFAULT_CMD_TIMEOUT_S, env, 
+/** Sees every chunk as it arrives (a build log, decision 0038); `end` once per stream. */
+onOutput) {
+    return new Promise((resolve) => {
+        const proc = spawn('sh', ['-c', cmd], { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], detached: true });
+        let out = '';
+        let settled = false;
+        let timedOut = false;
+        const sink = (stream) => (d) => {
+            const text = d.toString();
+            out = (out + text).slice(-16_000);
+            try {
+                onOutput?.data(stream, text);
+            }
+            catch {
+                /* a log write must never fail the command */
+            }
+        };
+        proc.stdout.on('data', sink('out'));
+        proc.stderr.on('data', sink('err'));
+        proc.stdout.on('end', () => onOutput?.end('out'));
+        proc.stderr.on('end', () => onOutput?.end('err'));
+        const timer = setTimeout(() => {
+            timedOut = true;
+            try {
+                process.kill(-proc.pid, 'SIGKILL'); // the group, not just the sh wrapper
+            }
+            catch {
+                try {
+                    proc.kill('SIGKILL');
+                }
+                catch {
+                    /* already gone */
+                }
+            }
+        }, timeoutS * 1000);
+        timer.unref();
+        const done = (r) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolve(r);
+        };
+        // A spawn failure (EAGAIN/EMFILE under load) emits 'error' with no 'exit';
+        // without this the promise never settles and the caller's lock wedges.
+        proc.on('error', (err) => done({ code: 1, output: `${out}\nspawn error: ${err.message}`, timedOut }));
+        proc.on('exit', (code) => done({ code: code ?? 1, output: out, timedOut }));
+    });
+}
+//# sourceMappingURL=exec.js.map

package/dist/core/exec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"exec.js","sourceRoot":"","sources":["../../src/core/exec.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE3C,mEAAmE;AACnE,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC,wEAAwE;AACxE,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,WAAmB,qBAAqB;IAClE,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC;IAC5C,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE;QAAE,OAAO,QAAQ,CAAC;IACjD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACpB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACpD,CAAC;AAQD,MAAM,UAAU,UAAU,CACxB,GAAW,EACX,GAAW,EACX,WAAmB,qBAAqB,EACxC,GAAuB;AACvB,0FAA0F;AAC1F,QAAyG;IAEzG,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACvG,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,IAAI,GAAG,CAAC,MAAqB,EAAE,EAAE,CAAC,CAAC,CAAS,EAAE,EAAE;YACpD,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;YAC1B,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;YAClC,IAAI,CAAC;gBACH,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC;gBACP,6CAA6C;YAC/C,CAAC;QACH,CAAC,CAAC;QACF,IAAI,CAAC,MAAO,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,MAAO,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,MAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,MAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAEnD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC;YAChB,IAAI,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAI,EAAE,SAAS,CAAC,CAAC,CAAC,qCAAqC;YAC5E,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC;oBACH,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACvB,CAAC;gBAAC,MAAM,CAAC;oBACP,kBAAkB;gBACpB,CAAC;YACH,CAAC;QACH,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,CAAC;QACpB,KAAK,CAAC,KAAK,EAAE,CAAC;QAEd,MAAM,IAAI,GAAG,CAAC,CAAY,EAAE,EAAE;YAC5B,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,CAAC,CAAC,CAAC;QACb,CAAC,CAAC;QACF,2EAA2E;QAC3E,uEAAuE;QACvE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,GAAG,kBAAkB,GAAG,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;QACtG,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/core/journal.d.ts

@@ -0,0 +1,227 @@
+import type { EnvState, Hygiene, LeaseKind, ServicePid } from './types.js';
+/**
+ * The journal schema this build understands, stamped into `PRAGMA user_version`.
+ *
+ * Bump it when a change makes an OLDER daemon misread this journal — not for
+ * every additive column: a bump is unnecessary when an old daemon selecting
+ * a known subset of columns still reads the journal correctly.
+ *
+ * Schemas 1-3 belong to runly before 0.16 and are refused: their migrations
+ * were retired in 0.19 (upgrade through 0.18 first).
+ *
+ * Schema 4 (decision 0034): `active_services = '[]'` means "no services are
+ * wanted" (an older daemon reads an empty list as the whole app and would boot
+ * it), the `db_copies` table holds database copies an older daemon would never
+ * reap, and data-only environments are gone (0.18 migrated them away).
+ *
+ * What this exists to stop is the DOWNGRADE, which has already cost once. The
+ * sha256 env-id migration stranded pre-upgrade rows that then counted against
+ * POOL_MAX_TOTAL and held their ports forever (BACKLOG.md), because nothing on
+ * disk said which build wrote them. A running example of the same shape:
+ * `data_only` defaults to 0, so a 0.7.0 daemon re-binding a 0.8.0 data-only
+ * environment reads "not data-only" and boots the whole application into a
+ * test lane's database. Disk is truth (decision 0009), so the truth has to say
+ * what wrote it.
+ */
+export declare const JOURNAL_SCHEMA_VERSION = 4;
+export interface EnvRow {
+    id: string;
+    stack: string;
+    stackRoot: string;
+    state: EnvState;
+    root: string;
+    ports: Record<string, number>;
+    datastoreNs: Record<string, string>;
+    fingerprints: Record<string, string>;
+    presets: Record<string, string>;
+    bindCount: number;
+    createdAt: number;
+    lastUsedAt: number;
+    servicePids: Record<string, ServicePid>;
+    /** Consecutive bind failures — >= 2 auto-escalates the next bind to pristine (decision 0007). */
+    failStreak: number;
+    /**
+     * The services this environment's lease WANTS up (decision 0034): `up`
+     * adds to it, `down` takes away. Undefined means every service the
+     * manifest declares (so a service added to the manifest later is included);
+     * `[]` means none — the shape `down` with no names leaves. The running set
+     * is the supervisor's; this is the intent a quiesce or a daemon restart
+     * keeps, and the next `up` restores.
+     */
+    activeServices?: string[];
+    /**
+     * When each public port last carried a client byte (decision 0035), by port
+     * key. Persisted (throttled) so a daemon restart does not reset every idle
+     * clock to "active"; the proxy's in-memory counters start from it.
+     */
+    activity?: Record<string, number>;
+    /**
+     * How to drop each datastore namespace this environment holds, recorded
+     * when it is created (decision 0037): the templated drop command and where
+     * to run it, or the file to delete. Teardown needs neither the manifest nor
+     * the worktree — both may be gone by the time it runs.
+     */
+    dropRecipes?: Record<string, DropRecipe>;
+    /** The template each datastore was last restored from (decision 0037: templates are kept by reference). */
+    templates?: Record<string, string>;
+}
+/** Everything needed to drop one namespace without the manifest (decision 0037). */
+export interface DropRecipe {
+    cmd?: string;
+    cwd?: string;
+    /** The server-side namespace the drop removes (0.16.1+), referenced from before it is created. */
+    ns?: string;
+    /** A file (sqlite) to delete instead, with its -wal/-shm/-journal sidecars. */
+    path?: string;
+}
+export interface LeaseRow {
+    id: string;
+    envId: string;
+    kind: LeaseKind;
+    holder: string;
+    hygiene: Hygiene;
+    expiresAt: number;
+    /**
+     * The holder's process, when the caller supplied one.
+     *
+     * `holder` is a NAME (a worktree path by default) and nothing about a name
+     * can die — so an agent that crashed held its environment until the TTL
+     * expired, exempt from idle reclamation the whole time. A pid pinned by its
+     * start time can be checked, so a dead holder's lease is released in seconds.
+     */
+    holderPid?: number;
+    holderStart?: number;
+    /** Active lease-scoped public preview tunnel, when `runly preview` is running. */
+    previewService?: string;
+    previewUrl?: string;
+    previewPid?: number;
+    previewStart?: number;
+    /** Local port the tunnel was published against, so a rebind can spot drift. */
+    previewPort?: number;
+    /**
+     * What `runly preview` published, kept until the preview is stopped on
+     * purpose (0.20, decision 0045): a daemon restart reaps the tunnel and
+     * publishes it again from this. Written only by `setLeasePreviewRestore`,
+     * never by `saveLease`, so a stale lease snapshot cannot erase it.
+     */
+    previewRestore?: PreviewRestore;
+}
+/** A published preview, as a restart re-publishes it (decision 0045). */
+export interface PreviewRestore {
+    service: string;
+    /** The URL it had; a publisher that pins its address gives the same one again. */
+    url: string;
+    /** The environment's public port it was aimed at. */
+    port: number;
+    /** The publisher, by name; absent for a preview published before 0.20 (the manifest's then). */
+    publisher?: string;
+    /** The settings that reproduce the address (the manifest's `preview` block plus what the publisher pinned). */
+    settings?: Record<string, unknown>;
+}
+/**
+ * A database copy made by `runly db new|with` (decision 0034): a fresh restore
+ * from the same template an environment's datastore uses, with no lease, ports
+ * or services. Everything needed to DROP it is on the row — the command, where
+ * to run it, the files to remove — so it can be reaped after the worktree that
+ * made it is gone.
+ */
+export interface DbCopyRow {
+    name: string;
+    stack: string;
+    stackRoot: string;
+    datastore: string;
+    preset: string;
+    ns: string;
+    url: string;
+    /** `creating` until the restore finished; `dropping` once a drop started (or failed). */
+    state: 'creating' | 'ready' | 'dropping';
+    /** The caller's worktree, the same holder name `up` uses. */
+    holder: string;
+    /** The agent tether, when the caller named one (`--holder-pid`, BACKLOT_HOLDER_PID, or `db with`'s own CLI). */
+    holderPid?: number;
+    holderStart?: number;
+    /** The already-templated drop command (command-family drivers). */
+    dropCmd?: string;
+    /** Where it runs: the worktree, or the state root when that is gone. */
+    dropCwd?: string;
+    /** What to delete instead (sqlite): the copy's own directory under the state root. */
+    dropPath?: string;
+    createdAt: number;
+    dropAttempts: number;
+    nextDropAt: number;
+    /** The template it was restored from (`<stack>/<marker>`), so retention keeps it (decision 0037). */
+    template?: string;
+    /**
+     * `runly db with`'s command (decision 0039): its pid and start time, and its
+     * process group when it runs in its own. A copy dropped while the command
+     * still runs — its CLI was killed — takes the command down first, so
+     * nothing keeps working against a dropped database.
+     */
+    child?: {
+        pid: number;
+        start?: number;
+        pgid?: number;
+    };
+}
+export declare class Journal {
+    private db;
+    constructor(path?: string);
+    /**
+     * One real sqlite transaction around a multi-statement write. Each of these
+     * sequences used to be N independent writes, so a daemon SIGKILLed between
+     * them left a half-state on disk — deleteEnv's env-gone-lease-left was the
+     * reviewed case. The sweeper tolerates those half-states (journals outlive
+     * releases); this stops new ones being minted. BEGIN IMMEDIATE takes the
+     * write lock up front, so the sequence can't interleave with the concurrent
+     * writers that busy_timeout exists for either.
+     */
+    private withTx;
+    private rowToEnv;
+    saveEnv(e: EnvRow): void;
+    /**
+     * Record the proxy's activity clocks (decision 0035). Its own statement, so
+     * a snapshot saved by a verb can never roll them back, and merged so a
+     * clock only moves forward.
+     */
+    saveActivity(id: string, activity: Record<string, number>): void;
+    getEnv(id: string): EnvRow | undefined;
+    envsForStack(stack: string): EnvRow[];
+    allEnvs(): EnvRow[];
+    private rowToDbCopy;
+    saveDbCopy(c: DbCopyRow): void;
+    getDbCopy(name: string): DbCopyRow | undefined;
+    allDbCopies(): DbCopyRow[];
+    deleteDbCopy(name: string): void;
+    deleteEnv(id: string): void;
+    /**
+     * A per-stack env sequence that NEVER reuses a number, even after envs are
+     * reaped — so a recycled env's id can't collide with a live one. Monotonic
+     * in the journal, survives daemon restarts.
+     */
+    nextEnvSeq(stack: string): number;
+    /** Activity, NOT lease renewal: keeps idle reclamation honest without extending ownership. */
+    touchEnv(id: string): void;
+    /** Update just the recorded service pids (auto-restart keeps recovery honest). */
+    updateServicePids(id: string, pids: Record<string, ServicePid>): void;
+    saveLease(l: LeaseRow): void;
+    /**
+     * Forget a lease's preview tunnel — compare-and-swap on the pid.
+     *
+     * Every caller reached here holding a lease SNAPSHOT, so an unconditional
+     * clear let a slow one wipe the record of a tunnel someone else had already
+     * published in its place. That pid is then unnameable, and an unnameable
+     * preview pid is a public, unauthenticated URL that serves forever. Passing
+     * the pid that was actually stopped makes a stale caller a no-op; the
+     * pid-less form is only for a row that is being deleted outright.
+     */
+    clearLeasePreview(id: string, stoppedPid?: number): void;
+    /** Record (or, with null, forget) what a restart re-publishes for this lease (decision 0045). */
+    setLeasePreviewRestore(id: string, restore: PreviewRestore | null): void;
+    private rowToLease;
+    leaseForHolder(holder: string, stack: string): LeaseRow | undefined;
+    leaseForEnv(envId: string): LeaseRow | undefined;
+    allLeases(): LeaseRow[];
+    deleteLease(id: string): void;
+    /** Shift every deadline by `ms` — the sleep pardon (decision 0009). */
+    pardon(ms: number): void;
+}