@nylorun/runtime 0.22.0-beta → 0.24.0-beta

package/dist/store/postgres/store.js

@@ -15,7 +15,7 @@
  *
  * Every `tx` is one READ COMMITTED transaction on a pooled connection (Drizzle's
  * `transaction` on postgres.js `begin`). Tables are addressed by schema-qualified names,
- * never through `search_path`. Commit listeners and `afterCommit` callbacks run after `COMMIT`
+ * never through `search_path`. Commit listeners and outbox deliveries run after `COMMIT`
  * returns and never reject the committed `tx`. Nested `tx` calls are detected with
  * `AsyncLocalStorage` and rejected, and a `Tx` rejects every call once its callback settles.
  * A failed statement rejects `tx` with the driver's error (`driverError`), not Drizzle's
@@ -32,7 +32,7 @@
  *    head).
  * 2. A transaction that touches several sessions locks them in ascending id
  *    order, with `lockSessions` (`./locking.ts`), before writing any of them.
- * 3. A sandbox resource's row (`sandboxResource(id, { lock: true })`, which orders its
+ * 3. An Environment's row (`environment(id, { lock: true })`, which orders its
  *    lifecycle stream) is locked after every session row the transaction locks, never
  *    before one.
  *
@@ -50,15 +50,15 @@
  */
 import { AsyncLocalStorage } from "node:async_hooks";
 import { randomUUID } from "node:crypto";
-import { and, count, eq, getTableColumns, gte, inArray, isNotNull, isNull, lt, lte, ne, or, sql, } from "drizzle-orm";
-import { appendEvent, appendSandboxEvent } from "../../record/index.js";
+import { and, count, eq, getTableColumns, gte, gt, inArray, isNotNull, isNull, lt, lte, ne, or, sql, } from "drizzle-orm";
+import { appendEnvironmentEvent, appendEvent } from "../../record/index.js";
 import { OwnershipLostError, PrincipalRoleConflict } from "../ownership.js";
 import { followControlSignals, pruneSignals, writeSignal } from "./control.js";
 import { database, driverError } from "./db.js";
 import { expectedSchemaVersion, readSchemaVersion } from "./migrate.js";
 import { createPostgresRecordReader } from "./record.js";
-import { postgresRecordWriter, postgresSandboxRecordWriter } from "./record-writer.js";
-import { artifacts, artifactVersions, artifactContent, definitionFiles, definitionFileUses, commands, definitions, effects, links, modelBudgets, modelUsage, principals, sandboxes, sandboxEvents, sandboxResources, sessionEvents, sessionLogHeads, sessions, toJson, toolCrossings, signingKeys, tenant, tenantSettings, vaultAudit, vaultCredentials, vaultIdempotency, vaults, wakes, } from "./schema.js";
+import { postgresEnvironmentRecordWriter, postgresRecordWriter } from "./record-writer.js";
+import { artifacts, artifactVersions, artifactContent, definitionFiles, definitionFileUses, commands, definitions, effects, links, modelBudgets, modelUsage, principals, sandboxes, environmentEvents, environments, sessionEvents, sessionLogHeads, sessions, toJson, toolCrossings, signingKeys, tenant, tenantSettings, vaultAudit, vaultCredentials, vaultIdempotency, vaults, sandboxSignals, wakes, } from "./schema.js";
 export function createPostgresSessionStore(options) {
     return new PostgresSessionStore(options);
 }
@@ -88,6 +88,9 @@ class PostgresSessionStore {
     active = new AsyncLocalStorage();
     listeners = new Set();
     inflight = new Set();
+    /** Deliveries of committed outbox requests still running (`deliverTo`). */
+    deliveries = new Set();
+    deliver;
     now;
     onError;
     closed = false;
@@ -111,7 +114,7 @@ class PostgresSessionStore {
         let t;
         // Wrapped so `begin` does not treat an array result as queries to await.
         const run = this.db.transaction(async (db) => {
-            t = new PostgresTx(db, this.tenantId, this.now, this.deleteWake);
+            t = new PostgresTx(db, this.tenantId, this.now);
             try {
                 return { value: await this.active.run(this, () => fn(t)) };
             }
@@ -144,20 +147,51 @@ class PostgresSessionStore {
                 }
             }
         }
-        for (const callback of t.callbacks) {
-            try {
-                await callback();
-            }
-            catch (error) {
-                this.onError(error);
-            }
-        }
+        if (t.outbox.length > 0)
+            this.send(t.outbox);
         return result.value;
     }
-    /** Deletes a delivered outbox request, outside any transaction (`Tx.wake`). */
-    deleteWake = async (id) => {
-        await this.db.delete(wakes).where(eq(wakes.id, id));
-    };
+    deliverTo(deliver) {
+        this.deliver = deliver;
+    }
+    async delivered() {
+        while (this.deliveries.size > 0)
+            await Promise.allSettled([...this.deliveries]);
+    }
+    /**
+     * Hands a commit's outbox requests to the delivery, one after the other, without the caller
+     * waiting: a delivered one's row is deleted, and one that was not is due at the sweep's next
+     * pass.
+     */
+    send(committed) {
+        const deliver = this.deliver;
+        if (!deliver)
+            return;
+        const run = (async () => {
+            for (const { kind, id, request } of committed) {
+                let sent = false;
+                try {
+                    sent = await deliver(request);
+                }
+                catch (error) {
+                    this.onError(error);
+                }
+                const table = OUTBOX[kind];
+                try {
+                    if (sent === false)
+                        await this.db.update(table).set({ retryAt: sql `clock_timestamp()` }).where(eq(table.id, id));
+                    else
+                        await this.db.delete(table).where(eq(table.id, id));
+                }
+                catch (error) {
+                    // The row stays as it was: the sweep sends it again, under the same key.
+                    this.onError(error);
+                }
+            }
+        })();
+        this.deliveries.add(run);
+        void run.finally(() => this.deliveries.delete(run));
+    }
     onCommit(listener) {
         this.listeners.add(listener);
         return () => this.listeners.delete(listener);
@@ -186,10 +220,14 @@ class PostgresSessionStore {
             throw new Error("SessionStore is closed");
         return followControlSignals(this.sql, this.db, onSignal, options);
     }
-    /** Rejects new transactions and waits for running ones. Does not end the pool. */
+    /**
+     * Rejects new transactions and waits for running ones, and for the deliveries of committed
+     * outbox requests. Does not end the pool.
+     */
     async close() {
         this.closed = true;
         await Promise.allSettled([...this.inflight]);
+        await this.delivered();
     }
 }
 function storedSession(row) {
@@ -202,14 +240,21 @@ function ownership(row) {
         ownerExpiresAt: row.ownerExpiresAt?.toISOString() ?? null,
     };
 }
-function sandboxResourceOf(row) {
+function environmentOf(row) {
     return {
         id: row.id,
         kind: row.kind,
-        spec: row.spec,
         labels: row.labels,
+        recipe: row.recipe,
+        applied: row.applied,
+        version: row.version,
+        appliedVersion: row.appliedVersion,
+        ...(row.ownerUserId !== null ? { ownerUserId: row.ownerUserId } : {}),
+        lastUsedAt: new Date(row.lastUsedAt).toISOString(),
         createdAt: row.createdAt,
         updatedAt: row.updatedAt,
+        ...(row.diskOp ? { diskOp: row.diskOp } : {}),
+        ...(row.diskSeed ? { diskSeed: row.diskSeed } : {}),
         ...(row.k8sName !== null && row.desired !== null && row.observed !== null
             ? {
                 pod: {
@@ -222,7 +267,6 @@ function sandboxResourceOf(row) {
                     ...(row.joinTokenHash !== null ? { joinTokenHash: row.joinTokenHash } : {}),
                     rev: row.rev,
                     ...(row.lastActiveAt ? { lastActiveAt: row.lastActiveAt.toISOString() } : {}),
-                    ...(row.expiresAt ? { expiresAt: row.expiresAt.toISOString() } : {}),
                     ...(row.startedAt ? { startedAt: row.startedAt.toISOString() } : {}),
                     ...(row.reason !== null ? { reason: row.reason } : {}),
                     ...(row.retiring !== null ? { retiring: row.retiring } : {}),
@@ -245,7 +289,6 @@ function podColumns(patch) {
         joinTokenHash: patch.joinTokenHash,
         rev: patch.rev,
         lastActiveAt: date(patch.lastActiveAt),
-        expiresAt: date(patch.expiresAt),
         startedAt: date(patch.startedAt),
         reason: patch.reason,
         retiring: patch.retiring,
@@ -262,26 +305,49 @@ const SIGNING_KEY_STAMP = {
 /** The audit columns a row has (`ord` only orders them). */
 const { ord: _ord, ...AUDIT } = getTableColumns(vaultAudit);
 const SESSION_TABLES = [sessions, commands, effects, links, wakes];
-/** An outbox row as it is delivered: the wake's own dedupe key, or one made from the row id. */
+/**
+ * The outbox's tables, by the kind of request each holds. Their delivery columns (`id`,
+ * `created_at`, `attempts`, `retry_at`, `parked_at`) are the same.
+ */
+const OUTBOX = {
+    wake: wakes,
+    sandbox: sandboxSignals,
+};
+/** A wake outbox row as it is delivered: its own dedupe key, or one made from the row id. */
 function outboxWake(id, reason, dedupeKey) {
     return { reason, dedupeKey: dedupeKey ?? `wake:${id}` };
 }
+/** A sandbox signal outbox row as it is delivered: a reconcile is keyed by the row. */
+function outboxSignal(id, row) {
+    return row.kind === "arm"
+        ? { kind: "arm", timer: row.timer, at: row.at }
+        : { kind: "reconcile", key: `signal:${id}` };
+}
+/** When an outbox row of `table` is due, after a grace of `graceMs` (the database's clock). */
+function dueAt(table, graceMs) {
+    const grace = sql `make_interval(secs => ${Math.max(0, graceMs) / 1000})`;
+    return sql `coalesce(${table.retryAt}, ${table.createdAt} + ${grace})`;
+}
+/** Due rows of `table`, soonest first: their epoch seconds, to merge two tables' rows. */
+const DUE = (table, graceMs) => ({
+    due: dueAt(table, graceMs),
+    dueSeconds: sql `extract(epoch from ${dueAt(table, graceMs)})::float8`,
+});
 /** The inserted row's `column` (`ON CONFLICT … DO UPDATE`). */
 const excluded = (column) => sql.raw(`excluded.${column}`);
 class PostgresTx {
     db;
     tenantId;
     now;
-    deleteWake;
     closed = false;
     events = [];
     generations = [];
-    callbacks = [];
-    constructor(db, tenantId, now, deleteWake) {
+    /** The outbox requests this transaction wrote, delivered after it commits. */
+    outbox = [];
+    constructor(db, tenantId, now) {
         this.db = db;
         this.tenantId = tenantId;
         this.now = now;
-        this.deleteWake = deleteWake;
     }
     check() {
         if (this.closed)
@@ -328,7 +394,7 @@ class PostgresTx {
         const [row] = await this.sessions().where(eq(sessions.id, id)).for("update");
         return row && storedSession(row);
     }
-    async event(sessionId, turnId, type, payload) {
+    async event(sessionId, turnId, type, payload, writer) {
         this.check();
         const db = this.db;
         // The session row lock orders the session's events; the log head allocates under it.
@@ -347,17 +413,14 @@ class PostgresTx {
             time: this.now(),
             type,
             payload,
+            ...(writer ? { writer } : {}),
         });
         this.events.push(event);
         this.generations.push(generation);
         return structuredClone(event);
     }
-    afterCommit(fn) {
-        this.check();
-        this.callbacks.push(fn);
-    }
     // --- wake outbox ---------------------------------------------------------
-    async wake(sessionId, wake, deliver) {
+    async wake(sessionId, wake) {
         this.check();
         const id = randomUUID();
         await this.db.insert(wakes).values({
@@ -366,34 +429,90 @@ class PostgresTx {
             reason: wake.reason,
             dedupeKey: wake.dedupeKey ?? null,
         });
-        const delivered = outboxWake(id, wake.reason, wake.dedupeKey ?? null);
-        // A rejection goes to the store's error hook (as every callback's), and the row stays.
-        this.callbacks.push(async () => {
-            if ((await deliver(sessionId, delivered)) === false)
-                return;
-            await this.deleteWake(id);
+        this.outbox.push({
+            kind: "wake",
+            id,
+            request: { kind: "wake", sessionId, wake: outboxWake(id, wake.reason, wake.dedupeKey ?? null) },
         });
     }
-    async pendingWakes(before, limit) {
+    async signalSandbox(sandboxId, signal) {
         this.check();
-        const rows = await this.db
-            .select()
+        const id = randomUUID();
+        const row = {
+            kind: signal.kind,
+            timer: signal.kind === "arm" ? signal.timer : null,
+            at: signal.kind === "arm" ? signal.at : null,
+        };
+        await this.db.insert(sandboxSignals).values({ id, sandboxId, ...row });
+        this.outbox.push({
+            kind: "sandbox",
+            id,
+            request: { kind: "sandbox", sandboxId, signal: outboxSignal(id, row) },
+        });
+    }
+    async pendingOutbox(graceMs, limit) {
+        this.check();
+        // Compared on the database's clock, which wrote `created_at` and `retry_at`.
+        const now = sql `clock_timestamp()`;
+        const wake = DUE(wakes, graceMs);
+        const signal = DUE(OUTBOX.sandbox, graceMs);
+        const wakeRows = await this.db
+            .select({ ...getTableColumns(wakes), dueSeconds: wake.dueSeconds })
             .from(wakes)
-            .where(lt(wakes.createdAt, before))
-            .orderBy(wakes.createdAt, wakes.id)
+            .where(and(isNull(wakes.parkedAt), lte(wake.due, now)))
+            .orderBy(wake.due, wakes.id)
             .limit(limit);
-        return rows.map((row) => ({
-            id: row.id,
-            sessionId: row.sessionId,
-            wake: outboxWake(row.id, row.reason, row.dedupeKey),
-            createdAt: row.createdAt.toISOString(),
-        }));
+        const signalRows = await this.db
+            .select({ ...getTableColumns(sandboxSignals), dueSeconds: signal.dueSeconds })
+            .from(sandboxSignals)
+            .where(and(isNull(sandboxSignals.parkedAt), lte(signal.due, now)))
+            .orderBy(signal.due, sandboxSignals.id)
+            .limit(limit);
+        const rows = [
+            ...wakeRows.map((row) => ({
+                kind: "wake",
+                id: row.id,
+                request: {
+                    kind: "wake",
+                    sessionId: row.sessionId,
+                    wake: outboxWake(row.id, row.reason, row.dedupeKey),
+                },
+                createdAt: row.createdAt.toISOString(),
+                attempts: row.attempts,
+                due: row.dueSeconds,
+            })),
+            ...signalRows.map((row) => ({
+                kind: "sandbox",
+                id: row.id,
+                request: { kind: "sandbox", sandboxId: row.sandboxId, signal: outboxSignal(row.id, row) },
+                createdAt: row.createdAt.toISOString(),
+                attempts: row.attempts,
+                due: row.dueSeconds,
+            })),
+        ];
+        rows.sort((a, b) => a.due - b.due || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+        return rows.slice(0, limit).map(({ due: _due, ...row }) => row);
+    }
+    async deleteOutbox(rows) {
+        this.check();
+        for (const kind of ["wake", "sandbox"]) {
+            const ids = rows.filter((row) => row.kind === kind).map((row) => row.id);
+            if (ids.length > 0)
+                await this.db.delete(OUTBOX[kind]).where(inArray(OUTBOX[kind].id, ids));
+        }
     }
-    async deleteWakes(ids) {
+    async failOutbox(row, retry) {
         this.check();
-        if (ids.length === 0)
-            return;
-        await this.db.delete(wakes).where(inArray(wakes.id, [...ids]));
+        const table = OUTBOX[row.kind];
+        const secs = Math.max(0, retry.retryInMs) / 1000;
+        await this.db
+            .update(table)
+            .set({
+            attempts: sql `${table.attempts} + 1`,
+            retryAt: sql `clock_timestamp() + make_interval(secs => ${secs})`,
+            ...(retry.park ? { parkedAt: sql `clock_timestamp()` } : {}),
+        })
+            .where(eq(table.id, row.id));
     }
     // --- ownership -----------------------------------------------------------
     async takeOwnership(sessionId, claim) {
@@ -470,12 +589,11 @@ class PostgresTx {
             .orderBy(sessions.id);
         return rows.map((row) => storedSession(row));
     }
-    async orphanedSessions(now, limit) {
+    async orphanedSessions(now, limit, after) {
         this.check();
         const rows = await this.sessions()
-            .where(and(inArray(sessions.status, OPEN_SESSION), or(isNull(sessions.owner), isNull(sessions.ownerExpiresAt), lte(sessions.ownerExpiresAt, now))))
-            // Never-owned sessions first, then the longest expired.
-            .orderBy(sql `CASE WHEN ${sessions.owner} IS NULL THEN NULL ELSE ${sessions.ownerExpiresAt} END ASC NULLS FIRST`, sessions.id)
+            .where(and(inArray(sessions.status, OPEN_SESSION), or(isNull(sessions.owner), isNull(sessions.ownerExpiresAt), lte(sessions.ownerExpiresAt, now)), after === undefined ? undefined : gt(sessions.id, after)))
+            .orderBy(sessions.id)
             .limit(limit);
         return rows.map((row) => storedSession(row));
     }
@@ -500,88 +618,92 @@ class PostgresTx {
         this.check();
         return this.bodies(sandboxes);
     }
-    // --- sandbox resources ---------------------------------------------------
-    async sandboxResource(id, options = {}) {
+    // --- Environments ----------------------------------------------------------
+    async environment(id, options = {}) {
         this.check();
-        const query = this.db.select().from(sandboxResources).where(eq(sandboxResources.id, id));
+        const query = this.db.select().from(environments).where(eq(environments.id, id));
         const [row] = options.lock ? await query.for("update") : await query;
-        return row && sandboxResourceOf(row);
+        return row && environmentOf(row);
     }
-    async createSandboxResource(row, limit) {
+    async createEnvironment(row, limit) {
         this.check();
         // Every create takes this lock, so two never both see room for one more.
-        await this.db.execute(sql `SELECT pg_advisory_xact_lock(hashtext('nylorun.sandbox_resources'))`);
+        await this.db.execute(sql `SELECT pg_advisory_xact_lock(hashtext('nylorun.environments'))`);
         const [found] = await this.db
-            .select({ id: sandboxResources.id })
-            .from(sandboxResources)
-            .where(eq(sandboxResources.id, row.id));
+            .select({ id: environments.id })
+            .from(environments)
+            .where(eq(environments.id, row.id));
         if (found)
             return "exists";
-        const [counted] = await this.db.select({ n: count() }).from(sandboxResources);
+        const [counted] = await this.db.select({ n: count() }).from(environments);
         if ((counted?.n ?? 0) >= limit)
             return "limit";
-        const { pod, ...resource } = row;
-        await this.db.insert(sandboxResources).values({ ...resource, ...(pod ? podColumns(pod) : {}) });
+        const { pod, ownerUserId, diskOp, diskSeed, ...environment } = row;
+        await this.db.insert(environments).values({
+            ...environment,
+            ownerUserId: ownerUserId ?? null,
+            diskOp: diskOp ?? null,
+            diskSeed: diskSeed ?? null,
+            ...(pod ? podColumns(pod) : {}),
+        });
         return "created";
     }
-    async updateSandboxSpec(id, spec, updatedAt) {
-        this.check();
-        await this.db.update(sandboxResources).set({ spec, updatedAt }).where(eq(sandboxResources.id, id));
-    }
-    async updateSandboxPod(id, patch, updatedAt) {
+    async updateEnvironment(id, patch, updatedAt) {
         this.check();
+        const columns = Object.fromEntries(Object.entries(patch).filter(([, value]) => value !== undefined));
         await this.db
-            .update(sandboxResources)
-            .set({ ...podColumns(patch), updatedAt })
-            .where(eq(sandboxResources.id, id));
+            .update(environments)
+            .set({ ...columns, updatedAt })
+            .where(eq(environments.id, id));
     }
-    async updateSandboxLabels(id, labels, updatedAt) {
+    async updateEnvironmentPod(id, patch, updatedAt) {
         this.check();
         await this.db
-            .update(sandboxResources)
-            .set({ labels, updatedAt })
-            .where(eq(sandboxResources.id, id));
+            .update(environments)
+            .set({ ...podColumns(patch), updatedAt })
+            .where(eq(environments.id, id));
     }
-    async deleteSandboxResource(id) {
+    async deleteEnvironment(id) {
         this.check();
-        await this.db.delete(sandboxResources).where(eq(sandboxResources.id, id));
+        // Its vault and credentials go with it (ON DELETE CASCADE).
+        await this.db.delete(environments).where(eq(environments.id, id));
     }
-    async listSandboxResources(filter = {}) {
+    async listEnvironments(filter = {}) {
         this.check();
         const labels = filter.labels;
         const rows = await this.db
             .select()
-            .from(sandboxResources)
+            .from(environments)
             .where(labels === undefined || Object.keys(labels).length === 0
             ? undefined
-            : sql `nylorun.doc(${sandboxResources.labels}) @> ${toJson(labels)}::jsonb`)
-            .orderBy(sandboxResources.id);
-        return rows.map(sandboxResourceOf);
+            : sql `nylorun.doc(${environments.labels}) @> ${toJson(labels)}::jsonb`)
+            .orderBy(environments.id);
+        return rows.map(environmentOf);
     }
-    async sessionsOnSandbox(sandboxId) {
+    async sessionsOnEnvironment(environmentId) {
         this.check();
         const rows = await this.sessions()
-            .where(eq(sessions.sandboxId, sandboxId))
+            .where(eq(sessions.environmentId, environmentId))
             .orderBy(sessions.id);
         return rows.map((row) => storedSession(row));
     }
-    async sandboxEvent(sandboxId, type, payload) {
+    async environmentEvent(environmentId, type, payload) {
         this.check();
-        return appendSandboxEvent(postgresSandboxRecordWriter(this.db), {
+        return appendEnvironmentEvent(postgresEnvironmentRecordWriter(this.db), {
             tenantId: this.tenantId,
-            sandboxId,
+            environmentId,
             time: this.now(),
             type,
             payload,
         });
     }
-    async sandboxEvents(sandboxId, options = {}) {
+    async environmentEvents(environmentId, options = {}) {
         this.check();
         const rows = await this.db
-            .select({ body: sandboxEvents.body })
-            .from(sandboxEvents)
-            .where(and(eq(sandboxEvents.sandboxId, sandboxId), options.fromSeq === undefined ? undefined : gte(sandboxEvents.seq, options.fromSeq)))
-            .orderBy(sandboxEvents.seq)
+            .select({ body: environmentEvents.body })
+            .from(environmentEvents)
+            .where(and(eq(environmentEvents.environmentId, environmentId), options.fromSeq === undefined ? undefined : gte(environmentEvents.seq, options.fromSeq)))
+            .orderBy(environmentEvents.seq)
             .limit(options.limit ?? 1000);
         return rows.map((row) => row.body);
     }
@@ -719,21 +841,13 @@ class PostgresTx {
         const [row] = await this.db.select().from(vaults).where(eq(vaults.id, id));
         return row;
     }
-    async vaultsByOwner(ownerUserId) {
-        this.check();
-        return this.db
-            .select()
-            .from(vaults)
-            .where(and(eq(vaults.ownerUserId, ownerUserId), eq(vaults.scope, "user")))
-            .orderBy(vaults.createdAt, vaults.id);
-    }
-    async installationVaults() {
+    async vaultOfEnvironment(environmentId) {
         this.check();
-        return this.db
+        const [row] = await this.db
             .select()
             .from(vaults)
-            .where(eq(vaults.scope, "installation"))
-            .orderBy(vaults.createdAt, vaults.id);
+            .where(eq(vaults.environmentId, environmentId));
+        return row;
     }
     async updateVaultMetadata(id, metadataJson) {
         this.check();
@@ -756,6 +870,14 @@ class PostgresTx {
             .where(eq(vaultCredentials.id, id));
         return row;
     }
+    async credentialByName(vaultId, name) {
+        this.check();
+        const [row] = await this.db
+            .select()
+            .from(vaultCredentials)
+            .where(and(eq(vaultCredentials.vaultId, vaultId), eq(vaultCredentials.name, name)));
+        return row;
+    }
     async credentialsForVault(vaultId, filter = {}) {
         this.check();
         return this.db
@@ -864,6 +986,16 @@ class PostgresTx {
             .returning();
         return inserted;
     }
+    async modelUsageOf(effectKey) {
+        this.check();
+        const [row] = await this.db
+            .select()
+            .from(modelUsage)
+            .where(and(eq(modelUsage.effectKey, effectKey), eq(modelUsage.duplicate, false)))
+            .orderBy(modelUsage.createdAt, modelUsage.id)
+            .limit(1);
+        return row;
+    }
     async modelUsageTotals(query) {
         this.check();
         const [row] = await this.db
@@ -904,6 +1036,15 @@ class PostgresTx {
             .returning({ key: toolCrossings.key });
         return inserted.length === 1;
     }
+    async restartToolCrossing(row) {
+        this.check();
+        const restarted = await this.db
+            .update(toolCrossings)
+            .set({ startedAt: row.startedAt })
+            .where(and(eq(toolCrossings.key, row.key), eq(toolCrossings.hash, row.hash), isNull(toolCrossings.settledAt)))
+            .returning({ key: toolCrossings.key });
+        return restarted.length === 1;
+    }
     async settleToolCrossing(key, answer, settledAt) {
         this.check();
         await this.db
@@ -926,6 +1067,12 @@ class PostgresTx {
         const [row] = options.lock ? await query.for("update") : await query;
         return row;
     }
+    async diskArtifact(environmentId, options = {}) {
+        this.check();
+        const query = this.db.select().from(artifacts).where(eq(artifacts.environmentId, environmentId));
+        const [row] = options.lock ? await query.for("update") : await query;
+        return row;
+    }
     async listArtifacts(filter = {}) {
         this.check();
         return this.db
@@ -976,6 +1123,24 @@ class PostgresTx {
         await this.db.delete(artifacts).where(eq(artifacts.id, id));
         return deleted.map((row) => row.blobKey);
     }
+    async deleteArtifactVersion(artifactId, version) {
+        this.check();
+        const [deleted] = await this.db
+            .delete(artifactVersions)
+            .where(and(eq(artifactVersions.artifactId, artifactId), eq(artifactVersions.version, version)))
+            .returning({ blobKey: artifactVersions.blobKey });
+        if (!deleted)
+            return undefined;
+        const [left] = await this.db
+            .select({ latest: sql `max(${artifactVersions.version})` })
+            .from(artifactVersions)
+            .where(eq(artifactVersions.artifactId, artifactId));
+        await this.db
+            .update(artifacts)
+            .set({ latestVersion: Number(left?.latest ?? 0), updatedAt: new Date().toISOString() })
+            .where(eq(artifacts.id, artifactId));
+        return deleted.blobKey;
+    }
     async lockArtifactQuota() {
         this.check();
         await this.db.execute(sql `SELECT pg_advisory_xact_lock(hashtext('nylorun.artifacts'))`);
@@ -1116,10 +1281,12 @@ class PostgresTx {
                 basinGeneration: sql `${tenant.basinGeneration} + 1`,
             });
         }
-        if (scope === "sandboxes" || scope === "all") {
+        if (scope === "environments" || scope === "all") {
             await db.delete(sandboxes);
-            await db.delete(sandboxResources);
-            await db.delete(sandboxEvents);
+            // Their vaults and credentials go with them (ON DELETE CASCADE).
+            await db.delete(environments);
+            await db.delete(environmentEvents);
+            await db.delete(sandboxSignals);
         }
         if (scope === "all") {
             await db.delete(definitions);
@@ -1128,7 +1295,6 @@ class PostgresTx {
             await db.delete(modelUsage);
             await db.delete(modelBudgets);
             await db.delete(artifacts);
-            await db.delete(vaults).where(ne(vaults.scope, "host"));
         }
     }
 }