@nylorun/runtime 0.15.2-beta → 0.17.0-beta

package/dist/store/postgres/store.js

@@ -32,6 +32,9 @@
  *    head).
  * 2. A transaction that touches several sessions locks them in ascending id
  *    order, with `lockSessions` (`./locking.ts`), before writing any of them.
+ * 3. A sandbox resource's row (`sandboxResource(id, { lock: true })`, which orders its
+ *    lifecycle stream) is locked after every session row the transaction locks, never
+ *    before one.
  *
  * Serialization failures and deadlocks are not retried here: with the order
  * above, READ COMMITTED transactions do not raise them. A `40P01` means a
@@ -47,17 +50,19 @@
  */
 import { AsyncLocalStorage } from "node:async_hooks";
 import { and, count, eq, getTableColumns, gte, inArray, isNotNull, isNull, lt, lte, ne, or, sql, } from "drizzle-orm";
-import { appendEvent } from "../../record/index.js";
+import { appendEvent, appendSandboxEvent } from "../../record/index.js";
 import { OwnershipLostError } from "../ownership.js";
 import { database, driverError } from "./db.js";
 import { expectedSchemaVersion, readSchemaVersion } from "./migrate.js";
 import { createPostgresRecordReader } from "./record.js";
-import { postgresRecordWriter } from "./record-writer.js";
-import { actions, commands, definitions, effects, endpoints, links, modelBudgets, modelUsage, principals, publishableKeys, sandboxes, sessionEvents, sessionLogHeads, sessions, toolCrossings, signingKeys, subjectEpochs, subjectUsage, tenant, tenantSettings, vaultAudit, vaultCredentials, vaultIdempotency, vaults, } from "./schema.js";
+import { postgresRecordWriter, postgresSandboxRecordWriter } from "./record-writer.js";
+import { actions, artifacts, artifactVersions, artifactContent, commands, definitions, effects, endpoints, links, modelBudgets, modelUsage, principals, sandboxes, sandboxEvents, sandboxResources, sessionEvents, sessionLogHeads, sessions, toJson, toolCrossings, signingKeys, tenant, tenantSettings, vaultAudit, vaultCredentials, vaultIdempotency, vaults, oauthPending, } from "./schema.js";
 export function createPostgresSessionStore(options) {
     return new PostgresSessionStore(options);
 }
 const OPEN_SESSION = ["running", "runnable"];
+/** Rows per statement when a folder version names many files. */
+const CONTENT_BATCH = 1000;
 const DOCUMENTS = {
     definitions,
     sessions: sessions,
@@ -187,6 +192,57 @@ function ownership(row) {
         ownerExpiresAt: row.ownerExpiresAt?.toISOString() ?? null,
     };
 }
+function sandboxResourceOf(row) {
+    return {
+        id: row.id,
+        kind: row.kind,
+        spec: row.spec,
+        labels: row.labels,
+        createdAt: row.createdAt,
+        updatedAt: row.updatedAt,
+        ...(row.k8sName !== null && row.desired !== null && row.observed !== null
+            ? {
+                pod: {
+                    k8sName: row.k8sName,
+                    volumeGen: row.volumeGen,
+                    desired: row.desired,
+                    observed: row.observed,
+                    ...(row.podUid !== null ? { podUid: row.podUid } : {}),
+                    hostEpoch: row.hostEpoch,
+                    ...(row.joinTokenHash !== null ? { joinTokenHash: row.joinTokenHash } : {}),
+                    rev: row.rev,
+                    ...(row.lastActiveAt ? { lastActiveAt: row.lastActiveAt.toISOString() } : {}),
+                    ...(row.expiresAt ? { expiresAt: row.expiresAt.toISOString() } : {}),
+                    ...(row.startedAt ? { startedAt: row.startedAt.toISOString() } : {}),
+                    ...(row.reason !== null ? { reason: row.reason } : {}),
+                    ...(row.retiring !== null ? { retiring: row.retiring } : {}),
+                    ...(row.volumeId !== null ? { volumeId: row.volumeId } : {}),
+                },
+            }
+            : {}),
+    };
+}
+/** A pod patch as column values: ISO times become dates, `null` clears. */
+function podColumns(patch) {
+    const date = (value) => value === undefined ? undefined : value === null ? null : new Date(value);
+    const columns = {
+        k8sName: patch.k8sName,
+        volumeGen: patch.volumeGen,
+        desired: patch.desired,
+        observed: patch.observed,
+        podUid: patch.podUid,
+        hostEpoch: patch.hostEpoch,
+        joinTokenHash: patch.joinTokenHash,
+        rev: patch.rev,
+        lastActiveAt: date(patch.lastActiveAt),
+        expiresAt: date(patch.expiresAt),
+        startedAt: date(patch.startedAt),
+        reason: patch.reason,
+        retiring: patch.retiring,
+        volumeId: patch.volumeId,
+    };
+    return Object.fromEntries(Object.entries(columns).filter(([, value]) => value !== undefined));
+}
 /** An endpoint row with its unset (null) columns left out, as `EndpointRow` has them. */
 function endpointRow(row) {
     return Object.fromEntries(Object.entries(row).filter(([, value]) => value !== null));
@@ -345,6 +401,18 @@ class PostgresTx {
             throw new OwnershipLostError(sessionId, epoch, session.epoch);
         return session;
     }
+    async runState(sessionId) {
+        this.check();
+        const [row] = await this.db
+            .select({
+            epoch: sessions.epoch,
+            status: sessions.status,
+            activeTurnId: sql `nylorun.doc(${sessions.body})->>'activeTurnId'`,
+        })
+            .from(sessions)
+            .where(eq(sessions.id, sessionId));
+        return row && { epoch: row.epoch, status: row.status ?? "", activeTurnId: row.activeTurnId ?? null };
+    }
     // --- typed queries -------------------------------------------------------
     async sessionsWithStatus(statuses) {
         this.check();
@@ -383,6 +451,91 @@ class PostgresTx {
         this.check();
         return this.bodies(sandboxes);
     }
+    // --- sandbox resources ---------------------------------------------------
+    async sandboxResource(id, options = {}) {
+        this.check();
+        const query = this.db.select().from(sandboxResources).where(eq(sandboxResources.id, id));
+        const [row] = options.lock ? await query.for("update") : await query;
+        return row && sandboxResourceOf(row);
+    }
+    async createSandboxResource(row, limit) {
+        this.check();
+        // Every create takes this lock, so two never both see room for one more.
+        await this.db.execute(sql `SELECT pg_advisory_xact_lock(hashtext('nylorun.sandbox_resources'))`);
+        const [found] = await this.db
+            .select({ id: sandboxResources.id })
+            .from(sandboxResources)
+            .where(eq(sandboxResources.id, row.id));
+        if (found)
+            return "exists";
+        const [counted] = await this.db.select({ n: count() }).from(sandboxResources);
+        if ((counted?.n ?? 0) >= limit)
+            return "limit";
+        const { pod, ...resource } = row;
+        await this.db.insert(sandboxResources).values({ ...resource, ...(pod ? podColumns(pod) : {}) });
+        return "created";
+    }
+    async updateSandboxSpec(id, spec, updatedAt) {
+        this.check();
+        await this.db.update(sandboxResources).set({ spec, updatedAt }).where(eq(sandboxResources.id, id));
+    }
+    async updateSandboxPod(id, patch, updatedAt) {
+        this.check();
+        await this.db
+            .update(sandboxResources)
+            .set({ ...podColumns(patch), updatedAt })
+            .where(eq(sandboxResources.id, id));
+    }
+    async updateSandboxLabels(id, labels, updatedAt) {
+        this.check();
+        await this.db
+            .update(sandboxResources)
+            .set({ labels, updatedAt })
+            .where(eq(sandboxResources.id, id));
+    }
+    async deleteSandboxResource(id) {
+        this.check();
+        await this.db.delete(sandboxResources).where(eq(sandboxResources.id, id));
+    }
+    async listSandboxResources(filter = {}) {
+        this.check();
+        const labels = filter.labels;
+        const rows = await this.db
+            .select()
+            .from(sandboxResources)
+            .where(labels === undefined || Object.keys(labels).length === 0
+            ? undefined
+            : sql `nylorun.doc(${sandboxResources.labels}) @> ${toJson(labels)}::jsonb`)
+            .orderBy(sandboxResources.id);
+        return rows.map(sandboxResourceOf);
+    }
+    async sessionsOnSandbox(sandboxId) {
+        this.check();
+        const rows = await this.sessions()
+            .where(eq(sessions.sandboxId, sandboxId))
+            .orderBy(sessions.id);
+        return rows.map((row) => storedSession(row));
+    }
+    async sandboxEvent(sandboxId, type, payload) {
+        this.check();
+        return appendSandboxEvent(postgresSandboxRecordWriter(this.db), {
+            tenantId: this.tenantId,
+            sandboxId,
+            time: this.now(),
+            type,
+            payload,
+        });
+    }
+    async sandboxEvents(sandboxId, options = {}) {
+        this.check();
+        const rows = await this.db
+            .select({ body: sandboxEvents.body })
+            .from(sandboxEvents)
+            .where(and(eq(sandboxEvents.sandboxId, sandboxId), options.fromSeq === undefined ? undefined : gte(sandboxEvents.seq, options.fromSeq)))
+            .orderBy(sandboxEvents.seq)
+            .limit(options.limit ?? 1000);
+        return rows.map((row) => row.body);
+    }
     async actionBodies(where) {
         const rows = await this.db
             .select({ body: actions.body })
@@ -607,6 +760,30 @@ class PostgresTx {
             .orderBy(principals.id);
         return rows.map((row) => row.tokenHash);
     }
+    async listPrincipals() {
+        this.check();
+        return await this.db.select().from(principals).orderBy(principals.id);
+    }
+    async putPrincipal(id, tokenHash, createdAt) {
+        this.check();
+        const [row] = await this.db
+            .insert(principals)
+            .values({ id, role: "application", tokenHash, idempotencyKey: null, createdAt })
+            .onConflictDoUpdate({
+            target: principals.id,
+            set: { tokenHash, createdAt, idempotencyKey: null },
+        })
+            .returning();
+        return row;
+    }
+    async deletePrincipal(id) {
+        this.check();
+        const rows = await this.db
+            .delete(principals)
+            .where(eq(principals.id, id))
+            .returning({ id: principals.id });
+        return rows.length > 0;
+    }
     // --- vault ---------------------------------------------------------------
     async insertVault(row) {
         this.check();
@@ -625,6 +802,14 @@ class PostgresTx {
             .where(and(eq(vaults.ownerUserId, ownerUserId), eq(vaults.scope, "user")))
             .orderBy(vaults.createdAt, vaults.id);
     }
+    async installationVaults() {
+        this.check();
+        return this.db
+            .select()
+            .from(vaults)
+            .where(eq(vaults.scope, "installation"))
+            .orderBy(vaults.createdAt, vaults.id);
+    }
     async updateVaultMetadata(id, metadataJson) {
         this.check();
         await this.db.update(vaults).set({ metadataJson }).where(eq(vaults.id, id));
@@ -705,7 +890,24 @@ class PostgresTx {
         this.check();
         await this.db.insert(vaultIdempotency).values(row);
     }
-    // --- subject tokens ------------------------------------------------------
+    async insertOAuthPending(row) {
+        this.check();
+        await this.db.insert(oauthPending).values(row);
+    }
+    async takeOAuthPending(stateHash) {
+        this.check();
+        const [row] = await this.db
+            .delete(oauthPending)
+            .where(eq(oauthPending.stateHash, stateHash))
+            .returning();
+        return row;
+    }
+    async deleteExpiredOAuthPending(now) {
+        this.check();
+        const result = await this.db.delete(oauthPending).where(lt(oauthPending.expiresAt, now));
+        return result.count;
+    }
+    // --- signing keys --------------------------------------------------------
     async lockSigningKeys() {
         this.check();
         await this.db.execute(sql `SELECT pg_advisory_xact_lock(hashtext('nylorun.signing_keys'))`);
@@ -742,103 +944,6 @@ class PostgresTx {
         const [row] = await this.db.select({ n: count() }).from(signingKeys);
         return row.n;
     }
-    async subjectEpoch(subject) {
-        this.check();
-        const [row] = await this.db
-            .select({ epoch: subjectEpochs.epoch })
-            .from(subjectEpochs)
-            .where(eq(subjectEpochs.subject, subject));
-        return row?.epoch ?? 0;
-    }
-    async subjectEpochs(subjects) {
-        this.check();
-        if (subjects.length === 0)
-            return new Map();
-        const rows = await this.db
-            .select({ subject: subjectEpochs.subject, epoch: subjectEpochs.epoch })
-            .from(subjectEpochs)
-            .where(inArray(subjectEpochs.subject, subjects));
-        return new Map(rows.map((row) => [row.subject, row.epoch]));
-    }
-    async bumpSubjectEpoch(subject, at) {
-        this.check();
-        const [row] = await this.db
-            .insert(subjectEpochs)
-            .values({ subject, epoch: 1, revokedAt: at })
-            .onConflictDoUpdate({
-            target: subjectEpochs.subject,
-            set: { epoch: sql `${subjectEpochs.epoch} + 1`, revokedAt: excluded("revoked_at") },
-        })
-            .returning({ epoch: subjectEpochs.epoch });
-        return row.epoch;
-    }
-    async lockSubjectUsage(initial) {
-        this.check();
-        await this.db.insert(subjectUsage).values(initial).onConflictDoNothing();
-        const [row] = await this.db
-            .select()
-            .from(subjectUsage)
-            .where(eq(subjectUsage.subject, initial.subject))
-            .for("update");
-        return row;
-    }
-    async putSubjectUsage(row) {
-        this.check();
-        await this.db
-            .insert(subjectUsage)
-            .values(row)
-            .onConflictDoUpdate({
-            target: subjectUsage.subject,
-            set: { turnTokens: excluded("turn_tokens"), refilledAt: excluded("refilled_at") },
-        });
-    }
-    async countOwnerSessions(ownerUserId, statuses) {
-        this.check();
-        const [row] = await this.db
-            .select({ n: count() })
-            .from(sessions)
-            .where(and(eq(sessions.ownerUserId, ownerUserId), inArray(sessions.status, statuses)));
-        return row.n;
-    }
-    // --- publishable keys ----------------------------------------------------
-    async insertPublishableKey(row) {
-        this.check();
-        await this.db.insert(publishableKeys).values(row);
-    }
-    async publishableKeyByKey(key) {
-        this.check();
-        const [row] = await this.db
-            .select()
-            .from(publishableKeys)
-            .where(eq(publishableKeys.key, key));
-        return row;
-    }
-    async publishableKey(id) {
-        this.check();
-        const [row] = await this.db
-            .select()
-            .from(publishableKeys)
-            .where(eq(publishableKeys.id, id));
-        return row;
-    }
-    async publishableKeys() {
-        this.check();
-        return this.db
-            .select()
-            .from(publishableKeys)
-            .orderBy(publishableKeys.createdAt, publishableKeys.id);
-    }
-    async updatePublishableKey(id, patch) {
-        this.check();
-        if (patch.originsJson === undefined && patch.revokedAt === undefined)
-            return (await this.publishableKey(id)) !== undefined;
-        const rows = await this.db
-            .update(publishableKeys)
-            .set(patch)
-            .where(eq(publishableKeys.id, id))
-            .returning({ id: publishableKeys.id });
-        return rows.length === 1;
-    }
     // --- model usage ---------------------------------------------------------
     async recordModelUsage(row) {
         this.check();
@@ -906,6 +1011,132 @@ class PostgresTx {
             .returning({ key: toolCrossings.key });
         return deleted.length;
     }
+    // --- file artifacts --------------------------------------------------------
+    async artifact(id, options = {}) {
+        this.check();
+        const query = this.db.select().from(artifacts).where(eq(artifacts.id, id));
+        const [row] = options.lock ? await query.for("update") : await query;
+        return row;
+    }
+    async listArtifacts(filter = {}) {
+        this.check();
+        return this.db
+            .select()
+            .from(artifacts)
+            .where(filter.sessionId === undefined ? undefined : eq(artifacts.sessionId, filter.sessionId))
+            .orderBy(artifacts.createdAt, artifacts.id);
+    }
+    async insertArtifact(row, version) {
+        this.check();
+        await this.db.insert(artifacts).values(row);
+        await this.db.insert(artifactVersions).values(version);
+    }
+    async insertArtifactVersion(version) {
+        this.check();
+        await this.db.insert(artifactVersions).values(version);
+        await this.db
+            .update(artifacts)
+            .set({
+            latestVersion: version.version,
+            contentType: version.contentType,
+            updatedAt: version.createdAt,
+        })
+            .where(eq(artifacts.id, version.artifactId));
+    }
+    async artifactVersion(artifactId, version) {
+        this.check();
+        const [row] = await this.db
+            .select()
+            .from(artifactVersions)
+            .where(and(eq(artifactVersions.artifactId, artifactId), eq(artifactVersions.version, version)));
+        return row;
+    }
+    async artifactVersions(artifactId) {
+        this.check();
+        return this.db
+            .select()
+            .from(artifactVersions)
+            .where(eq(artifactVersions.artifactId, artifactId))
+            .orderBy(artifactVersions.version);
+    }
+    async deleteArtifact(id) {
+        this.check();
+        const deleted = await this.db
+            .delete(artifactVersions)
+            .where(eq(artifactVersions.artifactId, id))
+            .returning({ blobKey: artifactVersions.blobKey });
+        await this.db.delete(artifacts).where(eq(artifacts.id, id));
+        return deleted.map((row) => row.blobKey);
+    }
+    async lockArtifactQuota() {
+        this.check();
+        await this.db.execute(sql `SELECT pg_advisory_xact_lock(hashtext('nylorun.artifacts'))`);
+    }
+    async artifactBytes() {
+        this.check();
+        const [files] = await this.db
+            .select({
+            bytes: sql `coalesce(sum(${artifactVersions.size}), 0)::bigint`.mapWith(Number),
+        })
+            .from(artifactVersions)
+            .innerJoin(artifacts, eq(artifacts.id, artifactVersions.artifactId))
+            .where(eq(artifacts.kind, "file"));
+        // A folder's files are stored once per distinct hash, however many versions name them.
+        const distinct = this.db
+            .select({ size: sql `max(${artifactContent.size})`.as("size") })
+            .from(artifactContent)
+            .groupBy(artifactContent.sha256)
+            .as("distinct_content");
+        const [content] = await this.db
+            .select({ bytes: sql `coalesce(sum(${distinct.size}), 0)::bigint`.mapWith(Number) })
+            .from(distinct);
+        return files.bytes + content.bytes;
+    }
+    async insertArtifactContent(artifactId, version, files) {
+        this.check();
+        for (let at = 0; at < files.length; at += CONTENT_BATCH)
+            await this.db
+                .insert(artifactContent)
+                .values(files
+                .slice(at, at + CONTENT_BATCH)
+                .map((file) => ({ artifactId, version, sha256: file.sha256, size: file.size })))
+                .onConflictDoNothing();
+    }
+    async referencedArtifactContent(shas) {
+        this.check();
+        const found = new Set();
+        for (let at = 0; at < shas.length; at += CONTENT_BATCH) {
+            const rows = await this.db
+                .selectDistinct({ sha256: artifactContent.sha256 })
+                .from(artifactContent)
+                .where(inArray(artifactContent.sha256, shas.slice(at, at + CONTENT_BATCH)));
+            for (const row of rows)
+                found.add(row.sha256);
+        }
+        return found;
+    }
+    async artifactContentShas(scope) {
+        this.check();
+        const rows = await this.db
+            .selectDistinct({ sha256: artifactContent.sha256 })
+            .from(artifactContent)
+            .innerJoin(artifacts, eq(artifacts.id, artifactContent.artifactId))
+            .where(scope === "all"
+            ? undefined
+            : scope === "sessions"
+                ? isNotNull(artifacts.sessionId)
+                : eq(artifacts.id, scope.artifactId));
+        return rows.map((row) => row.sha256);
+    }
+    async artifactBlobKeys(scope) {
+        this.check();
+        const rows = await this.db
+            .select({ blobKey: artifactVersions.blobKey })
+            .from(artifactVersions)
+            .innerJoin(artifacts, eq(artifacts.id, artifactVersions.artifactId))
+            .where(scope === "sessions" ? isNotNull(artifacts.sessionId) : undefined);
+        return rows.map((row) => row.blobKey);
+    }
     // --- settings ------------------------------------------------------------
     async getSetting(key) {
         this.check();
@@ -929,8 +1160,9 @@ class PostgresTx {
         if (scope === "sessions" || scope === "all") {
             for (const table of SESSION_TABLES)
                 await db.delete(table);
-            await db.delete(subjectUsage);
             await db.delete(toolCrossings);
+            // A session's artifacts go with it (their versions by cascade).
+            await db.delete(artifacts).where(isNotNull(artifacts.sessionId));
             // The record goes with the sessions, and the Tenant moves to a new basin: the ids it
             // frees start again in an empty one (Durable Streams §8.1).
             await db.delete(sessionEvents);
@@ -940,13 +1172,17 @@ class PostgresTx {
                 basinGeneration: sql `${tenant.basinGeneration} + 1`,
             });
         }
-        if (scope === "sandboxes" || scope === "all")
+        if (scope === "sandboxes" || scope === "all") {
             await db.delete(sandboxes);
+            await db.delete(sandboxResources);
+            await db.delete(sandboxEvents);
+        }
         if (scope === "all") {
             await db.delete(definitions);
             await db.delete(endpoints);
             await db.delete(modelUsage);
             await db.delete(modelBudgets);
+            await db.delete(artifacts);
             await db.delete(vaults).where(ne(vaults.scope, "host"));
         }
     }

package/dist/store/postgres/tenant.d.ts

@@ -16,7 +16,7 @@ export interface TenantCreation {
     /** Ignored when the Tenant exists. */
     name: string;
     /**
-     * The principals the Tenant must have, given its id (derived keys depend on it). Created
+     * The principals the Tenant must have, given its id (Studio's derived key depends on it). Created
      * with the Tenant; on later opens, the missing ones are added.
      */
     principals?(tenantId: string): readonly InitialPrincipal[];

package/dist/store/postgres/tenant.js

@@ -14,8 +14,8 @@
  * 3. When `nylorun.tenant` is empty, create the Tenant: its row (id, name, created time) and
  *    its first principals. A database whose row is gone but that holds a Tenant's data
  *    (principals, sessions, agents, vaults, keys) is `envelope-invalid`, never given a new
- *    Tenant. Principals given later that the Tenant does not have yet are added (a derived
- *    principal configured after the first start); existing ones are kept.
+ *    Tenant. Principals given later that the Tenant does not have yet are added (the
+ *    ephemeral Runtime's application key); existing ones are kept, as ordinary keys.
  * 4. Read the envelope back (`envelope-invalid` when it does not parse), and open the store.
  *
  * A lost connection or an unavailable server says nothing about the Tenant: it is thrown as