@nylorun/runtime 0.15.1-beta → 0.16.0-beta

package/dist/store/postgres/schema.js

@@ -6,8 +6,8 @@
  *
  * - `nylorun` holds the Tenant's state: the one `tenant` row, the document tables, Action
  *   endpoints, principals, vaults and credentials, signing keys, publishable keys, subject
- *   epochs and usage, settings, the model usage ledger, the model budgets and the Tool Gate's
- *   crossings.
+ *   epochs and usage, settings, the model usage ledger, the model budgets, the Tool Gate's
+ *   crossings and file artifacts with their versions.
  * - `nylorun_streams` holds the record (Durable Streams §6): `session_events`,
  *   `session_log_heads` and the relay's `relay_slots`. The relay's publication is custom SQL
  *   (`drizzle/0002_stream_relay.sql`).
@@ -103,10 +103,13 @@ export const sessions = nylorun.table("sessions", {
     ownerExpiresAt: timestamp({ withTimezone: true, mode: "date" }),
     /** The session owner (`ownerUserId`), not the lease column `owner`. */
     ownerUserId: field("->>'ownerUserId'"),
+    /** The sandbox resource the session is attached to (`sandbox: { id }`). */
+    sandboxId: field("->>'sandboxId'"),
 }, (t) => [
     index("sessions_status").on(t.status, t.ownerExpiresAt),
     index("sessions_agent").on(t.agentId),
     index("sessions_owner_user").on(t.ownerUserId),
+    index("sessions_sandbox").on(t.sandboxId),
 ]);
 export const commands = documents("commands");
 export const effects = nylorun.table("effects", {
@@ -335,8 +338,133 @@ export const toolCrossings = nylorun.table("tool_crossings", {
     /** The gate's answer (`McpAnswer`), once the call ended; null while it runs or once lost. */
     answer: jsonText(),
 }, (t) => [index("tool_crossings_settled").on(t.settledAt)]);
+/**
+ * A sandbox resource (blueprint D39, F7.1): its id, kind, the spec resolved against the
+ * Tenant's limits when it was created, and the developer's labels. Sessions attach to it by id
+ * (the session body's `sandboxId`). Not the `sandboxes` document table, which holds the virtual
+ * backend's compute records, keyed by workspace key.
+ */
+export const sandboxResources = nylorun.table("sandbox_resources", {
+    id: textC().primaryKey(),
+    kind: text({ enum: ["virtual", "pod"] }).notNull(),
+    /** The resolved `SandboxManifest`. */
+    spec: jsonText().notNull(),
+    /** `{ key: value }`, filtered through `nylorun.doc(labels)`. */
+    labels: jsonText().notNull(),
+    createdAt: textC().notNull(),
+    updatedAt: textC().notNull(),
+    // Kind `pod` (F7.2, D34): null for a virtual sandbox. `tenant/sandboxes.ts` and the
+    // `Sandbox` object's reconcile (`sandbox/pods/reconcile.ts`) are the only writers.
+    /** The agent-sandbox Sandbox's name, `sbx-<16 base32>-g<volume_gen>` (`podName`). */
+    k8sName: textC(),
+    /** Bumped by every reset: a new Sandbox on a new volume. */
+    volumeGen: integer().notNull().default(0),
+    /** `running`, `suspended` or `deleted`. */
+    desired: text({ enum: ["running", "suspended", "deleted"] }),
+    /** `creating`, `running`, `suspended`, `expired`, `lost`, `failed` or `deleting`. */
+    observed: text({
+        enum: ["creating", "running", "suspended", "expired", "lost", "failed", "deleting"],
+    }),
+    /** The UID of the pod that last joined. */
+    podUid: text(),
+    /**
+     * Bumped by every join (`host.join`), relaunch, loss and reset: host and egress tokens carry
+     * it (`epc`), and one of an older epoch is refused (egress-gate reads it too).
+     */
+    hostEpoch: integer().notNull().default(0),
+    /** sha256 (hex) of the incarnation's join token; the token itself is never stored. */
+    joinTokenHash: text(),
+    /** Bumped by every change the Runtime asks of the pod: the sandboxes service's `opId`. */
+    rev: integer().notNull().default(0),
+    /** When a turn last ended on it, for the idle timer. */
+    lastActiveAt: timestamp({ withTimezone: true }),
+    /** When its TTL passes (`createdAt + lifecycle.ttl`). */
+    expiresAt: timestamp({ withTimezone: true }),
+    /** When the current incarnation was asked to run, for the ready deadline. */
+    startedAt: timestamp({ withTimezone: true }),
+    /** Why it failed or was lost. */
+    reason: text(),
+    /** An earlier incarnation's Sandbox still to delete (after a reset or a loss). */
+    retiring: textC(),
+    /**
+     * The id the engine keeps on the volume (`/harness/volume-id`), recorded at the first join:
+     * a join with another one means the volume was replaced (its files are gone), so `lost`.
+     */
+    volumeId: text(),
+});
+/**
+ * Artifacts (blueprint D35, F8.1 and F8.2): an id, a name and numbered versions. The bytes are in
+ * the Object store under each version's `blobKey` (a folder's: its manifest); a blob counts only
+ * once its version row commits. `sessionId` is the session the artifact belongs to, or null for
+ * a Tenant-wide one an application made. `latestVersion` is the newest version's number.
+ */
+export const artifacts = nylorun.table("artifacts", {
+    id: textC().primaryKey(),
+    kind: text({ enum: ["file", "folder"] }).notNull(),
+    name: text().notNull(),
+    contentType: text().notNull(),
+    sessionId: textC(),
+    latestVersion: integer().notNull(),
+    /** Labels as JSON text, or null. */
+    labelsJson: text(),
+    createdAt: textC().notNull(),
+    updatedAt: text().notNull(),
+}, (t) => [
+    index("artifacts_session").on(t.sessionId, t.createdAt, t.id),
+    check("artifacts_kind_check", sql `kind IN ('file', 'folder')`),
+]);
+/** The versions of an artifact, immutable once written. */
+export const artifactVersions = nylorun.table("artifact_versions", {
+    artifactId: textC().notNull(),
+    version: integer().notNull(),
+    /** Where the bytes are in the Object store (`BlobStore`). */
+    blobKey: text().notNull(),
+    size: bigint({ mode: "number" }).notNull(),
+    sha256: text().notNull(),
+    contentType: text().notNull(),
+    source: text({ enum: ["upload", "engine", "export"] }).notNull(),
+    createdAt: textC().notNull(),
+}, (t) => [
+    primaryKey({ name: "artifact_versions_pkey", columns: [t.artifactId, t.version] }),
+    foreignKey({
+        name: "artifact_versions_artifact_id_fkey",
+        columns: [t.artifactId],
+        foreignColumns: [artifacts.id],
+    }).onDelete("cascade"),
+]);
+/**
+ * The content-addressed files each folder version references (F8.2), one row per distinct
+ * SHA-256: the index that says which `blobs/sha256/<hex>` objects are still in use, so deleting
+ * a folder removes only the files no other version names, and the Tenant total counts each file
+ * once. The manifest itself stays the source of truth for paths.
+ */
+export const artifactContent = nylorun.table("artifact_content", {
+    artifactId: textC().notNull(),
+    version: integer().notNull(),
+    sha256: textC().notNull(),
+    size: bigint({ mode: "number" }).notNull(),
+}, (t) => [
+    primaryKey({ name: "artifact_content_pkey", columns: [t.artifactId, t.version, t.sha256] }),
+    foreignKey({
+        name: "artifact_content_version_fkey",
+        columns: [t.artifactId, t.version],
+        foreignColumns: [artifactVersions.artifactId, artifactVersions.version],
+    }).onDelete("cascade"),
+    index("artifact_content_sha256").on(t.sha256),
+]);
 // ---------------------------------------------------------------------------
 // nylorun_streams: the record
+/**
+ * Each sandbox's lifecycle stream (`nylorun.sandbox-event/1`), numbered from 0 per sandbox id.
+ * Written only by the record module; not published to the stream relay.
+ */
+export const sandboxEvents = nylorunStreams.table("sandbox_events", {
+    sandboxId: textC().notNull(),
+    seq: bigint({ mode: "number" }).notNull(),
+    type: text().notNull(),
+    body: jsonText().notNull(),
+    committedAt: timestamp({ withTimezone: true, mode: "string" }).notNull().defaultNow(),
+}, (t) => [primaryKey({ name: "sandbox_events_pkey", columns: [t.sandboxId, t.seq] })]);
 /** Each session's log head: the next seq and the basin generation its events go to. */
 export const sessionLogHeads = nylorunStreams.table("session_log_heads", {
     sessionId: textC().primaryKey(),

package/dist/store/postgres/store.js

@@ -32,6 +32,9 @@
  *    head).
  * 2. A transaction that touches several sessions locks them in ascending id
  *    order, with `lockSessions` (`./locking.ts`), before writing any of them.
+ * 3. A sandbox resource's row (`sandboxResource(id, { lock: true })`, which orders its
+ *    lifecycle stream) is locked after every session row the transaction locks, never
+ *    before one.
  *
  * Serialization failures and deadlocks are not retried here: with the order
  * above, READ COMMITTED transactions do not raise them. A `40P01` means a
@@ -47,17 +50,19 @@
  */
 import { AsyncLocalStorage } from "node:async_hooks";
 import { and, count, eq, getTableColumns, gte, inArray, isNotNull, isNull, lt, lte, ne, or, sql, } from "drizzle-orm";
-import { appendEvent } from "../../record/index.js";
+import { appendEvent, appendSandboxEvent } from "../../record/index.js";
 import { OwnershipLostError } from "../ownership.js";
 import { database, driverError } from "./db.js";
 import { expectedSchemaVersion, readSchemaVersion } from "./migrate.js";
 import { createPostgresRecordReader } from "./record.js";
-import { postgresRecordWriter } from "./record-writer.js";
-import { actions, commands, definitions, effects, endpoints, links, modelBudgets, modelUsage, principals, publishableKeys, sandboxes, sessionEvents, sessionLogHeads, sessions, toolCrossings, signingKeys, subjectEpochs, subjectUsage, tenant, tenantSettings, vaultAudit, vaultCredentials, vaultIdempotency, vaults, } from "./schema.js";
+import { postgresRecordWriter, postgresSandboxRecordWriter } from "./record-writer.js";
+import { actions, artifacts, artifactVersions, artifactContent, commands, definitions, effects, endpoints, links, modelBudgets, modelUsage, principals, publishableKeys, sandboxes, sandboxEvents, sandboxResources, sessionEvents, sessionLogHeads, sessions, toJson, toolCrossings, signingKeys, subjectEpochs, subjectUsage, tenant, tenantSettings, vaultAudit, vaultCredentials, vaultIdempotency, vaults, } from "./schema.js";
 export function createPostgresSessionStore(options) {
     return new PostgresSessionStore(options);
 }
 const OPEN_SESSION = ["running", "runnable"];
+/** Rows per statement when a folder version names many files. */
+const CONTENT_BATCH = 1000;
 const DOCUMENTS = {
     definitions,
     sessions: sessions,
@@ -187,6 +192,57 @@ function ownership(row) {
         ownerExpiresAt: row.ownerExpiresAt?.toISOString() ?? null,
     };
 }
+function sandboxResourceOf(row) {
+    return {
+        id: row.id,
+        kind: row.kind,
+        spec: row.spec,
+        labels: row.labels,
+        createdAt: row.createdAt,
+        updatedAt: row.updatedAt,
+        ...(row.k8sName !== null && row.desired !== null && row.observed !== null
+            ? {
+                pod: {
+                    k8sName: row.k8sName,
+                    volumeGen: row.volumeGen,
+                    desired: row.desired,
+                    observed: row.observed,
+                    ...(row.podUid !== null ? { podUid: row.podUid } : {}),
+                    hostEpoch: row.hostEpoch,
+                    ...(row.joinTokenHash !== null ? { joinTokenHash: row.joinTokenHash } : {}),
+                    rev: row.rev,
+                    ...(row.lastActiveAt ? { lastActiveAt: row.lastActiveAt.toISOString() } : {}),
+                    ...(row.expiresAt ? { expiresAt: row.expiresAt.toISOString() } : {}),
+                    ...(row.startedAt ? { startedAt: row.startedAt.toISOString() } : {}),
+                    ...(row.reason !== null ? { reason: row.reason } : {}),
+                    ...(row.retiring !== null ? { retiring: row.retiring } : {}),
+                    ...(row.volumeId !== null ? { volumeId: row.volumeId } : {}),
+                },
+            }
+            : {}),
+    };
+}
+/** A pod patch as column values: ISO times become dates, `null` clears. */
+function podColumns(patch) {
+    const date = (value) => value === undefined ? undefined : value === null ? null : new Date(value);
+    const columns = {
+        k8sName: patch.k8sName,
+        volumeGen: patch.volumeGen,
+        desired: patch.desired,
+        observed: patch.observed,
+        podUid: patch.podUid,
+        hostEpoch: patch.hostEpoch,
+        joinTokenHash: patch.joinTokenHash,
+        rev: patch.rev,
+        lastActiveAt: date(patch.lastActiveAt),
+        expiresAt: date(patch.expiresAt),
+        startedAt: date(patch.startedAt),
+        reason: patch.reason,
+        retiring: patch.retiring,
+        volumeId: patch.volumeId,
+    };
+    return Object.fromEntries(Object.entries(columns).filter(([, value]) => value !== undefined));
+}
 /** An endpoint row with its unset (null) columns left out, as `EndpointRow` has them. */
 function endpointRow(row) {
     return Object.fromEntries(Object.entries(row).filter(([, value]) => value !== null));
@@ -345,6 +401,18 @@ class PostgresTx {
             throw new OwnershipLostError(sessionId, epoch, session.epoch);
         return session;
     }
+    async runState(sessionId) {
+        this.check();
+        const [row] = await this.db
+            .select({
+            epoch: sessions.epoch,
+            status: sessions.status,
+            activeTurnId: sql `nylorun.doc(${sessions.body})->>'activeTurnId'`,
+        })
+            .from(sessions)
+            .where(eq(sessions.id, sessionId));
+        return row && { epoch: row.epoch, status: row.status ?? "", activeTurnId: row.activeTurnId ?? null };
+    }
     // --- typed queries -------------------------------------------------------
     async sessionsWithStatus(statuses) {
         this.check();
@@ -383,6 +451,91 @@ class PostgresTx {
         this.check();
         return this.bodies(sandboxes);
     }
+    // --- sandbox resources ---------------------------------------------------
+    async sandboxResource(id, options = {}) {
+        this.check();
+        const query = this.db.select().from(sandboxResources).where(eq(sandboxResources.id, id));
+        const [row] = options.lock ? await query.for("update") : await query;
+        return row && sandboxResourceOf(row);
+    }
+    async createSandboxResource(row, limit) {
+        this.check();
+        // Every create takes this lock, so two never both see room for one more.
+        await this.db.execute(sql `SELECT pg_advisory_xact_lock(hashtext('nylorun.sandbox_resources'))`);
+        const [found] = await this.db
+            .select({ id: sandboxResources.id })
+            .from(sandboxResources)
+            .where(eq(sandboxResources.id, row.id));
+        if (found)
+            return "exists";
+        const [counted] = await this.db.select({ n: count() }).from(sandboxResources);
+        if ((counted?.n ?? 0) >= limit)
+            return "limit";
+        const { pod, ...resource } = row;
+        await this.db.insert(sandboxResources).values({ ...resource, ...(pod ? podColumns(pod) : {}) });
+        return "created";
+    }
+    async updateSandboxSpec(id, spec, updatedAt) {
+        this.check();
+        await this.db.update(sandboxResources).set({ spec, updatedAt }).where(eq(sandboxResources.id, id));
+    }
+    async updateSandboxPod(id, patch, updatedAt) {
+        this.check();
+        await this.db
+            .update(sandboxResources)
+            .set({ ...podColumns(patch), updatedAt })
+            .where(eq(sandboxResources.id, id));
+    }
+    async updateSandboxLabels(id, labels, updatedAt) {
+        this.check();
+        await this.db
+            .update(sandboxResources)
+            .set({ labels, updatedAt })
+            .where(eq(sandboxResources.id, id));
+    }
+    async deleteSandboxResource(id) {
+        this.check();
+        await this.db.delete(sandboxResources).where(eq(sandboxResources.id, id));
+    }
+    async listSandboxResources(filter = {}) {
+        this.check();
+        const labels = filter.labels;
+        const rows = await this.db
+            .select()
+            .from(sandboxResources)
+            .where(labels === undefined || Object.keys(labels).length === 0
+            ? undefined
+            : sql `nylorun.doc(${sandboxResources.labels}) @> ${toJson(labels)}::jsonb`)
+            .orderBy(sandboxResources.id);
+        return rows.map(sandboxResourceOf);
+    }
+    async sessionsOnSandbox(sandboxId) {
+        this.check();
+        const rows = await this.sessions()
+            .where(eq(sessions.sandboxId, sandboxId))
+            .orderBy(sessions.id);
+        return rows.map((row) => storedSession(row));
+    }
+    async sandboxEvent(sandboxId, type, payload) {
+        this.check();
+        return appendSandboxEvent(postgresSandboxRecordWriter(this.db), {
+            tenantId: this.tenantId,
+            sandboxId,
+            time: this.now(),
+            type,
+            payload,
+        });
+    }
+    async sandboxEvents(sandboxId, options = {}) {
+        this.check();
+        const rows = await this.db
+            .select({ body: sandboxEvents.body })
+            .from(sandboxEvents)
+            .where(and(eq(sandboxEvents.sandboxId, sandboxId), options.fromSeq === undefined ? undefined : gte(sandboxEvents.seq, options.fromSeq)))
+            .orderBy(sandboxEvents.seq)
+            .limit(options.limit ?? 1000);
+        return rows.map((row) => row.body);
+    }
     async actionBodies(where) {
         const rows = await this.db
             .select({ body: actions.body })
@@ -906,6 +1059,132 @@ class PostgresTx {
             .returning({ key: toolCrossings.key });
         return deleted.length;
     }
+    // --- file artifacts --------------------------------------------------------
+    async artifact(id, options = {}) {
+        this.check();
+        const query = this.db.select().from(artifacts).where(eq(artifacts.id, id));
+        const [row] = options.lock ? await query.for("update") : await query;
+        return row;
+    }
+    async listArtifacts(filter = {}) {
+        this.check();
+        return this.db
+            .select()
+            .from(artifacts)
+            .where(filter.sessionId === undefined ? undefined : eq(artifacts.sessionId, filter.sessionId))
+            .orderBy(artifacts.createdAt, artifacts.id);
+    }
+    async insertArtifact(row, version) {
+        this.check();
+        await this.db.insert(artifacts).values(row);
+        await this.db.insert(artifactVersions).values(version);
+    }
+    async insertArtifactVersion(version) {
+        this.check();
+        await this.db.insert(artifactVersions).values(version);
+        await this.db
+            .update(artifacts)
+            .set({
+            latestVersion: version.version,
+            contentType: version.contentType,
+            updatedAt: version.createdAt,
+        })
+            .where(eq(artifacts.id, version.artifactId));
+    }
+    async artifactVersion(artifactId, version) {
+        this.check();
+        const [row] = await this.db
+            .select()
+            .from(artifactVersions)
+            .where(and(eq(artifactVersions.artifactId, artifactId), eq(artifactVersions.version, version)));
+        return row;
+    }
+    async artifactVersions(artifactId) {
+        this.check();
+        return this.db
+            .select()
+            .from(artifactVersions)
+            .where(eq(artifactVersions.artifactId, artifactId))
+            .orderBy(artifactVersions.version);
+    }
+    async deleteArtifact(id) {
+        this.check();
+        const deleted = await this.db
+            .delete(artifactVersions)
+            .where(eq(artifactVersions.artifactId, id))
+            .returning({ blobKey: artifactVersions.blobKey });
+        await this.db.delete(artifacts).where(eq(artifacts.id, id));
+        return deleted.map((row) => row.blobKey);
+    }
+    async lockArtifactQuota() {
+        this.check();
+        await this.db.execute(sql `SELECT pg_advisory_xact_lock(hashtext('nylorun.artifacts'))`);
+    }
+    async artifactBytes() {
+        this.check();
+        const [files] = await this.db
+            .select({
+            bytes: sql `coalesce(sum(${artifactVersions.size}), 0)::bigint`.mapWith(Number),
+        })
+            .from(artifactVersions)
+            .innerJoin(artifacts, eq(artifacts.id, artifactVersions.artifactId))
+            .where(eq(artifacts.kind, "file"));
+        // A folder's files are stored once per distinct hash, however many versions name them.
+        const distinct = this.db
+            .select({ size: sql `max(${artifactContent.size})`.as("size") })
+            .from(artifactContent)
+            .groupBy(artifactContent.sha256)
+            .as("distinct_content");
+        const [content] = await this.db
+            .select({ bytes: sql `coalesce(sum(${distinct.size}), 0)::bigint`.mapWith(Number) })
+            .from(distinct);
+        return files.bytes + content.bytes;
+    }
+    async insertArtifactContent(artifactId, version, files) {
+        this.check();
+        for (let at = 0; at < files.length; at += CONTENT_BATCH)
+            await this.db
+                .insert(artifactContent)
+                .values(files
+                .slice(at, at + CONTENT_BATCH)
+                .map((file) => ({ artifactId, version, sha256: file.sha256, size: file.size })))
+                .onConflictDoNothing();
+    }
+    async referencedArtifactContent(shas) {
+        this.check();
+        const found = new Set();
+        for (let at = 0; at < shas.length; at += CONTENT_BATCH) {
+            const rows = await this.db
+                .selectDistinct({ sha256: artifactContent.sha256 })
+                .from(artifactContent)
+                .where(inArray(artifactContent.sha256, shas.slice(at, at + CONTENT_BATCH)));
+            for (const row of rows)
+                found.add(row.sha256);
+        }
+        return found;
+    }
+    async artifactContentShas(scope) {
+        this.check();
+        const rows = await this.db
+            .selectDistinct({ sha256: artifactContent.sha256 })
+            .from(artifactContent)
+            .innerJoin(artifacts, eq(artifacts.id, artifactContent.artifactId))
+            .where(scope === "all"
+            ? undefined
+            : scope === "sessions"
+                ? isNotNull(artifacts.sessionId)
+                : eq(artifacts.id, scope.artifactId));
+        return rows.map((row) => row.sha256);
+    }
+    async artifactBlobKeys(scope) {
+        this.check();
+        const rows = await this.db
+            .select({ blobKey: artifactVersions.blobKey })
+            .from(artifactVersions)
+            .innerJoin(artifacts, eq(artifacts.id, artifactVersions.artifactId))
+            .where(scope === "sessions" ? isNotNull(artifacts.sessionId) : undefined);
+        return rows.map((row) => row.blobKey);
+    }
     // --- settings ------------------------------------------------------------
     async getSetting(key) {
         this.check();
@@ -931,6 +1210,8 @@ class PostgresTx {
                 await db.delete(table);
             await db.delete(subjectUsage);
             await db.delete(toolCrossings);
+            // A session's artifacts go with it (their versions by cascade).
+            await db.delete(artifacts).where(isNotNull(artifacts.sessionId));
             // The record goes with the sessions, and the Tenant moves to a new basin: the ids it
             // frees start again in an empty one (Durable Streams §8.1).
             await db.delete(sessionEvents);
@@ -940,13 +1221,17 @@ class PostgresTx {
                 basinGeneration: sql `${tenant.basinGeneration} + 1`,
             });
         }
-        if (scope === "sandboxes" || scope === "all")
+        if (scope === "sandboxes" || scope === "all") {
             await db.delete(sandboxes);
+            await db.delete(sandboxResources);
+            await db.delete(sandboxEvents);
+        }
         if (scope === "all") {
             await db.delete(definitions);
             await db.delete(endpoints);
             await db.delete(modelUsage);
             await db.delete(modelBudgets);
+            await db.delete(artifacts);
             await db.delete(vaults).where(ne(vaults.scope, "host"));
         }
     }