@nylorun/runtime 0.14.0-beta → 0.15.1-beta

package/dist/store/postgres/migrate.js

@@ -0,0 +1,183 @@
+/**
+ * The migrations of a Tenant database (session-store.md §3) and the runner that applies them.
+ *
+ * drizzle-kit generates the migrations from `schema.ts` into `drizzle/` (SQL files and
+ * `meta/_journal.json`); custom SQL migrations carry what it does not model (the schemas,
+ * `nylorun.doc()`, the relay's publication). The build copies `drizzle/` next to this module
+ * in `dist/`, and the Runtime reads it from there.
+ *
+ * The runner applies them at Host startup, before any service opens the store
+ * (`tenant.ts`), in one transaction:
+ *
+ * 1. It lifts `statement_timeout` for the transaction and takes the database's migration
+ *    lock (a transaction-level advisory lock), so two processes never migrate at once.
+ * 2. It refuses an old layout (`database-layout-old`): `tenant_<id>` schemas, a record keyed
+ *    by Tenant, or the hand-written migrations' `schema_version` tables of the pre-release
+ *    builds of one Tenant per database. It never changes such a database.
+ * 3. It reads the journal, `nylorun.__drizzle_migrations` (Drizzle's format: `id`, `hash`,
+ *    `created_at`, so `drizzle-kit migrate` and Drizzle Studio read it too). A journal holding
+ *    a migration this Runtime does not ship was written by a newer Runtime: `schema-too-new`,
+ *    and nothing is changed. Drizzle's own migrator does not refuse; this is why the Runtime
+ *    applies the files itself.
+ * 4. It applies every missing migration in order and records each in the journal.
+ *
+ * The database's schema version is the number of migrations its journal records; this
+ * Runtime expects `shippedMigrations().length`.
+ */
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { sql } from "drizzle-orm";
+import { readMigrationFiles } from "drizzle-orm/migrator";
+import { openError } from "../../tenant/cause.js";
+import { database, driverError } from "./db.js";
+import { STREAMS_SCHEMA, TENANT_SCHEMA } from "./schema.js";
+/** Where the migrations are, next to this module (`src/` in development, `dist/` built). */
+export const MIGRATIONS_FOLDER = fileURLToPath(new URL("./drizzle", import.meta.url));
+/** The journal of applied migrations: `nylorun.__drizzle_migrations`. */
+export const MIGRATIONS_SCHEMA = TENANT_SCHEMA;
+export const MIGRATIONS_TABLE = "__drizzle_migrations";
+const JOURNAL = sql.raw(`"${MIGRATIONS_SCHEMA}"."${MIGRATIONS_TABLE}"`);
+/**
+ * What schemas were called when one database held several Tenants (`"tenant_<id>"`). A
+ * database that still has one was written by an older Runtime and is refused.
+ */
+export const OLD_TENANT_SCHEMA_PREFIX = "tenant_";
+let shipped;
+/** The migrations this Runtime ships, in order. */
+export function shippedMigrations() {
+    if (!shipped) {
+        const journal = JSON.parse(readFileSync(`${MIGRATIONS_FOLDER}/meta/_journal.json`, "utf8"));
+        shipped = readMigrationFiles({ migrationsFolder: MIGRATIONS_FOLDER }).map((file, index) => ({
+            tag: journal.entries[index].tag,
+            hash: file.hash,
+            when: file.folderMillis,
+            statements: file.sql,
+        }));
+    }
+    return shipped;
+}
+/** The schema version this Runtime writes and expects: the number of shipped migrations. */
+export function expectedSchemaVersion() {
+    return shippedMigrations().length;
+}
+/**
+ * Lifts `statement_timeout` for the transaction and takes the database's migration lock
+ * until it ends. Waiting for the lock, and the migrations done under it, may outlast the
+ * pool's statement timeout; a migration that timed out would fail the Host's readiness.
+ */
+export async function lockMigrations(tx) {
+    await tx.execute(sql `SET LOCAL statement_timeout = 0`);
+    await tx.execute(sql `SELECT pg_advisory_xact_lock(hashtextextended('nylorun.migrations', 0))`);
+}
+/**
+ * Throws `database-layout-old` when the database was written by an older Runtime: one that
+ * kept several Tenants in one database (`tenant_<id>` schemas, a record keyed by Tenant), or a
+ * pre-release build of one Tenant per database whose hand-written migrations recorded
+ * themselves in `schema_version` tables.
+ */
+export async function assertCurrentLayout(q) {
+    const [row] = await q.execute(sql `
+    SELECT
+      EXISTS (
+        SELECT 1 FROM pg_namespace WHERE starts_with(nspname, ${OLD_TENANT_SCHEMA_PREFIX})
+      ) AS tenant_schemas,
+      EXISTS (
+        SELECT 1 FROM information_schema.columns
+        WHERE table_schema = ${STREAMS_SCHEMA} AND table_name = 'session_events'
+          AND column_name = 'tenant_id'
+      ) AS keyed_record,
+      to_regclass(${`${TENANT_SCHEMA}.schema_version`}) IS NOT NULL
+        OR to_regclass(${`${STREAMS_SCHEMA}.schema_version`}) IS NOT NULL AS versioned`);
+    if (row?.tenant_schemas || row?.keyed_record)
+        throw openError("database-layout-old", "The database was written by an older Runtime that kept several Tenants in one database " +
+            "(tenant_<id> schemas). This release starts fresh with one Tenant per database and " +
+            "never changes the old one: point the Runtime at a new database (locally, a new Tenant: " +
+            "`nylorun start --tenant <new name>`).");
+    if (row?.versioned)
+        throw openError("database-layout-old", "The database was created by a pre-release build of one Tenant per database, whose " +
+            "migrations this release replaces. It is never changed: point the Runtime at a new " +
+            "database (locally, `nylorun reset` and then `nylorun start`).");
+}
+/**
+ * The database's schema version: the number of migrations its journal records, or undefined
+ * when it has no journal (a database never migrated).
+ */
+export async function readSchemaVersion(q) {
+    const [exists] = await q.execute(sql `
+    SELECT to_regclass(${`${MIGRATIONS_SCHEMA}.${MIGRATIONS_TABLE}`}) IS NOT NULL AS journal`);
+    if (!exists?.journal)
+        return undefined;
+    const [row] = await q.execute(sql `SELECT count(*)::int AS n FROM ${JOURNAL}`);
+    return row?.n ?? 0;
+}
+/**
+ * Applies every migration of `migrations` the journal does not record, in `tx`. Take
+ * `lockMigrations` first (`migrateDatabase` does). Throws `schema-too-new` when the journal
+ * records a migration `migrations` does not have, and `migration-failed` when Postgres
+ * rejects a migration.
+ */
+export async function applyMigrations(tx, migrations = shippedMigrations()) {
+    await tx.execute(sql `CREATE SCHEMA IF NOT EXISTS ${sql.raw(`"${MIGRATIONS_SCHEMA}"`)}`);
+    // Drizzle's journal table, as its migrator creates it.
+    await tx.execute(sql `
+    CREATE TABLE IF NOT EXISTS ${JOURNAL} (
+      id SERIAL PRIMARY KEY,
+      hash text NOT NULL,
+      created_at bigint
+    )`);
+    const applied = await tx.execute(sql `SELECT hash FROM ${JOURNAL} ORDER BY id`);
+    const known = new Set(migrations.map((migration) => migration.hash));
+    const unknown = applied.filter((row) => !known.has(row.hash)).length;
+    if (unknown > 0)
+        throw openError("schema-too-new", `The database has ${unknown} migration${unknown === 1 ? "" : "s"} this Runtime does not ` +
+            `know (it is at schema version ${applied.length}; this Runtime's is ${migrations.length}): ` +
+            "it was migrated by a newer Runtime");
+    applied.forEach((row, index) => {
+        if (row.hash !== migrations[index].hash)
+            throw openError("migration-failed", `The database's migration ${index + 1} is not this Runtime's ${migrations[index].tag}`);
+    });
+    for (const migration of migrations.slice(applied.length)) {
+        try {
+            for (const statement of migration.statements)
+                if (statement.trim())
+                    await tx.execute(sql.raw(statement));
+        }
+        catch (thrown) {
+            const error = driverError(thrown);
+            if (!isStatementError(error))
+                throw error;
+            throw openError("migration-failed", `Migration ${migration.tag} failed: ${error.message}`);
+        }
+        await tx.execute(sql `
+      INSERT INTO ${JOURNAL} (hash, created_at) VALUES (${migration.hash}, ${migration.when})`);
+    }
+    return { from: applied.length, to: migrations.length };
+}
+/**
+ * Refuses an old layout and applies the missing migrations, in one transaction under the
+ * migration lock. For tests and tools; the Host migrates through `openTenantDatabase`.
+ */
+export async function migrateDatabase(pool, migrations = shippedMigrations()) {
+    const db = database(pool);
+    try {
+        return await db.transaction(async (tx) => {
+            await lockMigrations(tx);
+            await assertCurrentLayout(tx);
+            return applyMigrations(tx, migrations);
+        });
+    }
+    catch (error) {
+        throw driverError(error);
+    }
+}
+/**
+ * Whether Postgres rejected a statement (a SQLSTATE), rather than the connection or the
+ * server failing (classes 08, 53, 57, and the driver's own connection errors). Only the
+ * first says something about the Tenant's database.
+ */
+export function isStatementError(error) {
+    const code = error?.code;
+    return (typeof code === "string" &&
+        /^[0-9A-Z]{5}$/.test(code) &&
+        !/^(08|53|57)/.test(code));
+}

package/dist/store/postgres/record-writer.d.ts

@@ -0,0 +1,4 @@
+import type { RecordWriter } from "../../record/index.js";
+import type { Transaction } from "./db.js";
+/** The record writer of transaction `db`. */
+export declare function postgresRecordWriter(db: Transaction): RecordWriter;

package/dist/store/postgres/record-writer.js

@@ -0,0 +1,34 @@
+/**
+ * The record's Postgres statements (Durable Streams §6), which the record module runs through
+ * `RecordWriter` (`record/append.ts`, blueprint D27): the only inserts into `session_events`
+ * and `session_log_heads`. They sit here, behind the driver boundary (session-store.md §3), and
+ * run in the store's transaction under the caller's session row lock.
+ */
+import { sql } from "drizzle-orm";
+import { sessionEvents, sessionLogHeads, tenant } from "./schema.js";
+/** The record writer of transaction `db`. */
+export function postgresRecordWriter(db) {
+    return {
+        async advance(sessionId) {
+            const [row] = await db
+                .insert(sessionLogHeads)
+                .values({
+                sessionId,
+                generation: sql `coalesce((SELECT ${tenant.basinGeneration} FROM ${tenant}), 0)`,
+                head: 1,
+            })
+                .onConflictDoUpdate({
+                target: sessionLogHeads.sessionId,
+                set: { head: sql `${sessionLogHeads.head} + 1` },
+            })
+                .returning({
+                seq: sql `${sessionLogHeads.head} - 1`.mapWith(Number),
+                generation: sessionLogHeads.generation,
+            });
+            return row;
+        },
+        async insert(row) {
+            await db.insert(sessionEvents).values(row);
+        },
+    };
+}

package/dist/store/postgres/record.d.ts

@@ -1,11 +1,5 @@
-/**
- * Reads the shared record back for the stream relay (Durable Streams §7.2–7.3): a session's
- * rows to refill a gap in S2, every log head to reconcile after a fresh slot, and a Tenant's
- * current basin generation to tell obsolete rows apart. Reads every Tenant's rows by design.
- */
 import type { Sql } from "postgres";
 import type { RecordReader } from "../../streams/relay/types.js";
-/** Every Tenant's record (the stream relay), or one Tenant's (`tenantId`, its Session Store). */
-export declare function createPostgresRecordReader(sql: Sql, options?: {
-    tenantId?: string;
+export declare function createPostgresRecordReader(sql: Sql, options: {
+    tenantId: string;
 }): RecordReader;

package/dist/store/postgres/record.js

@@ -1,64 +1,60 @@
-import { STREAMS_SCHEMA } from "./migrations/shared/index.js";
-import { quoteIdentifier, tenantSchemaName } from "./names.js";
-/** Every Tenant's record (the stream relay), or one Tenant's (`tenantId`, its Session Store). */
-export function createPostgresRecordReader(sql, options = {}) {
-    const only = options.tenantId;
-    const events = sql(`${STREAMS_SCHEMA}.session_events`);
-    const heads = sql(`${STREAMS_SCHEMA}.session_log_heads`);
+/**
+ * Reads the record back for the stream relay (Durable Streams §7.2–7.3): a session's rows to
+ * refill a gap in S2, every log head to reconcile after a fresh slot, and the Tenant's current
+ * basin generation to tell obsolete rows apart.
+ *
+ * The database holds one Tenant, so the record has no Tenant column: the reader is made for
+ * that Tenant (`tenantId`), fills `RecordRow.tenantId` and `LogHead.tenantId` with it, and
+ * reads nothing for any other id.
+ */
+import { and, asc, eq, gt, gte, lt } from "drizzle-orm";
+import { database, driverError } from "./db.js";
+import { sessionEvents, sessionLogHeads, tenant } from "./schema.js";
+export function createPostgresRecordReader(sql, options) {
+    const own = options.tenantId;
+    const db = database(sql);
+    const read = (query) => Promise.resolve(query).catch((error) => {
+        throw driverError(error);
+    });
     return {
         async readRange(tenantId, sessionId, from, to) {
-            const rows = await sql `
-        SELECT seq, generation, body FROM ${events}
-        WHERE tenant_id = ${tenantId} AND session_id = ${sessionId}
-          AND seq >= ${from} AND seq < ${to}
-        ORDER BY seq`;
-            return rows.map((row) => ({
-                tenantId,
-                sessionId,
-                seq: Number(row.seq),
-                generation: row.generation,
-                body: row.body,
-            }));
+            if (tenantId !== own)
+                return [];
+            const rows = await read(db
+                .select({
+                seq: sessionEvents.seq,
+                generation: sessionEvents.generation,
+                body: sessionEvents.body,
+            })
+                .from(sessionEvents)
+                .where(and(eq(sessionEvents.sessionId, sessionId), gte(sessionEvents.seq, from), lt(sessionEvents.seq, to)))
+                .orderBy(asc(sessionEvents.seq)));
+            return rows.map((row) => ({ tenantId, sessionId, ...row }));
         },
         async heads(after, limit) {
-            const rows = await sql `
-        SELECT tenant_id, session_id, generation, head FROM ${heads}
-        WHERE head > 0
-          ${only === undefined ? sql `` : sql `AND tenant_id = ${only}`}
-          ${after ? sql `AND (tenant_id, session_id) > (${after.tenantId}, ${after.sessionId})` : sql ``}
-        ORDER BY tenant_id, session_id
-        LIMIT ${limit}`;
-            return rows.map((row) => ({
-                tenantId: row.tenant_id,
-                sessionId: row.session_id,
-                generation: row.generation,
-                head: Number(row.head),
-            }));
+            // Heads are ordered by (Tenant, session): every one of this Tenant's comes after a
+            // Tenant id that sorts before it, and none after one that sorts after it.
+            if (after && after.tenantId > own)
+                return [];
+            const from = after?.tenantId === own ? after.sessionId : undefined;
+            const rows = await read(db
+                .select({
+                sessionId: sessionLogHeads.sessionId,
+                generation: sessionLogHeads.generation,
+                head: sessionLogHeads.head,
+            })
+                .from(sessionLogHeads)
+                .where(and(gt(sessionLogHeads.head, 0), from === undefined ? undefined : gt(sessionLogHeads.sessionId, from)))
+                .orderBy(asc(sessionLogHeads.sessionId))
+                .limit(limit));
+            return rows.map((row) => ({ tenantId: own, ...row }));
         },
         async generation(tenantId) {
-            let schema;
-            try {
-                schema = tenantSchemaName(tenantId);
-            }
-            catch {
+            if (tenantId !== own)
                 return undefined;
-            }
-            const [exists] = await sql `
-        SELECT to_regclass(${`${quoteIdentifier(schema)}.tenant`}) IS NOT NULL AS exists`;
-            if (!exists?.exists)
-                return undefined;
-            try {
-                const [row] = await sql `
-          SELECT basin_generation FROM ${sql(`${schema}.tenant`)}`;
-                // A schema without its Tenant row (store tests) is at generation 0.
-                return row ? Number(row.basin_generation) : 0;
-            }
-            catch (error) {
-                // Deleted between the two reads.
-                if (error.code === "42P01")
-                    return undefined;
-                throw error;
-            }
+            const [row] = await read(db.select({ generation: tenant.basinGeneration }).from(tenant));
+            // A database whose Tenant row is not written yet is at generation 0.
+            return row?.generation ?? 0;
         },
     };
 }