@volter/world-runtime 3.0.144 → 3.0.146

package/dist/src/native-postgres-backing.js

@@ -1,14 +1,17 @@
 // World-managed multi-session PostgreSQL. The declaration, data, process and teardown
 // belong to the infrastructure service; a recipe never operates postgres itself.
 import { spawn } from 'node:child_process';
-import { accessSync, closeSync, constants, existsSync, mkdirSync, openSync, readFileSync, readdirSync, realpathSync, rmSync, statSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import { accessSync, closeSync, constants, existsSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { connect } from 'node:net';
+import { runtimeManifest } from "./package-manifest.js";
 import { delimiter, dirname, join, resolve } from 'node:path';
 import { commandLifetime, COMMAND_GRACE_MS, CommandRetirementError } from "./command-lifetime.js";
 import { survivingOwnedGroups } from "./process-groups.js";
 import { publishProcessReceipt, readProcessReceipt, readSpawnReceipt, spawnReceiptPath, verifySpawnIdentity, signalOwnedProcess } from "./process-receipt.js";
 import { signalChild, childIdentity } from './process-signal.mjs';
 import { errorMessage } from "./error-message.js";
+import { PREPARED_DIRECTORY, databasePrepareOf, expectedMigration, forgetStaleMigration, keepFailureEvidence, runDatabasePrepare, takePreparedCluster } from "./pglite-backing.js";
 const starts = new Map();
 const wait = (ms) => new Promise(done => setTimeout(done, ms));
 const recordPath = (root) => join(root, 'native-postgres.owner.json');
@@ -126,6 +129,20 @@ async function listening(port, deadline, signal) {
             abort();
     });
 }
+/** The last lines of a child's stream for an error, labelled, or nothing when it said nothing. */
+function streamTail(label, text, lines = 40) {
+    const tail = text.trim().split('\n').slice(-lines).join('\n');
+    return tail ? `\n${label}:\n${tail}` : '';
+}
+/** The postmaster's log as an error carries it: its last lines. A failed preparation also keeps the whole file (nativePrepare). */
+function logTail(log) {
+    try {
+        return streamTail(`log ${log}`, readFileSync(log, 'utf8'));
+    }
+    catch {
+        return '';
+    }
+}
 /** Every helper joins exit and its private group, including spawn errors and abort. */
 async function tool(command, args, env, deadline, root, signal) {
     signal?.throwIfAborted();
@@ -170,8 +187,10 @@ async function tool(command, args, env, deadline, root, signal) {
             throw new Error(`native postgres readiness deadline elapsed during ${command}`);
         if (spawnError)
             throw spawnError;
+        // Both streams, labelled: a tool's last words on stdout say how far it got (initdb's bootstrap steps) and are
+        // otherwise lost with the process.
         if (result.code !== 0 || lifetime.forwarded)
-            throw new Error(`${command} exited ${result.code}/${result.signal}: ${errorOutput.trim()}`);
+            throw new Error(`${command} exited ${result.code}/${result.signal}${streamTail('stderr', errorOutput)}${streamTail('stdout', output)}`);
         return output;
     }
     catch (error) {
@@ -246,6 +265,141 @@ export function nativePostgresTools(env) {
     }
     return undefined;
 }
+/** initdb's arguments after `-D`, the same at up and at prepare, and so in the stamp. */
+function initdbArguments(user) {
+    return [`--username=${user}`, '--auth=trust', '--no-locale', '--encoding=UTF8'];
+}
+/** Where a World's prepared native cluster lives, beside its world.json, and its stamp (ADR 0018's layout). */
+export function nativePreparedCluster(worldConfig) {
+    const directory = join(dirname(worldConfig), PREPARED_DIRECTORY, 'native-postgres-data');
+    return { directory, stamp: `${directory}.json` };
+}
+function nativeStamp(tools, user, database, digests, migration) {
+    const manifest = runtimeManifest();
+    const programs = {};
+    for (const name of ['postgres', 'initdb']) {
+        const file = programFile(join(tools, name));
+        programs[name] = { bytes: statSync(file).size,
+            ...(digests ? { sha256: createHash('sha256').update(readFileSync(file)).digest('hex') } : {}) };
+    }
+    return { runtime: manifest.version ?? 'unversioned', backing: 'native', programs, initdb: initdbArguments(user), user, database,
+        ...(migration ? { migration } : {}) };
+}
+/**
+ * The file a command runs: itself, or, for a host shim that runs another build of the program (an image's wasm32
+ * PostgreSQL at its image build: browser-substrate wali-host-program.ts), the program it declares on a
+ * `# program: <file>` line in its first 1 KiB. The stamp is about the program that makes the cluster, not its launcher.
+ */
+function programFile(command) {
+    const file = realpathSync(command);
+    // Its first 1 KiB only: read whole, postgres and initdb crossed the tab's file system at every up for this line.
+    const bytes = Buffer.alloc(1024);
+    const fd = openSync(file, 'r');
+    let length;
+    try {
+        length = readSync(fd, bytes, 0, bytes.length, 0);
+    }
+    finally {
+        closeSync(fd);
+    }
+    const head = bytes.subarray(0, length).toString('latin1');
+    return head.startsWith('#!') ? /^# program: (.+)$/m.exec(head)?.[1]?.trim() || file : file;
+}
+/** Whether a stamp names this runtime, these programs' sizes, these initdb arguments, this identity and this migration
+ *  (up's check): a migrated cluster is taken only for the step and input digests it was migrated from, an unmigrated one
+ *  only by a World that declares no step. */
+function nativeStampMatches(text, expected) {
+    let stamp;
+    try {
+        stamp = JSON.parse(text);
+    }
+    catch {
+        return false;
+    }
+    return stamp.runtime === expected.runtime && stamp.backing === 'native' && stamp.user === expected.user
+        && stamp.database === expected.database && JSON.stringify(stamp.initdb) === JSON.stringify(expected.initdb)
+        && Object.keys(expected.programs).every(name => stamp.programs?.[name]?.bytes === expected.programs[name].bytes)
+        && JSON.stringify(stamp.migration) === JSON.stringify(expected.migration);
+}
+/**
+ * `volter-world-infra prepare` for a World whose PostgreSQL is native (architecture, managed infrastructure; ADR
+ * 0018's phase): this lifecycle's own steps once, into `<World dir>/.volter-prepared/native-postgres-data`. initdb with
+ * up's arguments, the postmaster on the World's declared loopback port, the declared database, and a clean SIGINT stop (its
+ * shutdown checkpoint; no postmaster.pid left), then the stamp. Where the World declares runtime.database.prepare, the
+ * step runs against that postmaster before its stop, and a step that fails leaves no cluster. The image build runs it with the programs the tab
+ * runs, under the tab's kernel, so the cluster is the one a first boot there would write.
+ */
+export async function nativePrepare(service, definition, worldConfig, launch = {}) {
+    if (service.kind !== 'postgres')
+        throw new Error('only a declared PostgreSQL is prepared');
+    const env = { ...(launch.env ?? process.env) };
+    // The preparation makes the cluster; it never takes one.
+    delete env.VOLTER_WORLD_CONFIG;
+    const tools = nativePostgresTools(env);
+    if (!tools)
+        throw new Error('native prepare requires the PostgreSQL programs: a postgres server on PATH, or VOLTER_WORLD_POSTGRES_BIN');
+    const { directory, stamp } = nativePreparedCluster(worldConfig);
+    const root = dirname(directory);
+    mkdirSync(root, { recursive: true });
+    if (ownership(root).present)
+        await nativePostgresDown(root);
+    rmSync(directory, { recursive: true, force: true });
+    rmSync(stamp, { force: true });
+    const { user, database } = settings(definition);
+    // On the World's declared port, the one up serves: the image build's sandbox admits exactly that loopback endpoint
+    // (browser-substrate world-cluster.ts), and a port already held is refused by up's own check.
+    // A preparation that fails keeps the postmaster's WHOLE log outside this directory, which its caller may remove,
+    // and says where (pglite-backing.ts keepFailureEvidence). The log is removed only once the cluster is prepared.
+    const log = join(root, 'native-postgres.log');
+    const failed = (error) => {
+        const kept = existsSync(log) ? keepFailureEvidence('native-postgres-log', { file: log }, env) : '';
+        if (error instanceof Error) {
+            error.message += kept;
+            return error;
+        }
+        return new Error(`${String(error)}${kept}`);
+    };
+    try {
+        await nativePostgresUp([service], root, definition, { ...launch, env });
+    }
+    catch (error) {
+        throw failed(error);
+    }
+    // The app's own migration, where the World declares one, against the postmaster this lifecycle has up: the same step
+    // the PGlite backing runs against its host. Its failure waits for the stop below, then removes the cluster.
+    const step = databasePrepareOf(worldConfig);
+    let migration;
+    let stepFailure;
+    if (step) {
+        try {
+            migration = await runDatabasePrepare(step, worldConfig, { ...launch, env });
+        }
+        catch (error) {
+            stepFailure = { error };
+        }
+    }
+    // The clean stop: SIGINT is the postmaster's fast shutdown, which writes its checkpoint and removes postmaster.pid.
+    // Its wait is the lifecycle's own readiness budget, not a new limit; down then clears the receipts.
+    const identity = readSpawnReceipt(recordPath(root)).value;
+    if (identity) {
+        await signalOwnedProcess(identity, 'SIGINT', true);
+        const deadline = Date.now() + READY_TIMEOUT_MS;
+        while (liveGroup(identity.pid) && Date.now() < deadline)
+            await wait(100);
+    }
+    await nativePostgresDown(root);
+    if (stepFailure) {
+        rmSync(directory, { recursive: true, force: true });
+        throw failed(stepFailure.error);
+    }
+    if (!existsSync(join(directory, 'PG_VERSION')) || existsSync(join(directory, 'postmaster.pid'))) {
+        rmSync(directory, { recursive: true, force: true });
+        throw failed(new Error('native prepare ended without a cleanly stopped cluster'));
+    }
+    rmSync(log, { force: true });
+    writeFileSync(stamp, `${JSON.stringify(nativeStamp(tools, user, database, true, migration))}\n`);
+    return directory;
+}
 export async function nativePostgresUp(services, root, definition, launch) {
     if (services.length !== 1 || services[0].kind !== 'postgres')
         throw new Error('native postgres requires one declared postgres service');
@@ -275,10 +429,27 @@ export async function nativePostgresUp(services, root, definition, launch) {
     if (await listening(service.hostPort, deadline, launch.signal))
         throw new Error(`native postgres port ${service.hostPort} is already occupied`);
     const data = resolve(root, 'native-postgres-data');
-    mkdirSync(data, { recursive: true, mode: 0o700 });
     const { user, database } = settings(definition);
+    // An empty data directory takes the World's prepared cluster when its stamp matches (ADR 0018's placement; the
+    // cluster was made by these same programs at image build). initdb then finds PG_VERSION, and the database exists.
+    forgetStaleMigration(data);
+    const prepared = env.VOLTER_WORLD_CONFIG ? nativePreparedCluster(env.VOLTER_WORLD_CONFIG) : undefined;
+    // The check reads two files' metadata and the stamp; it runs only when a prepared cluster is there to take.
+    if (prepared && existsSync(join(prepared.directory, 'PG_VERSION')) && !existsSync(join(data, 'PG_VERSION'))) {
+        // A migrated cluster is the app's only for the step inputs it was migrated from (pglite-backing.ts, expectedMigration).
+        const wanted = expectedMigration(env.VOLTER_WORLD_CONFIG, env, prepared.directory);
+        if ('note' in wanted)
+            launch.note?.(wanted.note);
+        else {
+            const expected = nativeStamp(binaries, user, database, false, wanted.migration);
+            const placed = takePreparedCluster(prepared, text => nativeStampMatches(text, expected), data, 'other PostgreSQL programs, initdb arguments, identity or migration', wanted.migration);
+            if (placed.note)
+                launch.note?.(placed.note);
+        }
+    }
+    mkdirSync(data, { recursive: true, mode: 0o700 });
     if (!existsSync(join(data, 'PG_VERSION'))) {
-        await tool(join(binaries, 'initdb'), ['-D', data, `--username=${user}`, '--auth=trust', '--no-locale', '--encoding=UTF8'], env, deadline, root, launch.signal);
+        await tool(join(binaries, 'initdb'), ['-D', data, ...initdbArguments(user)], env, deadline, root, launch.signal);
     }
     launch.signal?.throwIfAborted();
     const binary = join(binaries, 'postgres');
@@ -321,9 +492,9 @@ export async function nativePostgresUp(services, root, definition, launch) {
             if (started.error)
                 throw started.error;
             if (child.exitCode !== null || child.signalCode !== null || started.closed)
-                throw new Error(`native postgres exited before readiness; see ${log}`);
+                throw new Error(`native postgres exited before readiness; see ${log}${logTail(log)}`);
             if (Date.now() >= deadline)
-                throw new Error(`native postgres did not become ready; see ${log}`);
+                throw new Error(`native postgres did not become ready; see ${log}${logTail(log)}`);
             if (existsSync(join(data, 'postmaster.pid'))) {
                 verifyOwner(current);
                 const lines = readFileSync(join(data, 'postmaster.pid'), 'utf8').split('\n');
@@ -340,7 +511,7 @@ export async function nativePostgresUp(services, root, definition, launch) {
         }
         launch.signal?.throwIfAborted();
         if (child.exitCode !== null || child.signalCode !== null)
-            throw new Error(`native postgres exited during readiness; see ${log}`);
+            throw new Error(`native postgres exited during readiness; see ${log}${logTail(log)}`);
         rmSync(startingPath(root));
         child.unref();
     }

package/dist/src/package-manifest.d.ts

@@ -0,0 +1,7 @@
+export interface RuntimeManifest {
+    name?: string;
+    version?: string;
+    dependencies?: Record<string, string>;
+}
+/** The runtime's package.json, from wherever this module runs. */
+export declare function runtimeManifest(): RuntimeManifest;

package/dist/src/package-manifest.js

@@ -0,0 +1,29 @@
+// THIS PACKAGE'S OWN MANIFEST, found by its name walking up from this module. One finder for every module of the
+// runtime that needs it (a prepared cluster's stamp records the runtime's version and its PGlite's).
+//
+// A fixed hop is wrong in one of the two places the runtime runs from: src/ in the repository is one directory below
+// the package root, dist/src/ in the published package is two, so `..` from a published module is dist/, which
+// holds no package.json. That mistake was made twice and fixed once: the PGlite backing walked up, the native
+// backing still hopped, and a native preparation failed after its migration had run (Cal run 28: "ENOENT ...
+// node_modules/@volter/world-runtime/dist/package.json").
+//
+// Nothing runs at import: a module that builds a URL from import.meta.url when it loads fails to load in workerd,
+// which bundles the runtime into the hosted World.
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const NAME = '@volter/world-runtime';
+/** The runtime's package.json, from wherever this module runs. */
+export function runtimeManifest() {
+    const from = fileURLToPath(new URL('.', import.meta.url));
+    for (let dir = from;; dir = dirname(dir)) {
+        const candidate = join(dir, 'package.json');
+        if (existsSync(candidate)) {
+            const manifest = JSON.parse(readFileSync(candidate, 'utf8'));
+            if (manifest.name === NAME)
+                return manifest;
+        }
+        if (dirname(dir) === dir)
+            throw new Error(`no ${NAME} package.json above ${from}`);
+    }
+}

package/dist/src/pglite-backing.d.ts

@@ -1,3 +1,4 @@
+import { type WorldDatabasePrepare } from './schema.js';
 export interface InfraService {
     kind: string;
     hostPort: number;
@@ -42,8 +43,92 @@ export declare function preparedCluster(worldConfig: string): {
     directory: string;
     stamp: string;
 };
+/**
+ * What a prepared cluster must have been made with: this runtime (whose pglite-host.mjs is the open path that made
+ * it), its exact PGlite pin (initdb's encoding and locale are PGlite's defaults: the host passes no initdb
+ * parameters), and the declared user and database. The password is not in the cluster: the role is created without
+ * one (LOGIN SUPERUSER) and the host's gateway checks the declared password on each connection, so a World with
+ * another password takes the same cluster and authenticates against its own. Extensions are loaded into the module
+ * at each open, never created in the cluster by the open path.
+ */
+/** A prepared cluster's migration, as its stamp records it: the step's command and the digest of each of its inputs. */
+export interface PreparedMigration {
+    command: string[];
+    inputs: Record<string, string>;
+}
+/**
+ * The SHA-256 of each of a migration step's inputs, under its cwd in the app's tree: a file's bytes; a directory's
+ * sorted relative paths, each with its type, its mode and its bytes (a link, its target). What the image build records
+ * for an image, and what `up` hashes only for a World run outside one.
+ */
+export declare function migrationInputDigests(cwd: string, inputs: readonly string[]): Record<string, string>;
+/** The World's declared migration step for its prepared database, if any (runtime.database.prepare). */
+export declare function databasePrepareOf(worldConfig: string): WorldDatabasePrepare | undefined;
+/**
+ * The World's declared step (runtime.database.prepare) run once against a prepared cluster its backing is serving on
+ * the World's port: the declared command in the app's tree, with the image's environment and the World's values over
+ * it. The build host's own PATH finds the command's tools (the image's PATH names its own paths). Its failure throws,
+ * with its output; its answer is what the cluster's stamp records.
+ */
+/**
+ * A FAILED PREPARATION KEEPS ITS WHOLE EVIDENCE. An error carries a tail, and the preparation's own directory is
+ * its caller's to remove (an image build prepares in a staging directory it retires when the preparation fails), so
+ * the tail was all that survived: a postmaster's first lines, its shared-memory setup among them, were gone with the
+ * directory (Cal runs 23 and 24). The whole text or file is written under the preparation's temporary directory
+ * (TMPDIR, which a failed build retains; the system's otherwise), outside the World's directory, and the sentence
+ * naming it is added to the error. Never on success, and a copy that fails says so instead of hiding the failure.
+ */
+export declare function keepFailureEvidence(name: string, evidence: string | {
+    file: string;
+}, env: NodeJS.ProcessEnv): string;
+export declare function runDatabasePrepare(step: WorldDatabasePrepare, worldConfig: string, launch: HostLaunch): Promise<PreparedMigration>;
 /** Run the host's open path once into the World's prepared directory; replaces an earlier preparation. */
 export declare function pglitePrepare(service: InfraService, worldConfig: string, launch?: HostLaunch): Promise<string>;
+/**
+ * The migration a prepared cluster must have been made with for `up` to take it, on either backing: none when the
+ * World declares no step; else the step's command and its inputs' digests, as the app's image recorded them
+ * (VOLTER_WORLD_APP_DIGESTS, no read of the tree) or, for a World run outside an image, as its tree has them. With
+ * neither given a migrated cluster is not taken, and the note says so.
+ */
+export declare function expectedMigration(worldConfig: string, env: NodeJS.ProcessEnv, directory: string): {
+    migration?: PreparedMigration;
+} | {
+    note: string;
+};
+/** A data directory with no cluster carries no migration: a record left by data since removed is dropped. */
+export declare function forgetStaleMigration(serviceData: string): void;
+/**
+ * The declared step's command when the LIVE database carries the migration the app's image expects, else undefined
+ * (architecture, managed infrastructure). A fact of the data, not of this boot's placement: kept data is still
+ * migrated on a later boot, and kept data under an image whose step inputs have changed is older than the image.
+ */
+export declare function liveDatabaseMigration(worldConfig: string, env: NodeJS.ProcessEnv, serviceData: string): string[] | undefined;
+/**
+ * The declared step's command when the World declares one and the live database does NOT carry the migration the
+ * image expects: data kept from an earlier image, or a cluster that was initialized here because the prepared one
+ * was not taken. Undefined with no step, no cluster yet, or a migrated one.
+ */
+export declare function liveDatabaseBehind(worldConfig: string, env: NodeJS.ProcessEnv, serviceData: string): string[] | undefined;
+/**
+ * Whether this is a development World whose app may migrate its database when it starts
+ * (runtime.development.migrateDatabaseAtRuntime; schema.ts WorldDevelopment says whose the setting is).
+ */
+export declare function developmentMigratesAtRuntime(worldConfig: string): boolean;
+/** What a development World says each time it is brought up or built into an image, so its use is never silent. */
+export declare const DEVELOPMENT_MIGRATION_NOTICE = "development World: its database migrates at run time; it is not prepared";
+/**
+ * The one placement of a prepared cluster, for every backing that prepares one (PGlite here, native PostgreSQL in
+ * native-postgres-backing.ts): an empty `serviceData` takes `prepared.directory` by rename when the stamp beside it
+ * is exactly `expected`; a mismatch is named, never taken. A migrated cluster's `migration` is recorded beside the data
+ * it becomes (liveDatabaseMigration).
+ */
+export declare function takePreparedCluster(prepared: {
+    directory: string;
+    stamp: string;
+}, expected: string | ((stamp: string) => boolean), serviceData: string, mismatch: string, migration?: PreparedMigration): {
+    placed: boolean;
+    note?: string;
+};
 export declare function pgliteUp(services: InfraService[], dataDir: string, launch?: HostLaunch): Promise<void>;
 export declare function pgliteStatus(services: InfraService[], dataDir: string): Promise<{
     ok: boolean;