@volter/world-runtime 3.0.143 → 3.0.145

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/init.ts CHANGED
@@ -40,6 +40,7 @@ import {
40
40
  coverWorld,
41
41
  detectRepoVendors,
42
42
  envNameVendor,
43
+ coverageRecord,
43
44
  formatCoverageReport,
44
45
  injectorEnvNameForKey,
45
46
  injectorVendorKeysFor,
@@ -50,7 +51,7 @@ import {
50
51
  import { fakeEnvValue, isAppKeyName, isGoogleOAuthClientEnvName, isGoogleServiceAccountEnvName } from './fixture-env.ts';
51
52
  import { findInstalledPackage, packCli, packEntry, resolveCatalog, type Catalog } from './catalog.ts';
52
53
  import { CATALOG_INDEX_PACKAGE, defaults as catalogDefaults } from './catalog-index.ts';
53
- import { projectEnvReads, projectManifestDirs } from './project-inspect.ts';
54
+ import { projectEnvReads, projectManifestDirs, projectMigrationTools, type ProjectMigrationTool } from './project-inspect.ts';
54
55
  import { overlayEndpointEnv, packFacts } from './pack-facts.ts';
55
56
  import { VENDOR_SERVER_BACKINGS } from './backings.ts';
56
57
  import { APP_URL_ENV, DEFAULT_WORLD_SELECTION, ISSUE_ENV_PREFIX, MINT_ENV, worldConfigDocument, worldSelectionIncludes, type WorldConfig, type WorldSelection, type WorldServiceConfig } from './schema.ts';
@@ -201,6 +202,9 @@ export type InitPlan = {
201
202
  envSources: string[];
202
203
  /** empty example values read by production source — prominent warning, not vendor-coverage fact */
203
204
  bootRisks: InitBootRisk[];
205
+ /** migration tools found in the repository while the World manages a database and declares no step
206
+ * (runtime.database.prepare): reported, never written, and no part of the vendor-coverage verdict */
207
+ undeclaredMigrations: ProjectMigrationTool[];
204
208
  infra: InitInfraStub[];
205
209
  /** the services emitted into the private managed-infrastructure definition */
206
210
  compose: InitComposeService[];
@@ -1233,6 +1237,9 @@ export function planWorldInit(name: string, repoPath: string, options: InitOptio
1233
1237
  envRows,
1234
1238
  envSources,
1235
1239
  bootRisks: [...bootRisks].sort((a, b) => a.name.localeCompare(b.name)),
1240
+ // A migration belongs in the World's step, run once where the image is built. init can see the tool, not the step.
1241
+ undeclaredMigrations: existing?.databasePrepare !== undefined || !(compose.some((service) => service.kind === 'postgres' || service.kind === 'mysql') || existingIds.has('infrastructure'))
1242
+ ? [] : projectMigrationTools(repo),
1236
1243
  infra,
1237
1244
  compose,
1238
1245
  composePath: compose.length > 0 ? resolve(out, 'world.infrastructure.yml') : null,
@@ -1437,6 +1444,10 @@ export function initWorld(name: string, repoPath: string, options: InitOptions =
1437
1444
  // The proof was addressed BY PATH (so it reads the bytes just written, not the in-memory plan);
1438
1445
  // report it under the world's id, which is what every other verb takes.
1439
1446
  const coverage: CoverageReport = { ...proof, world: plan.name };
1447
+ // The proof is kept in the config it was made against, beside the selection: the report printed below, as data.
1448
+ // Written after the proof because the proof reads the config's bytes; it reads nothing of `discovery.coverage`.
1449
+ plan.config.coverage = coverageRecord(coverage);
1450
+ writeFileSync(plan.configPath, `${JSON.stringify(worldConfigDocument(plan.config), null, 2)}\n`);
1440
1451
  const next = plan.inRepo && resolve(options.root ?? plan.repo) === plan.repo
1441
1452
  ? [
1442
1453
  'volter world up # starts the twins and loads the default data; the live env lands in .volter/world.env',
@@ -1519,6 +1530,16 @@ export function formatInitReport(result: InitResult): string {
1519
1530
  }
1520
1531
  lines.push(' Resolve these before expecting the app to listen. Vendor coverage can be green while the app rejects its own env.');
1521
1532
  }
1533
+ if (plan.undeclaredMigrations.length > 0) {
1534
+ lines.push('');
1535
+ lines.push(`MIGRATION TOOL FOUND, NO STEP DECLARED (${plan.undeclaredMigrations.length}):`);
1536
+ for (const found of plan.undeclaredMigrations) lines.push(` ${found.tool} — migrations at ${found.migrations} (schema ${found.schema})`);
1537
+ lines.push(' The World manages this app\'s database and declares no runtime.database.prepare, so the app would migrate it at');
1538
+ lines.push(' every start. Declare the step in the World: `command` (the migration as the app\'s image runs it, an argv), `cwd`');
1539
+ lines.push(' (the image path it runs in) and `inputs` (the schema and migrations, as paths under it). These are the IMAGE\'s');
1540
+ lines.push(' paths, which its Dockerfile decides and this repository does not state; an image build that reads the image');
1541
+ lines.push(' prints the step.');
1542
+ }
1522
1543
  if (plan.compose.length > 0) {
1523
1544
  lines.push('');
1524
1545
  lines.push('Infrastructure (owned by the World lifecycle):');
@@ -1,15 +1,17 @@
1
1
  // World-managed multi-session PostgreSQL. The declaration, data, process and teardown
2
2
  // belong to the infrastructure service; a recipe never operates postgres itself.
3
3
  import { spawn, type ChildProcess } from 'node:child_process';
4
- import { accessSync, closeSync, constants, existsSync, mkdirSync, openSync, readFileSync, readdirSync, realpathSync, rmSync, statSync } from 'node:fs';
4
+ import { createHash } from 'node:crypto';
5
+ import { accessSync, closeSync, constants, existsSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, rmSync, statSync, writeFileSync } from 'node:fs';
5
6
  import { connect } from 'node:net';
7
+ import { runtimeManifest } from './package-manifest.ts';
6
8
  import { delimiter, dirname, join, resolve } from 'node:path';
7
9
  import { commandLifetime, COMMAND_GRACE_MS, CommandRetirementError } from './command-lifetime.ts';
8
10
  import { survivingOwnedGroups } from './process-groups.ts';
9
11
  import { publishProcessReceipt, readProcessReceipt, readSpawnReceipt, spawnReceiptPath, verifySpawnIdentity, signalOwnedProcess, type SpawnIdentity } from './process-receipt.ts';
10
12
  import { signalChild, childIdentity } from './process-signal.mjs';
11
13
  import { errorMessage } from './error-message.ts';
12
- import type { HostLaunch, InfraService } from './pglite-backing.ts';
14
+ import { PREPARED_DIRECTORY, databasePrepareOf, expectedMigration, forgetStaleMigration, keepFailureEvidence, runDatabasePrepare, takePreparedCluster, type HostLaunch, type InfraService, type PreparedMigration } from './pglite-backing.ts';
13
15
 
14
16
  interface Ownership { pid: number; data: string; port: number; binary: string }
15
17
  interface Started { child: ChildProcess; owned: Ownership; closed: boolean; identity?: SpawnIdentity; birth?: Promise<SpawnIdentity | undefined>; error?: Error }
@@ -108,6 +110,15 @@ async function listening(port: number, deadline: number, signal?: AbortSignal):
108
110
  if (signal?.aborted) abort();
109
111
  });
110
112
  }
113
+ /** The last lines of a child's stream for an error, labelled, or nothing when it said nothing. */
114
+ function streamTail(label: string, text: string, lines = 40): string {
115
+ const tail = text.trim().split('\n').slice(-lines).join('\n');
116
+ return tail ? `\n${label}:\n${tail}` : '';
117
+ }
118
+ /** The postmaster's log as an error carries it: its last lines. A failed preparation also keeps the whole file (nativePrepare). */
119
+ function logTail(log: string): string {
120
+ try { return streamTail(`log ${log}`, readFileSync(log, 'utf8')); } catch { return ''; }
121
+ }
111
122
  /** Every helper joins exit and its private group, including spawn errors and abort. */
112
123
  async function tool(command: string, args: string[], env: NodeJS.ProcessEnv, deadline: number, root: string, signal?: AbortSignal): Promise<string> {
113
124
  signal?.throwIfAborted();
@@ -138,7 +149,9 @@ async function tool(command: string, args: string[], env: NodeJS.ProcessEnv, dea
138
149
  signal?.throwIfAborted();
139
150
  if (timedOut) throw new Error(`native postgres readiness deadline elapsed during ${command}`);
140
151
  if (spawnError) throw spawnError;
141
- if (result.code !== 0 || lifetime.forwarded) throw new Error(`${command} exited ${result.code}/${result.signal}: ${errorOutput.trim()}`);
152
+ // Both streams, labelled: a tool's last words on stdout say how far it got (initdb's bootstrap steps) and are
153
+ // otherwise lost with the process.
154
+ if (result.code !== 0 || lifetime.forwarded) throw new Error(`${command} exited ${result.code}/${result.signal}${streamTail('stderr', errorOutput)}${streamTail('stdout', output)}`);
142
155
  return output;
143
156
  } catch (error) {
144
157
  failure = error;
@@ -200,6 +213,144 @@ export function nativePostgresTools(env: NodeJS.ProcessEnv): string | undefined
200
213
  return undefined;
201
214
  }
202
215
 
216
+ /** initdb's arguments after `-D`, the same at up and at prepare, and so in the stamp. */
217
+ function initdbArguments(user: string): string[] {
218
+ return [`--username=${user}`, '--auth=trust', '--no-locale', '--encoding=UTF8'];
219
+ }
220
+
221
+ /** Where a World's prepared native cluster lives, beside its world.json, and its stamp (ADR 0018's layout). */
222
+ export function nativePreparedCluster(worldConfig: string): { directory: string; stamp: string } {
223
+ const directory = join(dirname(worldConfig), PREPARED_DIRECTORY, 'native-postgres-data');
224
+ return { directory, stamp: `${directory}.json` };
225
+ }
226
+
227
+ /**
228
+ * What a prepared native cluster was made with: this runtime (its lifecycle made it), the postgres and initdb that
229
+ * made it, initdb's arguments, the declared user and database, and, for a migrated cluster, the World's step and its
230
+ * inputs' digests (pglite-backing.ts, runDatabasePrepare). initdb's own choices (its dynamic shared memory
231
+ * probe, its configuration trials) are in the cluster's postgresql.conf, which its programs make again. Neither the
232
+ * port nor a password is in a trust-authenticated cluster.
233
+ *
234
+ * Each program is recorded by its size and, at prepare, its sha256: the record of what made the cluster. At up only
235
+ * the sizes are compared, from the files' metadata. Hashing the programs again there read the whole of postgres and
236
+ * of initdb through the tab's file system on every boot (initdb otherwise untouched when the cluster is taken,
237
+ * postgres then read a second time to run), inside the boot's budget. What the sizes do not prove, PostgreSQL proves
238
+ * itself: the server reads pg_control at start and refuses a cluster whose control version, catalog version, block
239
+ * size, alignment or float8 passing differ from its build (a 64-bit or another-version cluster), so a foreign cluster
240
+ * is never served, only refused.
241
+ */
242
+ interface NativeStamp {
243
+ runtime: string; backing: 'native';
244
+ programs: Record<string, { bytes: number; sha256?: string }>;
245
+ initdb: string[]; user: string; database: string;
246
+ migration?: PreparedMigration;
247
+ }
248
+
249
+ function nativeStamp(tools: string, user: string, database: string, digests: boolean, migration?: PreparedMigration): NativeStamp {
250
+ const manifest = runtimeManifest();
251
+ const programs: NativeStamp['programs'] = {};
252
+ for (const name of ['postgres', 'initdb']) {
253
+ const file = programFile(join(tools, name));
254
+ programs[name] = { bytes: statSync(file).size,
255
+ ...(digests ? { sha256: createHash('sha256').update(readFileSync(file)).digest('hex') } : {}) };
256
+ }
257
+ return { runtime: manifest.version ?? 'unversioned', backing: 'native', programs, initdb: initdbArguments(user), user, database,
258
+ ...(migration ? { migration } : {}) };
259
+ }
260
+
261
+ /**
262
+ * The file a command runs: itself, or, for a host shim that runs another build of the program (an image's wasm32
263
+ * PostgreSQL at its image build: browser-substrate wali-host-program.ts), the program it declares on a
264
+ * `# program: <file>` line in its first 1 KiB. The stamp is about the program that makes the cluster, not its launcher.
265
+ */
266
+ function programFile(command: string): string {
267
+ const file = realpathSync(command);
268
+ // Its first 1 KiB only: read whole, postgres and initdb crossed the tab's file system at every up for this line.
269
+ const bytes = Buffer.alloc(1024);
270
+ const fd = openSync(file, 'r');
271
+ let length: number;
272
+ try { length = readSync(fd, bytes, 0, bytes.length, 0); } finally { closeSync(fd); }
273
+ const head = bytes.subarray(0, length).toString('latin1');
274
+ return head.startsWith('#!') ? /^# program: (.+)$/m.exec(head)?.[1]?.trim() || file : file;
275
+ }
276
+
277
+ /** Whether a stamp names this runtime, these programs' sizes, these initdb arguments, this identity and this migration
278
+ * (up's check): a migrated cluster is taken only for the step and input digests it was migrated from, an unmigrated one
279
+ * only by a World that declares no step. */
280
+ function nativeStampMatches(text: string, expected: NativeStamp): boolean {
281
+ let stamp: NativeStamp;
282
+ try { stamp = JSON.parse(text) as NativeStamp; } catch { return false; }
283
+ return stamp.runtime === expected.runtime && stamp.backing === 'native' && stamp.user === expected.user
284
+ && stamp.database === expected.database && JSON.stringify(stamp.initdb) === JSON.stringify(expected.initdb)
285
+ && Object.keys(expected.programs).every(name => stamp.programs?.[name]?.bytes === expected.programs[name]!.bytes)
286
+ && JSON.stringify(stamp.migration) === JSON.stringify(expected.migration);
287
+ }
288
+
289
+ /**
290
+ * `volter-world-infra prepare` for a World whose PostgreSQL is native (architecture, managed infrastructure; ADR
291
+ * 0018's phase): this lifecycle's own steps once, into `<World dir>/.volter-prepared/native-postgres-data`. initdb with
292
+ * up's arguments, the postmaster on the World's declared loopback port, the declared database, and a clean SIGINT stop (its
293
+ * shutdown checkpoint; no postmaster.pid left), then the stamp. Where the World declares runtime.database.prepare, the
294
+ * step runs against that postmaster before its stop, and a step that fails leaves no cluster. The image build runs it with the programs the tab
295
+ * runs, under the tab's kernel, so the cluster is the one a first boot there would write.
296
+ */
297
+ export async function nativePrepare(service: InfraService, definition: string, worldConfig: string, launch: HostLaunch = {}): Promise<string> {
298
+ if (service.kind !== 'postgres') throw new Error('only a declared PostgreSQL is prepared');
299
+ const env: NodeJS.ProcessEnv = { ...(launch.env ?? process.env) };
300
+ // The preparation makes the cluster; it never takes one.
301
+ delete env.VOLTER_WORLD_CONFIG;
302
+ const tools = nativePostgresTools(env);
303
+ if (!tools) throw new Error('native prepare requires the PostgreSQL programs: a postgres server on PATH, or VOLTER_WORLD_POSTGRES_BIN');
304
+ const { directory, stamp } = nativePreparedCluster(worldConfig);
305
+ const root = dirname(directory);
306
+ mkdirSync(root, { recursive: true });
307
+ if (ownership(root).present) await nativePostgresDown(root);
308
+ rmSync(directory, { recursive: true, force: true });
309
+ rmSync(stamp, { force: true });
310
+ const { user, database } = settings(definition);
311
+ // On the World's declared port, the one up serves: the image build's sandbox admits exactly that loopback endpoint
312
+ // (browser-substrate world-cluster.ts), and a port already held is refused by up's own check.
313
+ // A preparation that fails keeps the postmaster's WHOLE log outside this directory, which its caller may remove,
314
+ // and says where (pglite-backing.ts keepFailureEvidence). The log is removed only once the cluster is prepared.
315
+ const log = join(root, 'native-postgres.log');
316
+ const failed = (error: unknown): unknown => {
317
+ const kept = existsSync(log) ? keepFailureEvidence('native-postgres-log', { file: log }, env) : '';
318
+ if (error instanceof Error) { error.message += kept; return error; }
319
+ return new Error(`${String(error)}${kept}`);
320
+ };
321
+ try { await nativePostgresUp([service], root, definition, { ...launch, env }); }
322
+ catch (error) { throw failed(error); }
323
+ // The app's own migration, where the World declares one, against the postmaster this lifecycle has up: the same step
324
+ // the PGlite backing runs against its host. Its failure waits for the stop below, then removes the cluster.
325
+ const step = databasePrepareOf(worldConfig);
326
+ let migration: PreparedMigration | undefined;
327
+ let stepFailure: { error: unknown } | undefined;
328
+ if (step) {
329
+ try { migration = await runDatabasePrepare(step, worldConfig, { ...launch, env }); }
330
+ catch (error) { stepFailure = { error }; }
331
+ }
332
+ // The clean stop: SIGINT is the postmaster's fast shutdown, which writes its checkpoint and removes postmaster.pid.
333
+ // Its wait is the lifecycle's own readiness budget, not a new limit; down then clears the receipts.
334
+ const identity = readSpawnReceipt(recordPath(root)).value;
335
+ if (identity) {
336
+ await signalOwnedProcess(identity, 'SIGINT', true);
337
+ const deadline = Date.now() + READY_TIMEOUT_MS;
338
+ while (liveGroup(identity.pid) && Date.now() < deadline) await wait(100);
339
+ }
340
+ await nativePostgresDown(root);
341
+ if (stepFailure) {
342
+ rmSync(directory, { recursive: true, force: true });
343
+ throw failed(stepFailure.error);
344
+ }
345
+ if (!existsSync(join(directory, 'PG_VERSION')) || existsSync(join(directory, 'postmaster.pid'))) {
346
+ rmSync(directory, { recursive: true, force: true });
347
+ throw failed(new Error('native prepare ended without a cleanly stopped cluster'));
348
+ }
349
+ rmSync(log, { force: true });
350
+ writeFileSync(stamp, `${JSON.stringify(nativeStamp(tools, user, database, true, migration))}\n`);
351
+ return directory;
352
+ }
353
+
203
354
  export async function nativePostgresUp(services: InfraService[], root: string, definition: string, launch: HostLaunch): Promise<void> {
204
355
  if (services.length !== 1 || services[0]!.kind !== 'postgres') throw new Error('native postgres requires one declared postgres service');
205
356
  const deadline = Date.now() + READY_TIMEOUT_MS;
@@ -223,10 +374,26 @@ export async function nativePostgresUp(services: InfraService[], root: string, d
223
374
  const service = services[0]!;
224
375
  if (await listening(service.hostPort, deadline, launch.signal)) throw new Error(`native postgres port ${service.hostPort} is already occupied`);
225
376
  const data = resolve(root, 'native-postgres-data');
226
- mkdirSync(data, { recursive: true, mode: 0o700 });
227
377
  const { user, database } = settings(definition);
378
+ // An empty data directory takes the World's prepared cluster when its stamp matches (ADR 0018's placement; the
379
+ // cluster was made by these same programs at image build). initdb then finds PG_VERSION, and the database exists.
380
+ forgetStaleMigration(data);
381
+ const prepared = env.VOLTER_WORLD_CONFIG ? nativePreparedCluster(env.VOLTER_WORLD_CONFIG) : undefined;
382
+ // The check reads two files' metadata and the stamp; it runs only when a prepared cluster is there to take.
383
+ if (prepared && existsSync(join(prepared.directory, 'PG_VERSION')) && !existsSync(join(data, 'PG_VERSION'))) {
384
+ // A migrated cluster is the app's only for the step inputs it was migrated from (pglite-backing.ts, expectedMigration).
385
+ const wanted = expectedMigration(env.VOLTER_WORLD_CONFIG!, env, prepared.directory);
386
+ if ('note' in wanted) launch.note?.(wanted.note);
387
+ else {
388
+ const expected = nativeStamp(binaries, user, database, false, wanted.migration);
389
+ const placed = takePreparedCluster(prepared, text => nativeStampMatches(text, expected), data,
390
+ 'other PostgreSQL programs, initdb arguments, identity or migration', wanted.migration);
391
+ if (placed.note) launch.note?.(placed.note);
392
+ }
393
+ }
394
+ mkdirSync(data, { recursive: true, mode: 0o700 });
228
395
  if (!existsSync(join(data, 'PG_VERSION'))) {
229
- await tool(join(binaries, 'initdb'), ['-D', data, `--username=${user}`, '--auth=trust', '--no-locale', '--encoding=UTF8'], env, deadline, root, launch.signal);
396
+ await tool(join(binaries, 'initdb'), ['-D', data, ...initdbArguments(user)], env, deadline, root, launch.signal);
230
397
  }
231
398
  launch.signal?.throwIfAborted();
232
399
  const binary = join(binaries, 'postgres');
@@ -260,8 +427,8 @@ export async function nativePostgresUp(services: InfraService[], root: string, d
260
427
  for (;;) {
261
428
  launch.signal?.throwIfAborted();
262
429
  if (started.error) throw started.error;
263
- if (child.exitCode !== null || child.signalCode !== null || started.closed) throw new Error(`native postgres exited before readiness; see ${log}`);
264
- if (Date.now() >= deadline) throw new Error(`native postgres did not become ready; see ${log}`);
430
+ if (child.exitCode !== null || child.signalCode !== null || started.closed) throw new Error(`native postgres exited before readiness; see ${log}${logTail(log)}`);
431
+ if (Date.now() >= deadline) throw new Error(`native postgres did not become ready; see ${log}${logTail(log)}`);
265
432
  if (existsSync(join(data, 'postmaster.pid'))) {
266
433
  verifyOwner(current);
267
434
  const lines = readFileSync(join(data, 'postmaster.pid'), 'utf8').split('\n');
@@ -275,7 +442,7 @@ export async function nativePostgresUp(services: InfraService[], root: string, d
275
442
  if (!output.trim()) await tool(join(binaries, 'createdb'), ['-h', '127.0.0.1', '-p', String(service.hostPort), '-U', user, '--', database], env, deadline, root, launch.signal);
276
443
  }
277
444
  launch.signal?.throwIfAborted();
278
- if (child.exitCode !== null || child.signalCode !== null) throw new Error(`native postgres exited during readiness; see ${log}`);
445
+ if (child.exitCode !== null || child.signalCode !== null) throw new Error(`native postgres exited during readiness; see ${log}${logTail(log)}`);
279
446
  rmSync(startingPath(root));
280
447
  child.unref();
281
448
  } catch (error) {
@@ -0,0 +1,31 @@
1
+ // THIS PACKAGE'S OWN MANIFEST, found by its name walking up from this module. One finder for every module of the
2
+ // runtime that needs it (a prepared cluster's stamp records the runtime's version and its PGlite's).
3
+ //
4
+ // A fixed hop is wrong in one of the two places the runtime runs from: src/ in the repository is one directory below
5
+ // the package root, dist/src/ in the published package is two, so `..` from a published module is dist/, which
6
+ // holds no package.json. That mistake was made twice and fixed once: the PGlite backing walked up, the native
7
+ // backing still hopped, and a native preparation failed after its migration had run (Cal run 28: "ENOENT ...
8
+ // node_modules/@volter/world-runtime/dist/package.json").
9
+ //
10
+ // Nothing runs at import: a module that builds a URL from import.meta.url when it loads fails to load in workerd,
11
+ // which bundles the runtime into the hosted World.
12
+ import { existsSync, readFileSync } from 'node:fs';
13
+ import { dirname, join } from 'node:path';
14
+ import { fileURLToPath } from 'node:url';
15
+
16
+ const NAME = '@volter/world-runtime';
17
+
18
+ export interface RuntimeManifest { name?: string; version?: string; dependencies?: Record<string, string> }
19
+
20
+ /** The runtime's package.json, from wherever this module runs. */
21
+ export function runtimeManifest(): RuntimeManifest {
22
+ const from = fileURLToPath(new URL('.', import.meta.url));
23
+ for (let dir = from; ; dir = dirname(dir)) {
24
+ const candidate = join(dir, 'package.json');
25
+ if (existsSync(candidate)) {
26
+ const manifest = JSON.parse(readFileSync(candidate, 'utf8')) as RuntimeManifest;
27
+ if (manifest.name === NAME) return manifest;
28
+ }
29
+ if (dirname(dir) === dir) throw new Error(`no ${NAME} package.json above ${from}`);
30
+ }
31
+ }
@@ -12,15 +12,18 @@
12
12
  // services only. A definition declaring mysql without a container runtime is
13
13
  // refused with the kinds named, never half-booted. Redis never reaches it: every
14
14
  // redis service is the redis twin's (redis-backing.ts).
15
- import { appendFileSync, closeSync, cpSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmdirSync, rmSync, writeFileSync } from 'node:fs';
16
- import { randomUUID } from 'node:crypto';
15
+ import { appendFileSync, closeSync, copyFileSync, cpSync, existsSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readlinkSync, renameSync, rmdirSync, rmSync, writeFileSync } from 'node:fs';
16
+ import { createHash, randomUUID } from 'node:crypto';
17
+ import { assertWorldConfig, type WorldDatabasePrepare } from './schema.ts';
17
18
  import { connect } from 'node:net';
18
19
  import { dirname, join } from 'node:path';
20
+ import { tmpdir } from 'node:os';
19
21
  import { spawn, type ChildProcess } from 'node:child_process';
20
22
  import { commandLifetime, COMMAND_GRACE_MS, COMMAND_CONFIRM_MS } from './command-lifetime.ts';
21
23
  import { survivingOwnedGroups } from './process-groups.ts';
22
24
  import { publishProcessReceipt, readProcessReceipt, readSpawnReceipt, spawnReceiptPath, verifySpawnIdentity, signalOwnedProcess, type SpawnIdentity } from './process-receipt.ts';
23
25
  import { errorMessage } from './error-message.ts';
26
+ import { runtimeManifest } from './package-manifest.ts';
24
27
  import { fileURLToPath } from 'node:url';
25
28
  import { parse } from 'yaml';
26
29
  import { mongodbArgv, mongodbReady, mongodbServes } from './mongodb-backing.ts';
@@ -220,24 +223,135 @@ export function preparedCluster(worldConfig: string): { directory: string; stamp
220
223
  * another password takes the same cluster and authenticates against its own. Extensions are loaded into the module
221
224
  * at each open, never created in the cluster by the open path.
222
225
  */
223
- /** This package's own manifest, found by its name walking up from this module: the package root from src/ in the
224
- * repository and from dist/src/ in the published package alike, where a fixed `..` reached dist/package.json. */
225
- function runtimeManifest(): { version?: string; dependencies?: Record<string, string> } {
226
- for (let dir = here(); ; dir = dirname(dir)) {
227
- const candidate = join(dir, 'package.json');
228
- if (existsSync(candidate)) {
229
- const manifest = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string; version?: string; dependencies?: Record<string, string> };
230
- if (manifest.name === '@volter/world-runtime') return manifest;
231
- }
232
- if (dirname(dir) === dir) throw new Error(`no @volter/world-runtime package.json above ${here()}`);
233
- }
234
- }
226
+ /** A prepared cluster's migration, as its stamp records it: the step's command and the digest of each of its inputs. */
227
+ export interface PreparedMigration { command: string[]; inputs: Record<string, string> }
235
228
 
236
- function preparedStamp(service: InfraService): string {
229
+ function preparedStamp(service: InfraService, migration?: PreparedMigration): string {
237
230
  const manifest = runtimeManifest();
238
231
  const pglite = manifest.dependencies?.['@electric-sql/pglite'];
239
232
  if (!pglite || !/^\d+\.\d+\.\d+$/.test(pglite)) throw new Error('world-runtime pins no exact @electric-sql/pglite version');
240
- return `${JSON.stringify({ runtime: manifest.version ?? 'unversioned', pglite, user: service.postgres!.user, database: service.postgres!.database })}\n`;
233
+ return `${JSON.stringify({ runtime: manifest.version ?? 'unversioned', pglite, user: service.postgres!.user, database: service.postgres!.database,
234
+ ...(migration ? { migration } : {}) })}\n`;
235
+ }
236
+
237
+ /**
238
+ * The SHA-256 of each of a migration step's inputs, under its cwd in the app's tree: a file's bytes; a directory's
239
+ * sorted relative paths, each with its type, its mode and its bytes (a link, its target). What the image build records
240
+ * for an image, and what `up` hashes only for a World run outside one.
241
+ */
242
+ export function migrationInputDigests(cwd: string, inputs: readonly string[]): Record<string, string> {
243
+ const digests: Record<string, string> = {};
244
+ for (const input of inputs) {
245
+ const hash = createHash('sha256');
246
+ const walk = (absolute: string, relative: string): void => {
247
+ const stat = lstatSync(absolute);
248
+ if (stat.isSymbolicLink()) { hash.update(`l ${relative} ${readlinkSync(absolute)}\n`); return; }
249
+ if (stat.isDirectory()) {
250
+ hash.update(`d ${relative} ${(stat.mode & 0o7777).toString(8)}\n`);
251
+ for (const name of readdirSync(absolute).sort()) walk(join(absolute, name), relative === '' ? name : `${relative}/${name}`);
252
+ return;
253
+ }
254
+ hash.update(`f ${relative} ${(stat.mode & 0o7777).toString(8)} ${stat.size}\n`);
255
+ hash.update(readFileSync(absolute));
256
+ };
257
+ walk(join(cwd, input), '');
258
+ digests[input] = `sha256:${hash.digest('hex')}`;
259
+ }
260
+ return digests;
261
+ }
262
+
263
+ /** The World's declared migration step for its prepared database, if any (runtime.database.prepare). */
264
+ export function databasePrepareOf(worldConfig: string): WorldDatabasePrepare | undefined {
265
+ return assertWorldConfig(JSON.parse(readFileSync(worldConfig, 'utf8')), worldConfig).databasePrepare;
266
+ }
267
+
268
+ /** `NAME=value` lines (an image's environment, VOLTER_WORLD_APP_ENV), as an environment. */
269
+ function environmentFile(path: string): Record<string, string> {
270
+ const values: Record<string, string> = {};
271
+ for (const line of readFileSync(path, 'utf8').split('\n')) {
272
+ const at = line.indexOf('=');
273
+ if (at > 0) values[line.slice(0, at)] = line.slice(at + 1);
274
+ }
275
+ return values;
276
+ }
277
+
278
+ /**
279
+ * The World's declared step (runtime.database.prepare) run once against a prepared cluster its backing is serving on
280
+ * the World's port: the declared command in the app's tree, with the image's environment and the World's values over
281
+ * it. The build host's own PATH finds the command's tools (the image's PATH names its own paths). Its failure throws,
282
+ * with its output; its answer is what the cluster's stamp records.
283
+ */
284
+ /**
285
+ * A FAILED PREPARATION KEEPS ITS WHOLE EVIDENCE. An error carries a tail, and the preparation's own directory is
286
+ * its caller's to remove (an image build prepares in a staging directory it retires when the preparation fails), so
287
+ * the tail was all that survived: a postmaster's first lines, its shared-memory setup among them, were gone with the
288
+ * directory (Cal runs 23 and 24). The whole text or file is written under the preparation's temporary directory
289
+ * (TMPDIR, which a failed build retains; the system's otherwise), outside the World's directory, and the sentence
290
+ * naming it is added to the error. Never on success, and a copy that fails says so instead of hiding the failure.
291
+ */
292
+ export function keepFailureEvidence(name: string, evidence: string | { file: string }, env: NodeJS.ProcessEnv): string {
293
+ try {
294
+ const directory = env.TMPDIR ?? tmpdir();
295
+ mkdirSync(directory, { recursive: true });
296
+ const path = join(directory, `volter-world-prepare-${name}-${Date.now()}-${process.pid}.log`);
297
+ if (typeof evidence === 'string') writeFileSync(path, evidence); else copyFileSync(evidence.file, path);
298
+ return `\nThe whole ${name} is kept at ${path}`;
299
+ } catch (error) {
300
+ return `\n(the whole ${name} could not be kept: ${errorMessage(error)})`;
301
+ }
302
+ }
303
+
304
+ export async function runDatabasePrepare(step: WorldDatabasePrepare, worldConfig: string, launch: HostLaunch): Promise<PreparedMigration> {
305
+ const env = launch.env ?? process.env;
306
+ const appRoot = env.VOLTER_WORLD_APP_ROOT;
307
+ if (!appRoot) throw new Error('the World declares runtime.database.prepare: VOLTER_WORLD_APP_ROOT must name the app image\'s root');
308
+ const cwd = join(appRoot, step.cwd);
309
+ const imageEnv = env.VOLTER_WORLD_APP_ENV ? environmentFile(env.VOLTER_WORLD_APP_ENV) : {};
310
+ const { PATH: _imagePath, ...imageValues } = imageEnv;
311
+ const worldValues = assertWorldConfig(JSON.parse(readFileSync(worldConfig, 'utf8')), worldConfig).env ?? {};
312
+ const [program, ...words] = step.command;
313
+ const migration = spawn(program!, words, { windowsHide: true, cwd, env: { ...env, ...imageValues, ...worldValues }, stdio: ['ignore', 'pipe', 'pipe'] });
314
+ let output = '';
315
+ migration.stdout!.on('data', (chunk: Buffer) => { output += chunk.toString('utf8'); });
316
+ migration.stderr!.on('data', (chunk: Buffer) => { output += chunk.toString('utf8'); });
317
+ const { code } = await commandLifetime(migration, launch.signal).finish();
318
+ if (code !== 0) throw new Error(`the World's database prepare step (${step.command.join(' ')}) exited ${code ?? 'null'}\n${output.trim().split('\n').slice(-40).join('\n')}${keepFailureEvidence('database-prepare-step-output', output, env)}`);
319
+ launch.note?.(`database prepare step: ${step.command.join(' ')}\n${output.trim()}`);
320
+ return { command: [...step.command], inputs: migrationInputDigests(cwd, step.inputs) };
321
+ }
322
+
323
+ /**
324
+ * The app's own migration against a prepared PGlite cluster: the cluster served by the same host at the World's port
325
+ * for the step (runDatabasePrepare), then the host closed cleanly.
326
+ */
327
+ async function migratePreparedCluster(service: InfraService, directory: string, step: WorldDatabasePrepare, worldConfig: string, launch: HostLaunch): Promise<PreparedMigration> {
328
+ const env = launch.env ?? process.env;
329
+ const pidFile = join(dirname(directory), 'pglite-prepare.pid');
330
+ const boot = randomUUID();
331
+ rmSync(listeningReceiptPath(pidFile), { force: true });
332
+ const [command, ...args] = [...hostArgv(service, directory), '--prepared', '--listening-file', listeningReceiptPath(pidFile), '--boot', boot];
333
+ let hostOutput = '';
334
+ const host = spawn(command!, args, { windowsHide: true,
335
+ env: { ...env, VOLTER_WORLD_PGLITE_USER: service.postgres!.user, VOLTER_WORLD_PGLITE_DATABASE: service.postgres!.database, VOLTER_WORLD_PGLITE_PASSWORD: service.postgres!.password! },
336
+ stdio: ['ignore', 'pipe', 'pipe'] });
337
+ host.stdout!.on('data', (chunk: Buffer) => { hostOutput += chunk.toString('utf8'); });
338
+ host.stderr!.on('data', (chunk: Buffer) => { hostOutput += chunk.toString('utf8'); });
339
+ const hostLifetime = commandLifetime(host, launch.signal);
340
+ try {
341
+ const deadline = Date.now() + READY_TIMEOUT_MS;
342
+ while (!(host.pid && alive(host.pid) && await hostServes(service, pidFile, host.pid, boot))) {
343
+ if (!host.pid || !alive(host.pid)) throw new Error(`the prepared cluster's host exited before serving\n${hostOutput.trim().split('\n').slice(-20).join('\n')}`);
344
+ if (Date.now() > deadline) throw new Error(`the prepared cluster's host did not serve port ${service.hostPort} within ${READY_TIMEOUT_MS / 1000}s`);
345
+ launch.signal?.throwIfAborted();
346
+ await sleep(25);
347
+ }
348
+ return await runDatabasePrepare(step, worldConfig, launch);
349
+ } finally {
350
+ // A clean close: PGlite writes its cluster back before the host exits.
351
+ if (host.pid && alive(host.pid)) host.kill('SIGTERM');
352
+ await hostLifetime.finish();
353
+ rmSync(listeningReceiptPath(pidFile), { force: true });
354
+ }
241
355
  }
242
356
 
243
357
  /** Run the host's open path once into the World's prepared directory; replaces an earlier preparation. */
@@ -260,18 +374,108 @@ export async function pglitePrepare(service: InfraService, worldConfig: string,
260
374
  const { code } = await lifetime.finish();
261
375
  if (code !== 0 || !existsSync(join(directory, 'PG_VERSION'))) {
262
376
  rmSync(directory, { recursive: true, force: true });
263
- throw new Error(`pglite prepare exited ${code ?? 'null'} without a cluster\n${output.trim().split('\n').slice(-40).join('\n')}`);
377
+ throw new Error(`pglite prepare exited ${code ?? 'null'} without a cluster\n${output.trim().split('\n').slice(-40).join('\n')}${keepFailureEvidence('pglite-prepare-output', output, launch.env ?? process.env)}`);
264
378
  }
265
- writeFileSync(stamp, preparedStamp(service));
379
+ // The app's own migration, where the World declares one: a cluster it did not complete is never stamped.
380
+ const step = databasePrepareOf(worldConfig);
381
+ let migration: PreparedMigration | undefined;
382
+ if (step) {
383
+ try { migration = await migratePreparedCluster(service, directory, step, worldConfig, launch); }
384
+ catch (error) { rmSync(directory, { recursive: true, force: true }); throw error; }
385
+ }
386
+ writeFileSync(stamp, preparedStamp(service, migration));
266
387
  return directory;
267
388
  }
268
389
 
269
390
  /** Give an empty service data directory the World's prepared cluster when its stamp matches; says what it did. */
270
- function placePreparedCluster(service: InfraService, serviceData: string, worldConfig: string | undefined): { placed: boolean; note?: string } {
391
+ function placePreparedCluster(service: InfraService, serviceData: string, worldConfig: string | undefined, env: NodeJS.ProcessEnv): { placed: boolean; note?: string } {
392
+ if (service.kind === 'postgres') forgetStaleMigration(serviceData);
271
393
  if (service.kind !== 'postgres' || !worldConfig || existsSync(join(serviceData, 'PG_VERSION'))) return { placed: false };
272
- const { directory, stamp } = preparedCluster(worldConfig);
394
+ const prepared = preparedCluster(worldConfig);
395
+ if (!existsSync(join(prepared.directory, 'PG_VERSION')) || !existsSync(prepared.stamp)) return { placed: false };
396
+ const wanted = expectedMigration(worldConfig, env, prepared.directory);
397
+ if ('note' in wanted) return { placed: false, note: wanted.note };
398
+ return takePreparedCluster(prepared, preparedStamp(service, wanted.migration), serviceData, 'another PGlite pin, identity or migration', wanted.migration);
399
+ }
400
+
401
+ /**
402
+ * The migration a prepared cluster must have been made with for `up` to take it, on either backing: none when the
403
+ * World declares no step; else the step's command and its inputs' digests, as the app's image recorded them
404
+ * (VOLTER_WORLD_APP_DIGESTS, no read of the tree) or, for a World run outside an image, as its tree has them. With
405
+ * neither given a migrated cluster is not taken, and the note says so.
406
+ */
407
+ export function expectedMigration(worldConfig: string, env: NodeJS.ProcessEnv, directory: string): { migration?: PreparedMigration } | { note: string } {
408
+ const step = databasePrepareOf(worldConfig);
409
+ if (!step) return {};
410
+ let inputs: Record<string, string> | undefined;
411
+ if (env.VOLTER_WORLD_APP_DIGESTS) {
412
+ // The image wrote this; a value that is not a JSON object of digests is said, and the cluster is not taken.
413
+ let recorded: unknown;
414
+ try { recorded = JSON.parse(env.VOLTER_WORLD_APP_DIGESTS); } catch { recorded = undefined; }
415
+ if (!recorded || typeof recorded !== 'object' || Array.isArray(recorded) || Object.values(recorded).some((value) => typeof value !== 'string')) {
416
+ return { note: `prepared cluster ${directory} not taken: it is migrated, and VOLTER_WORLD_APP_DIGESTS is not a JSON object of its inputs' digests` };
417
+ }
418
+ inputs = recorded as Record<string, string>;
419
+ } else if (env.VOLTER_WORLD_APP_ROOT) inputs = migrationInputDigests(join(env.VOLTER_WORLD_APP_ROOT, step.cwd), step.inputs);
420
+ if (!inputs) return { note: `prepared cluster ${directory} not taken: it is migrated, and neither the app's recorded digests nor its tree were given to compare (VOLTER_WORLD_APP_DIGESTS, VOLTER_WORLD_APP_ROOT)` };
421
+ return { migration: { command: [...step.command], inputs: Object.fromEntries(step.inputs.map((input) => [input, inputs![input] ?? ''])) } };
422
+ }
423
+
424
+ /** Beside a service's data: the migration its live cluster carries, written when a migrated cluster is placed. */
425
+ const migrationRecord = (serviceData: string): string => `${serviceData}.migration.json`;
426
+
427
+ /** A data directory with no cluster carries no migration: a record left by data since removed is dropped. */
428
+ export function forgetStaleMigration(serviceData: string): void {
429
+ if (!existsSync(join(serviceData, 'PG_VERSION'))) rmSync(migrationRecord(serviceData), { force: true });
430
+ }
431
+
432
+ /**
433
+ * The declared step's command when the LIVE database carries the migration the app's image expects, else undefined
434
+ * (architecture, managed infrastructure). A fact of the data, not of this boot's placement: kept data is still
435
+ * migrated on a later boot, and kept data under an image whose step inputs have changed is older than the image.
436
+ */
437
+ export function liveDatabaseMigration(worldConfig: string, env: NodeJS.ProcessEnv, serviceData: string): string[] | undefined {
438
+ if (!existsSync(join(serviceData, 'PG_VERSION')) || !existsSync(migrationRecord(serviceData))) return undefined;
439
+ const wanted = expectedMigration(worldConfig, env, serviceData);
440
+ if ('note' in wanted || !wanted.migration) return undefined;
441
+ return readFileSync(migrationRecord(serviceData), 'utf8') === `${JSON.stringify(wanted.migration)}\n` ? [...wanted.migration.command] : undefined;
442
+ }
443
+
444
+ /**
445
+ * The declared step's command when the World declares one and the live database does NOT carry the migration the
446
+ * image expects: data kept from an earlier image, or a cluster that was initialized here because the prepared one
447
+ * was not taken. Undefined with no step, no cluster yet, or a migrated one.
448
+ */
449
+ export function liveDatabaseBehind(worldConfig: string, env: NodeJS.ProcessEnv, serviceData: string): string[] | undefined {
450
+ const step = databasePrepareOf(worldConfig);
451
+ if (!step || !existsSync(join(serviceData, 'PG_VERSION')) || liveDatabaseMigration(worldConfig, env, serviceData)) return undefined;
452
+ return [...step.command];
453
+ }
454
+
455
+ /**
456
+ * Whether this is a development World whose app may migrate its database when it starts
457
+ * (runtime.development.migrateDatabaseAtRuntime; schema.ts WorldDevelopment says whose the setting is).
458
+ */
459
+ export function developmentMigratesAtRuntime(worldConfig: string): boolean {
460
+ return assertWorldConfig(JSON.parse(readFileSync(worldConfig, 'utf8')), worldConfig).development?.migrateDatabaseAtRuntime === true;
461
+ }
462
+
463
+ /** What a development World says each time it is brought up or built into an image, so its use is never silent. */
464
+ export const DEVELOPMENT_MIGRATION_NOTICE = 'development World: its database migrates at run time; it is not prepared';
465
+
466
+ /**
467
+ * The one placement of a prepared cluster, for every backing that prepares one (PGlite here, native PostgreSQL in
468
+ * native-postgres-backing.ts): an empty `serviceData` takes `prepared.directory` by rename when the stamp beside it
469
+ * is exactly `expected`; a mismatch is named, never taken. A migrated cluster's `migration` is recorded beside the data
470
+ * it becomes (liveDatabaseMigration).
471
+ */
472
+ export function takePreparedCluster(prepared: { directory: string; stamp: string }, expected: string | ((stamp: string) => boolean), serviceData: string,
473
+ mismatch: string, migration?: PreparedMigration): { placed: boolean; note?: string } {
474
+ const { directory, stamp } = prepared;
475
+ if (existsSync(join(serviceData, 'PG_VERSION'))) return { placed: false };
273
476
  if (!existsSync(join(directory, 'PG_VERSION')) || !existsSync(stamp)) return { placed: false };
274
- if (readFileSync(stamp, 'utf8') !== preparedStamp(service)) return { placed: false, note: `prepared cluster ${directory} not taken: its stamp names another PGlite pin or identity` };
477
+ const text = readFileSync(stamp, 'utf8');
478
+ if (typeof expected === 'string' ? text !== expected : !expected(text)) return { placed: false, note: `prepared cluster ${directory} not taken: its stamp names ${mismatch}` };
275
479
  if (existsSync(serviceData)) {
276
480
  if (readdirSync(serviceData).length) return { placed: false };
277
481
  rmdirSync(serviceData);
@@ -283,6 +487,7 @@ function placePreparedCluster(service: InfraService, serviceData: string, worldC
283
487
  if ((error as NodeJS.ErrnoException).code !== 'EXDEV') throw error;
284
488
  cpSync(directory, serviceData, { recursive: true });
285
489
  }
490
+ if (migration) writeFileSync(migrationRecord(serviceData), `${JSON.stringify(migration)}\n`);
286
491
  return { placed: true, note: `prepared cluster ${directory} placed at ${serviceData}` };
287
492
  }
288
493
 
@@ -312,7 +517,7 @@ export async function pgliteUp(services: InfraService[], dataDir: string, launch
312
517
  const started: string[] = [];
313
518
  try {
314
519
  for (const { service, name, serviceData, log, pidFile, argv } of plan) {
315
- const placed = placePreparedCluster(service, serviceData, env.VOLTER_WORLD_CONFIG);
520
+ const placed = placePreparedCluster(service, serviceData, env.VOLTER_WORLD_CONFIG, env);
316
521
  mkdirSync(serviceData, { recursive: true });
317
522
  mkdirSync(dirname(log), { recursive: true });
318
523
  if (placed.note) { appendFileSync(log, `${placed.note}\n`); launch.note?.(placed.note); }