@bongos/core 1.20.38 → 1.20.39

package/modules/npm-release/preview/childlog.js

@@ -0,0 +1,95 @@
+'use strict';
+
+// modules/npm-release/preview/childlog.js — what the preview child printed, kept small and
+// with secrets taken out (task 1004465).
+//
+// WHY. The child used to be spawned with stdio 'ignore', so when a preview started but could
+// not serve a single page, its own error went nowhere and the only way to find out was ssh.
+// This keeps the last CAP bytes of its stdout and stderr, writes them to a file in the
+// preview's scratch directory, and hands the deploy page a short redacted tail.
+//
+// REDACTION IS BY VALUE AND BY SHAPE. Every value in the environment the child was given
+// (and the live one) whose name looks secret is replaced wherever it appears; then the common
+// shapes (a password in a connection URL, `token=...`, a bearer header, a GitHub or API key)
+// are replaced even if nobody listed them. The tail never reaches a caller unredacted: it is
+// redacted at read time as well as at write time, so a secret that arrives split across two
+// chunks is still caught.
+
+const CAP = 64 * 1024;
+// The file is rewritten at most this often, asynchronously: the child's output is handled
+// inside the LIVE hall's process, and a chatty or crash-looping child must never turn into a
+// run of blocking disk writes on its event loop.
+const FLUSH_MS = 500;
+const TAIL_LINES = 20;
+const MAX_LINE = 300;
+const SECRET_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PASS|KEY|CREDENTIAL|COOKIE|AUTH)/i;
+
+const SHAPES = [
+  [/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]*:)[^\s@/]+@/gi, '$1[redacted]@'],
+  [/\b((?:api[_-]?key|token|secret|password|passwd|authorization|cookie|signing[_-]?key)\w*\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, '$1[redacted]'],
+  [/\bBearer\s+[A-Za-z0-9._~+/=-]+/g, 'Bearer [redacted]'],
+  [/\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-[A-Za-z0-9_-]{16,}|xox[baprs]-[A-Za-z0-9-]{10,})/g, '[redacted]'],
+];
+
+/** The values in these environments that look like secrets and are long enough to matter. */
+function secretValues(...envs) {
+  const out = new Set();
+  for (const env of envs) {
+    for (const [k, v] of Object.entries(env || {})) {
+      if (SECRET_NAME.test(k) && typeof v === 'string' && v.length >= 6) out.add(v);
+    }
+  }
+  // Longest first, so a value that contains another is replaced whole.
+  return [...out].sort((a, b) => b.length - a.length);
+}
+
+/** Take the secrets out of one piece of text. Pure. */
+function redact(text, secrets = []) {
+  let s = String(text);
+  for (const v of secrets) s = s.split(v).join('[redacted]');
+  for (const [re, to] of SHAPES) s = s.replace(re, to);
+  return s;
+}
+
+/**
+ * @param {object} [o]
+ * @param {string[]} [o.secrets]  exact values to take out (from secretValues)
+ * @param {object} [o.fsImpl]     fs (writeFile, the callback form) — optional, with `file`
+ * @param {string} [o.file]       where to keep the capped raw-but-redacted log
+ * @param {number} [o.cap]
+ */
+function createChildLog({ secrets = [], fsImpl = null, file = null, cap = CAP, flushMs = FLUSH_MS } = {}) {
+  let buf = '';
+  let timer = null;
+  let closed = false;
+  const canWrite = () => !!(fsImpl && file && typeof fsImpl.writeFile === 'function');
+  function flush() {
+    timer = null;
+    if (closed || !canWrite()) return;
+    try { fsImpl.writeFile(file, buf, () => { /* best effort */ }); } catch { /* best effort */ }
+  }
+  return {
+    /** Add output. Never throws: a log that cannot be kept must not fail a start. */
+    append(chunk) {
+      try {
+        buf = (buf + redact(chunk, secrets)).slice(-cap);
+        if (canWrite() && !timer && !closed) {
+          timer = setTimeout(flush, flushMs);
+          if (timer && typeof timer.unref === 'function') timer.unref();
+        }
+      } catch { /* best effort */ }
+    },
+    /** Write the file now (tests; the timer does this in production). */
+    flush,
+    /** Stop writing: the scratch directory is about to be removed. */
+    close() { closed = true; if (timer) { clearTimeout(timer); timer = null; } },
+    /** The last `n` non-empty lines, each shortened. */
+    tail(n = TAIL_LINES) {
+      return redact(buf, secrets).split(/\r?\n/).filter((l) => l.trim())
+        .slice(-n).map((l) => (l.length > MAX_LINE ? `${l.slice(0, MAX_LINE)}…` : l));
+    },
+    size: () => buf.length,
+  };
+}
+
+module.exports = { createChildLog, redact, secretValues };

package/modules/npm-release/preview/commands.js

@@ -36,11 +36,49 @@ function installCommand({ cacheRoot, version }) {
   return { cmd: 'npm', args: ['install', '--prefix', versionDir(cacheRoot, version), `${PACKAGE}@${version}`, '--omit=dev'] };
 }
 
-/** Drop the copy if one is left over — used before a start and on every stop. */
+/**
+ * Drop the copy if one is left over — used before a start and on every stop. --force ends any
+ * connection still open on the copy (Postgres 13+), which is exactly the case that used to
+ * leave it behind: a child that had not finished exiting. The name is the fixed PREVIEW_DB
+ * and nothing is interpolated, so this can never name the live database.
+ */
 function dropDbCommand() {
+  return { cmd: 'dropdb', args: ['--if-exists', '--force', PREVIEW_DB] };
+}
+
+/** Plain drop, for a server whose dropdb has no --force (Postgres 12 and older). */
+function dropDbPlainCommand() {
   return { cmd: 'dropdb', args: ['--if-exists', PREVIEW_DB] };
 }
 
+/**
+ * End every connection to the copy and ONLY the copy: the database name is the literal
+ * PREVIEW_DB, never a parameter. Run on the maintenance database, since the copy cannot be
+ * connected to while it is being emptied.
+ */
+function terminateBackendsCommand() {
+  return {
+    cmd: 'psql',
+    args: ['-X', '-d', 'postgres', '-tAc',
+      `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${PREVIEW_DB}' AND pid <> pg_backend_pid()`],
+  };
+}
+
+/**
+ * Can this command only ever touch the copy? True for the drop commands above and nothing
+ * else. The supervisor checks it before running any drop, so a future edit that pointed one
+ * at another database is refused at run time, not discovered on the droplet.
+ */
+function targetsOnlyPreviewDb(command) {
+  if (!command || !Array.isArray(command.args)) return false;
+  if (command.cmd === 'dropdb') {
+    const last = command.args[command.args.length - 1];
+    return last === PREVIEW_DB && command.args.slice(0, -1).every((a) => a.startsWith('--'));
+  }
+  if (command.cmd === 'psql') return JSON.stringify(command.args) === JSON.stringify(terminateBackendsCommand().args);
+  return false;
+}
+
 /** An empty database for the copy. Deliberately no template (see the header). */
 function createDbCommand() {
   return { cmd: 'createdb', args: [PREVIEW_DB] };
@@ -65,17 +103,20 @@ function restoreCommand({ dumpFile }) {
  * DATABASE_URL is blanked because the pool prefers it to PGDATABASE, and PGDATABASE is
  * pinned to the copy so nothing can be applied to the live database by mistake.
  */
-function migrateCommand({ cacheRoot, version, env }) {
+function migrateCommand({ cacheRoot, version, env, instanceDir }) {
   const dir = versionDir(cacheRoot, version);
+  // With a scratch instance directory, THAT is the instance root (its config/ and
+  // migrations/instance/ are copies of the live instance's); without one, the install directory.
+  const root = instanceDir || dir;
   return {
     cmd: 'bash',
     args: [path.join(dir, 'node_modules', '@bongos', 'core', 'scripts', 'migrate.sh')],
-    cwd: dir,
-    env: { ...env, INIT_CWD: dir, PGDATABASE: PREVIEW_DB, DATABASE_URL: '' },
+    cwd: root,
+    env: { ...env, INIT_CWD: root, PGDATABASE: PREVIEW_DB, DATABASE_URL: '' },
   };
 }
 
 module.exports = {
   VERSION_RE, PACKAGE, versionDir, entryPath,
-  installCommand, dropDbCommand, createDbCommand, dumpCommand, restoreCommand, migrateCommand,
+  installCommand, dropDbCommand, dropDbPlainCommand, terminateBackendsCommand, targetsOnlyPreviewDb, createDbCommand, dumpCommand, restoreCommand, migrateCommand,
 };

package/modules/npm-release/preview/env.js

@@ -41,12 +41,15 @@ const POLLERS_OFF = Object.freeze([
 
 // Modules that talk to the outside world, off. npm-release is off too: a preview must not
 // offer previews of its own.
-const MODULES_OFF = Object.freeze([
-  'BONGOS_MODULE_DISCORD',
-  'BONGOS_MODULE_AGENTS',
-  'BONGOS_MODULE_PROVISIONING',
-  'BONGOS_MODULE_NPM_RELEASE',
-]);
+//
+// EVERY SPELLING OF THE PREFIX. The core reads <PREFIX>_MODULE_<KEY> with the PREFIX the
+// instance's branding pack names, then the legacy ones. The child now runs with a copy of the
+// live instance's config/ (task 1004465), so its prefix may be the live instance's own, and a
+// switch spelled only BONGOS_ would then be ignored — Discord and the agents would come back
+// on in the copy. So each switch is set under every prefix the core can resolve.
+const MODULE_KEYS = Object.freeze(['DISCORD', 'AGENTS', 'PROVISIONING', 'NPM_RELEASE']);
+const PREFIXES = Object.freeze(['BONGOS', 'CLOUDBONGOS', 'OTB', 'GDS', 'PMS']);
+const MODULES_OFF = Object.freeze(PREFIXES.flatMap((pfx) => MODULE_KEYS.map((k) => `${pfx}_MODULE_${k}`)));
 
 /**
  * Build the child's environment.
@@ -55,9 +58,12 @@ const MODULES_OFF = Object.freeze([
  * @param {object} [o.env]   the live process's environment (only CARRIED names are read)
  * @param {number|string} o.port   the loopback port the preview listens on
  * @param {string} o.scratchHome   an empty directory to serve as HOME
+ * @param {string} [o.envPrefix]   the env prefix of the copied instance's branding pack, if any
+ * @param {string} [o.instanceRoot]  the scratch directory shaped like an instance root (config/,
+ *                                   migrations/instance/); the child reads its host content from it
  * @returns {Record<string,string>} a fresh object, never `env` itself
  */
-function buildPreviewEnv({ env = process.env, port, scratchHome } = {}) {
+function buildPreviewEnv({ env = process.env, port, scratchHome, instanceRoot, envPrefix } = {}) {
   if (!port) throw new TypeError('buildPreviewEnv: port is required');
   if (!scratchHome) throw new TypeError('buildPreviewEnv: scratchHome is required');
   const out = {};
@@ -68,10 +74,15 @@ function buildPreviewEnv({ env = process.env, port, scratchHome } = {}) {
   out.PORT = String(port);
   out.HOST = '127.0.0.1';
   out.HOME = scratchHome;
+  // Named explicitly: without it the core takes the process's working directory as the instance
+  // root, and the install directory is not one (no config/, so no branding, no module switches).
+  if (instanceRoot) out.BONGOS_INSTANCE_ROOT = instanceRoot;
   out.PGDATABASE = PREVIEW_DB;
   out.CHAT_DRY_RUN = '1';
   for (const name of POLLERS_OFF) out[name] = '1';
   for (const name of MODULES_OFF) out[name] = '0';
+  // The copied instance's own prefix, when it is none of the above.
+  if (/^[A-Z][A-Z0-9_]*$/.test(String(envPrefix || ''))) for (const k of MODULE_KEYS) out[`${envPrefix}_MODULE_${k}`] = '0';
   return out;
 }
 

package/modules/npm-release/preview/runtime.js

@@ -22,6 +22,7 @@ const REAP_EVERY_MS = 60 * 1000;
 const PROGRAMS = Object.freeze({
   npm: (args, o) => spawn('npm', args, o),
   dropdb: (args, o) => spawn('dropdb', args, o),
+  psql: (args, o) => spawn('psql', args, o),
   createdb: (args, o) => spawn('createdb', args, o),
   pg_dump: (args, o) => spawn('pg_dump', args, o),
   pg_restore: (args, o) => spawn('pg_restore', args, o),
@@ -49,27 +50,60 @@ function runCommand({ cmd, args, env, cwd }) {
   });
 }
 
-/** Does something answer /healthz on the loopback port? */
-function probeHealth(port) {
+/** GET one loopback path: { status, body } (body capped), or { error } when nothing answered. */
+function getLoopback(port, urlPath) {
   return new Promise((resolve) => {
-    const req = http.get({ host: '127.0.0.1', port, path: '/healthz', timeout: 2000 }, (res) => {
-      res.resume();
-      resolve(res.statusCode === 200);
+    const req = http.get({ host: '127.0.0.1', port, path: urlPath, timeout: 3000, headers: { accept: 'application/json' } }, (res) => {
+      let body = '';
+      res.on('data', (d) => { if (body.length < 16384) body += d; });
+      res.on('end', () => resolve({ status: res.statusCode, body }));
+      res.on('error', (e) => resolve({ error: e.message }));
     });
-    req.on('error', () => resolve(false));
-    req.on('timeout', () => { req.destroy(); resolve(false); });
+    req.on('error', (e) => resolve({ error: e.message }));
+    req.on('timeout', () => { req.destroy(); resolve({ error: 'timed out' }); });
   });
 }
 
+/**
+ * Can the preview SERVE DATA? Not just "does something answer": /healthz touches no database,
+ * and on the first live run a preview that answered it 200 returned 500 on every page.
+ *
+ *   1. /healthz answers 200 (the process is up).
+ *   2. /version's schemaPending is not null. It is computed from a read of schema_migrations,
+ *      and is null exactly when that read fails — so this proves the copy is reachable. A
+ *      version too old to carry the field at all is passed on to step 3 instead.
+ *   3. a database-backed route is not a server error: /api/bongos/me answers 401 or 200 when
+ *      the database and auth path work, and 500 when they do not.
+ *
+ * Returns { ok, reason }; the reason is shown to the owner, so it names the failing step.
+ */
+async function probeHealth(port, get = getLoopback) {
+  const h = await get(port, '/healthz');
+  if (h.error || h.status !== 200) return { ok: false, reason: h.error ? `/healthz: ${h.error}` : `/healthz answered ${h.status}` };
+  const v = await get(port, '/version');
+  if (v.error || v.status !== 200) return { ok: false, reason: v.error ? `/version: ${v.error}` : `/version answered ${v.status}` };
+  let info = null;
+  try { info = JSON.parse(v.body); } catch { /* not JSON: handled below */ }
+  if (!info || typeof info !== 'object') return { ok: false, reason: '/version did not answer with JSON' };
+  if ('schemaPending' in info && info.schemaPending === null) {
+    return { ok: false, reason: '/version reports schemaPending null: the database copy could not be read' };
+  }
+  const me = await get(port, '/api/bongos/me');
+  if (me.error || me.status >= 500) {
+    return { ok: false, reason: me.error ? `/api/bongos/me: ${me.error}` : `a database-backed read (/api/bongos/me) answered ${me.status}` };
+  }
+  return { ok: true };
+}
+
 /**
  * Build the production supervisor: the port from NPM_RELEASE_PREVIEW_PORT (default 3190,
  * loopback only), installs cached under ~/.cache/bongos-preview, and a timer that stops
  * an idle preview and a hook that stops it when this process does.
  */
-function createRuntime({ liveDb, log, env = process.env } = {}) {
+function createRuntime({ liveDb, log, env = process.env, instanceRoot = null } = {}) {
   const port = Number(env.NPM_RELEASE_PREVIEW_PORT) || DEFAULT_PORT;
   const supervisor = createSupervisor({
-    run: runCommand, spawn, probe: probeHealth, liveDb, log, port, env,
+    run: runCommand, spawn, probe: probeHealth, liveDb, log, port, env, instanceRoot,
     cacheRoot: path.join(os.homedir(), '.cache', 'bongos-preview'),
   });
   const timer = setInterval(() => {

package/modules/npm-release/preview/supervisor.js

@@ -1,29 +1,44 @@
 'use strict';
 
 // modules/npm-release/preview/supervisor.js — starts, watches and stops the ONE version
-// preview this hall may run (task 1004300, ADR 0349).
+// preview this hall may run (task 1004300, ADR 0349; hardened by task 1004465).
 //
 // A preview is a second Bongos process, running an installed copy of an older core against
 // a COPY of the live database, reached through the live hall's own address (divert.js). This
 // file owns its lifecycle and nothing else: install → copy the database → migrate → launch →
-// wait until it answers. Every outward step goes through an injected function (run for a
-// command, spawn for the long-lived child, probe for the health check, sleep, now), which is
+// wait until it can serve data. Every outward step goes through an injected function (run for
+// a command, spawn for the long-lived child, probe for the health check, sleep, now), which is
 // what lets the whole lifecycle be tested without a process, a database or a network.
 //
 // ONE AT A TIME, STOPPED WHEN IDLE. A second preview is refused, not queued; a running one
 // that nothing has touched for IDLE_MS is stopped and its copy of the database dropped. That
 // bound is what keeps a preview from becoming a permanent second tenant of the droplet.
+//
+// THREE THINGS task 1004465 ADDED, each from the first live run:
+//   * the child's output is kept (redacted, capped) and shown when a start fails;
+//   * "running" means the child can serve DATA, not merely that it answers /healthz;
+//   * a stop waits for the child to exit and really drops the copy, or says it could not.
 
 const os = require('node:os');
 const fs = require('node:fs');
 const path = require('node:path');
-const { buildPreviewEnv } = require('./env');
+const { buildPreviewEnv, PREVIEW_DB } = require('./env');
 const cmds = require('./commands');
+const { createChildLog, redact, secretValues } = require('./childlog');
 
 const IDLE_MS = 20 * 60 * 1000;
 const DEFAULT_PORT = 3190;
 const HEALTH_TRIES = 40;
 const HEALTH_INTERVAL_MS = 1500;
+// How long a stopping child gets to exit on SIGTERM before SIGKILL, and how long after that.
+const KILL_WAIT_MS = 5000;
+const KILL_FINAL_WAIT_MS = 2000;
+
+// The ONLY files of the live instance's config/ copied into the preview's scratch instance:
+// the identity and switch packs a hall reads at boot. An allow-list, like the environment
+// (env.js): a deny-list forgets the next credential file someone adds, an allow-list cannot.
+// Anything else in config/ (go-live.json, key files, whatever comes next) stays behind.
+const CONFIG_COPY = /^(branding|modules|government|ideas|design-tokens)(\.neutral)?\.json$/;
 
 // How far behind the running version a preview may reach, counted in PUBLISHED versions.
 const WINDOW = 30;
@@ -52,43 +67,123 @@ function previewable(version, { published, running, window = WINDOW } = {}) {
   return { ok: true };
 }
 
+/**
+ * Copy the live instance's host content the child needs to boot as an instance rather than
+ * as a bare install: config/ (branding, module switches, government) and migrations/instance/.
+ * A source that is absent is skipped; any other failure fails the start. Only the allow-listed
+ * pack files of config/ are copied (CONFIG_COPY).
+ */
+function copyInstance({ fsImpl, from, to }) {
+  const parts = [
+    // The directory itself always passes the filter; files are copied only when allow-listed.
+    ['config', (src) => path.basename(src) === 'config' || CONFIG_COPY.test(path.basename(src))],
+    [path.join('migrations', 'instance'), () => true],
+  ];
+  for (const [rel, filter] of parts) {
+    const src = path.join(from, rel);
+    try {
+      if (!fsImpl.existsSync(src)) continue;
+      fsImpl.mkdirSync(path.dirname(path.join(to, rel)), { recursive: true });
+      fsImpl.cpSync(src, path.join(to, rel), { recursive: true, filter });
+    } catch (e) {
+      throw new Error(`could not copy the instance ${rel} into the preview: ${e && e.message}`);
+    }
+  }
+}
+
 /**
  * @param {object} o
  * @param {Function} o.run    (command) => Promise of { code, stderr } — runs one command to the end
  * @param {Function} o.spawn  node:child_process spawn (the long-lived child)
- * @param {Function} o.probe  (port) => Promise of boolean — does the preview answer its health check
+ * @param {Function} o.probe  (port) => Promise of boolean | { ok, reason } — can the preview serve data
  * @param {string} o.liveDb   the live database name (the source of the copy)
  * @param {string} o.cacheRoot  where versions are installed
+ * @param {string} [o.instanceRoot]  the live instance's root; its config/ and migrations/instance/
+ *                           are copied into the preview's scratch instance directory
  * @param {object} [o.env]    the live environment (read by buildPreviewEnv only)
  */
 function createSupervisor({
-  run, spawn, probe, liveDb, cacheRoot,
-  env = process.env, port = DEFAULT_PORT,
+  run, spawn, probe, liveDb, cacheRoot, instanceRoot = null,
+  env = process.env, port = DEFAULT_PORT, killWaitMs = KILL_WAIT_MS,
   now = Date.now, sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
   fsImpl = fs, tmpdir = os.tmpdir(), idleMs = IDLE_MS,
   log = { info() {}, warn() {}, error() {} },
+  copy = copyInstance,
 } = {}) {
+  // The one database a drop may ever name is the fixed copy. If the live database WERE that
+  // name, every stop would drop the live hall's data, so refuse to be built at all.
+  if (liveDb === PREVIEW_DB) throw new Error(`the live database cannot be named ${PREVIEW_DB}: that name is the preview's copy and is dropped on every stop`);
+
   let state = { state: 'idle', version: null, stage: null, error: null, startedAt: null, lastTouch: null };
   let child = null;
   let workDir = null;
+  let childLog = null;     // the running or just-stopped child's output
+  let keptTail = [];       // its tail, kept after the scratch directory is gone
+  let secrets = secretValues(env);
+  let exitInfo = null;     // { code } once the child has exited
+  let exited = Promise.resolve();
 
   const set = (patch) => { state = { ...state, ...patch }; };
-  const status = () => ({ ...state, port, idle_ms: idleMs });
+  const status = () => ({ ...state, port, idle_ms: idleMs, log_tail: childLog ? childLog.tail() : keptTail });
 
   async function step(stage, command) {
     set({ stage });
     const r = await run(command);
     if (!r || r.code !== 0) {
-      const tail = r && r.stderr ? `: ${String(r.stderr).trim().slice(-300)}` : '';
+      const tail = r && r.stderr ? `: ${redact(String(r.stderr).trim().slice(-300), secrets)}` : '';
       throw new Error(`${stage} failed (exit ${r ? r.code : '?'})${tail}`);
     }
   }
 
+  // Run a command only if it can touch nothing but the copy. Never throws for a failed command.
+  async function runDrop(command) {
+    if (!cmds.targetsOnlyPreviewDb(command)) throw new Error(`refusing a database command that does not name only ${PREVIEW_DB}`);
+    try { return (await run(command)) || { code: -1, stderr: 'no result' }; } catch (e) { return { code: -1, stderr: e && e.message }; }
+  }
+
+  /**
+   * Drop the copy, really. dropdb --force first (it ends stray connections itself); if that
+   * is refused, end the copy's connections — the copy's alone — and drop plainly. Returns
+   * null when the copy is gone, else a sentence saying it is not.
+   */
+  async function dropCopy() {
+    const first = await runDrop(cmds.dropDbCommand());
+    if (first.code === 0) return null;
+    await runDrop(cmds.terminateBackendsCommand());
+    const second = await runDrop(cmds.dropDbPlainCommand());
+    if (second.code === 0) return null;
+    const why = redact(String(second.stderr || first.stderr || '').trim().slice(-200), secrets);
+    return `the copy database ${PREVIEW_DB} could not be dropped (exit ${second.code})${why ? `: ${why}` : ''}`;
+  }
+
+  // Ask the child to stop and WAIT for it, then insist. The drop that follows needs the
+  // child's database connections gone; killing without waiting is what left the copy behind.
+  async function stopChild(c) {
+    if (exitInfo) return;
+    try { c.kill('SIGTERM'); } catch (e) { log.warn(`preview child would not stop: ${e && e.message}`); }
+    await Promise.race([exited, sleep(killWaitMs)]);
+    if (exitInfo) return;
+    try { c.kill('SIGKILL'); } catch { /* already gone */ }
+    await Promise.race([exited, sleep(KILL_FINAL_WAIT_MS)]);
+  }
+
   // Everything a start or a stop leaves behind: the child, the copy, the scratch directory.
+  // Returns null when all of it is gone, else the sentence saying the copy is not.
   async function cleanup() {
-    if (child) { try { child.kill(); } catch (e) { log.warn(`preview child would not stop: ${e && e.message}`); } child = null; }
-    try { await run(cmds.dropDbCommand()); } catch (e) { log.warn(`preview copy could not be dropped: ${e && e.message}`); }
+    const c = child;
+    child = null;
+    if (c) await stopChild(c);
+    if (childLog) { keptTail = childLog.tail(); childLog.close(); childLog = null; }
+    const dropError = await dropCopy();
+    if (dropError) log.error(`preview: ${dropError}`);
     if (workDir) { try { fsImpl.rmSync(workDir, { recursive: true, force: true }); } catch { /* best effort */ } workDir = null; }
+    return dropError;
+  }
+
+  // A few of the child's last lines, to sit in the error message itself.
+  function withTail(message, lg) {
+    const lines = lg ? lg.tail(5) : [];
+    return lines.length ? `${message}. Its last output: ${lines.join(' | ')}`.slice(0, 900) : message;
   }
 
   async function launch(version) {
@@ -97,41 +192,78 @@ function createSupervisor({
       const home = path.join(workDir, 'home');
       fsImpl.mkdirSync(home, { recursive: true });
       const dumpFile = path.join(workDir, 'live.dump');
-      const childEnv = buildPreviewEnv({ env, port, scratchHome: home });
+
+      // The child's working directory is a scratch INSTANCE: the live instance's config/ and
+      // migrations/instance/, copied. Run from the bare install directory it had no config/
+      // at all (no branding, no module switches) and was not shaped like a hall.
+      let instanceDir;
+      let prefix;
+      if (instanceRoot) {
+        instanceDir = path.join(workDir, 'instance');
+        fsImpl.mkdirSync(instanceDir, { recursive: true });
+        copy({ fsImpl, from: instanceRoot, to: instanceDir });
+        try { prefix = JSON.parse(fsImpl.readFileSync(path.join(instanceDir, 'config', 'branding.json'), 'utf8')).envPrefix; } catch { /* no branding pack: the default prefix */ }
+      }
+      const childEnv = buildPreviewEnv({ env, port, scratchHome: home, instanceRoot: instanceDir, envPrefix: prefix });
+      secrets = secretValues(env, childEnv);
+      keptTail = [];
+      childLog = createChildLog({ secrets, fsImpl, file: path.join(workDir, 'preview.log') });
+      exitInfo = null;
 
       await step('install', { ...cmds.installCommand({ cacheRoot, version }), env: childEnv });
-      await step('drop-old-copy', cmds.dropDbCommand());
+      set({ stage: 'drop-old-copy' });
+      const leftover = await dropCopy();
+      if (leftover) throw new Error(leftover);
       await step('create-copy', cmds.createDbCommand());
       await step('dump', cmds.dumpCommand({ liveDb, dumpFile }));
       await step('restore', cmds.restoreCommand({ dumpFile }));
-      await step('migrate', cmds.migrateCommand({ cacheRoot, version, env: childEnv }));
+      await step('migrate', cmds.migrateCommand({ cacheRoot, version, env: childEnv, instanceDir }));
 
       set({ stage: 'launch' });
-      child = spawn(process.execPath, [cmds.entryPath(cacheRoot, version)], {
-        cwd: cmds.versionDir(cacheRoot, version), env: childEnv, stdio: 'ignore',
+      // The child's output is kept (redacted, capped): its errors used to go nowhere.
+      const mine = childLog;
+      const spawned = spawn(process.execPath, [cmds.entryPath(cacheRoot, version)], {
+        cwd: instanceDir || cmds.versionDir(cacheRoot, version), env: childEnv, stdio: ['ignore', 'pipe', 'pipe'],
       });
-      child.on('exit', (code) => {
-        // A child that dies on its own must not leave a copy behind or a page waiting on it.
-        if (child && state.state === 'running') {
-          log.warn(`preview of ${version} exited (${code})`);
-          child = null;
-          set({ state: 'failed', stage: 'exited', error: `the preview process exited (${code})` });
-          cleanup();
-        }
+      child = spawned;
+      exited = new Promise((resolve) => {
+        spawned.on('exit', (code) => {
+          exitInfo = { code };
+          resolve();
+          // A child that dies on its own must not leave a copy behind or a page waiting on it.
+          if (child === spawned && state.state === 'running') {
+            log.warn(`preview of ${version} exited (${code})`);
+            set({ state: 'failed', stage: 'exited', error: withTail(`the preview process exited (${code})`, mine) });
+            cleanup().then((err) => { if (err) set({ error: `${state.error}; ${err}` }); });
+          }
+        });
       });
+      for (const stream of [spawned.stdout, spawned.stderr]) {
+        if (stream && typeof stream.on === 'function') stream.on('data', (d) => mine.append(String(d)));
+      }
 
       set({ stage: 'health' });
+      // READY MEANS IT CAN SERVE DATA. The probe must see the database and the auth path work,
+      // not just the process answer (probeHealth in runtime.js); a preview that cannot is
+      // failed here, with why, rather than reported running.
       let up = false;
+      let why = 'it did not answer';
       for (let i = 0; i < HEALTH_TRIES && !up; i++) {
-        up = await probe(port);
-        if (!up) await sleep(HEALTH_INTERVAL_MS);
+        if (exitInfo) throw new Error(withTail(`the preview process exited (${exitInfo.code}) before it could serve data`, mine));
+        const r = await probe(port);
+        up = r === true || !!(r && r.ok);
+        if (!up) {
+          if (r && r.reason) why = r.reason;
+          await sleep(HEALTH_INTERVAL_MS);
+        }
       }
-      if (!up) throw new Error('the preview did not answer in time');
+      if (!up) throw new Error(withTail(`the preview started but could not serve data (${why})`, mine));
       set({ state: 'running', stage: null, startedAt: now(), lastTouch: now() });
     } catch (e) {
       log.error(`preview of ${version} failed: ${e && e.message}`);
-      set({ state: 'failed', error: e && e.message ? e.message : String(e) });
-      await cleanup();
+      set({ state: 'failed', error: redact(e && e.message ? e.message : String(e), secrets) });
+      const dropError = await cleanup();
+      if (dropError) set({ error: `${state.error}; ${dropError}` });
     }
   }
 
@@ -146,13 +278,20 @@ function createSupervisor({
       if (state.version === version) return { ok: true, already: true, status: status(), done: Promise.resolve() };
       return { ok: false, code: 'preview_busy', status: status() };
     }
+    keptTail = [];
     set({ state: 'starting', version, stage: 'queued', error: null, startedAt: null, lastTouch: now() });
     return { ok: true, status: status(), done: launch(version) };
   }
 
   /** Stop the preview and drop its copy. Safe to call at any point. */
   async function stop() {
-    await cleanup();
+    const dropError = await cleanup();
+    // A copy that is still there is NOT idle: say so, where the deploy page will show it.
+    if (dropError) {
+      set({ state: 'failed', stage: 'drop', error: dropError, startedAt: null, lastTouch: null });
+      return status();
+    }
+    keptTail = [];
     set({ state: 'idle', version: null, stage: null, error: null, startedAt: null, lastTouch: null });
     return status();
   }
@@ -172,4 +311,4 @@ function createSupervisor({
   return { start, stop, status, touch, reapIdle };
 }
 
-module.exports = { createSupervisor, previewable, IDLE_MS, DEFAULT_PORT, WINDOW };
+module.exports = { createSupervisor, previewable, copyInstance, IDLE_MS, DEFAULT_PORT, WINDOW };

package/modules/npm-release/public/work.css

@@ -6,3 +6,6 @@
 /* A stage's sentence sits under its head (work.js says why) and needs air before the first
    row, which drops its own top rule and padding in oversight.css. */
 #nr-work .ov-group > .ov-row__sub { margin: 8px 0 6px; }
+
+/* the preview process output shown under a failed preview (task 1004465) */
+.ov-log { max-height: 16rem; overflow: auto; margin: 0.5rem 0 0; padding: 0.5rem 0.75rem; font: 12px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace; white-space: pre-wrap; word-break: break-word; background: rgba(127, 127, 127, 0.12); border-radius: 6px; }

package/modules/npm-release/public/work.js

@@ -192,7 +192,12 @@
       return `<div class="ov-note"><p><strong>Preparing a preview of ${v}</strong> — ${escapeHtml(PREVIEW_STEP[preview.stage] || 'working')}. This can take a few minutes.</p></div>`;
     }
     if (preview.state === 'failed') {
-      return `<div class="ov-note ov-note--warn"><p><strong>The preview of ${v} did not start.</strong> ${escapeHtml(preview.error || '')}</p><button type="button" class="ov-toggle" data-preview-stop>Clear</button></div>`;
+      // What the preview process itself printed (secrets already removed on the server), so a
+      // failed preview can be diagnosed here without a shell on the box (task 1004465).
+      const lines = Array.isArray(preview.log_tail) ? preview.log_tail : [];
+      const output = lines.length ? `<details open><summary>What the preview printed</summary><pre class="ov-log">${escapeHtml(lines.join('\n'))}</pre></details>` : '';
+      const headline = preview.stage === 'drop' ? `The copy of the data for ${v} could not be removed.` : `The preview of ${v} did not start.`;
+      return `<div class="ov-note ov-note--warn"><p><strong>${headline}</strong> ${escapeHtml(preview.error || '')}</p>${output}<button type="button" class="ov-toggle" data-preview-stop>${preview.stage === 'drop' ? 'Try again' : 'Clear'}</button></div>`;
     }
     return `<div class="ov-note"><p><strong>Preview of ${v} is running</strong> on a copy of this hall's data. Nothing done there reaches the live hall. It stops by itself after ${Math.round((preview.idle_ms || 0) / 60000)} minutes unused.</p>
       <a class="ov-toggle" href="${API}/npm-release/preview/enter?v=${encodeURIComponent(preview.version)}">Open the preview</a>

package/modules/npm-release/routes/preview.js

@@ -55,6 +55,7 @@ module.exports = function buildNpmReleasePreviewRouter({
   const router = express.Router();
   const sup = supervisor || lazySupervisor(() => require('../preview/runtime').createRuntime({
     liveDb: process.env.PGDATABASE || api.instanceDbName(),
+    instanceRoot: api.resolveInstanceRoot(),
     log,
   }));
   const mayPreview = api.requirePermission('core.pin.move');