@bongos/core 1.20.38 → 1.20.40

package/modules/npm-release/preview/childlog.js

@@ -0,0 +1,95 @@
+'use strict';
+
+// modules/npm-release/preview/childlog.js — what the preview child printed, kept small and
+// with secrets taken out (task 1004465).
+//
+// WHY. The child used to be spawned with stdio 'ignore', so when a preview started but could
+// not serve a single page, its own error went nowhere and the only way to find out was ssh.
+// This keeps the last CAP bytes of its stdout and stderr, writes them to a file in the
+// preview's scratch directory, and hands the deploy page a short redacted tail.
+//
+// REDACTION IS BY VALUE AND BY SHAPE. Every value in the environment the child was given
+// (and the live one) whose name looks secret is replaced wherever it appears; then the common
+// shapes (a password in a connection URL, `token=...`, a bearer header, a GitHub or API key)
+// are replaced even if nobody listed them. The tail never reaches a caller unredacted: it is
+// redacted at read time as well as at write time, so a secret that arrives split across two
+// chunks is still caught.
+
+const CAP = 64 * 1024;
+// The file is rewritten at most this often, asynchronously: the child's output is handled
+// inside the LIVE hall's process, and a chatty or crash-looping child must never turn into a
+// run of blocking disk writes on its event loop.
+const FLUSH_MS = 500;
+const TAIL_LINES = 20;
+const MAX_LINE = 300;
+const SECRET_NAME = /(SECRET|TOKEN|PASSWORD|PASSWD|PASS|KEY|CREDENTIAL|COOKIE|AUTH)/i;
+
+const SHAPES = [
+  [/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]*:)[^\s@/]+@/gi, '$1[redacted]@'],
+  [/\b((?:api[_-]?key|token|secret|password|passwd|authorization|cookie|signing[_-]?key)\w*\s*[=:]\s*)("[^"]*"|'[^']*'|\S+)/gi, '$1[redacted]'],
+  [/\bBearer\s+[A-Za-z0-9._~+/=-]+/g, 'Bearer [redacted]'],
+  [/\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|sk-[A-Za-z0-9_-]{16,}|xox[baprs]-[A-Za-z0-9-]{10,})/g, '[redacted]'],
+];
+
+/** The values in these environments that look like secrets and are long enough to matter. */
+function secretValues(...envs) {
+  const out = new Set();
+  for (const env of envs) {
+    for (const [k, v] of Object.entries(env || {})) {
+      if (SECRET_NAME.test(k) && typeof v === 'string' && v.length >= 6) out.add(v);
+    }
+  }
+  // Longest first, so a value that contains another is replaced whole.
+  return [...out].sort((a, b) => b.length - a.length);
+}
+
+/** Take the secrets out of one piece of text. Pure. */
+function redact(text, secrets = []) {
+  let s = String(text);
+  for (const v of secrets) s = s.split(v).join('[redacted]');
+  for (const [re, to] of SHAPES) s = s.replace(re, to);
+  return s;
+}
+
+/**
+ * @param {object} [o]
+ * @param {string[]} [o.secrets]  exact values to take out (from secretValues)
+ * @param {object} [o.fsImpl]     fs (writeFile, the callback form) — optional, with `file`
+ * @param {string} [o.file]       where to keep the capped raw-but-redacted log
+ * @param {number} [o.cap]
+ */
+function createChildLog({ secrets = [], fsImpl = null, file = null, cap = CAP, flushMs = FLUSH_MS } = {}) {
+  let buf = '';
+  let timer = null;
+  let closed = false;
+  const canWrite = () => !!(fsImpl && file && typeof fsImpl.writeFile === 'function');
+  function flush() {
+    timer = null;
+    if (closed || !canWrite()) return;
+    try { fsImpl.writeFile(file, buf, () => { /* best effort */ }); } catch { /* best effort */ }
+  }
+  return {
+    /** Add output. Never throws: a log that cannot be kept must not fail a start. */
+    append(chunk) {
+      try {
+        buf = (buf + redact(chunk, secrets)).slice(-cap);
+        if (canWrite() && !timer && !closed) {
+          timer = setTimeout(flush, flushMs);
+          if (timer && typeof timer.unref === 'function') timer.unref();
+        }
+      } catch { /* best effort */ }
+    },
+    /** Write the file now (tests; the timer does this in production). */
+    flush,
+    /** Stop writing: the scratch directory is about to be removed. */
+    close() { closed = true; if (timer) { clearTimeout(timer); timer = null; } },
+    /** The last `n` non-empty lines, each shortened. */
+    tail(n = TAIL_LINES) {
+      return redact(buf, secrets).split(/\r?\n/).filter((l) => l.trim())
+        .slice(-n).map((l) => (l.length > MAX_LINE ? `${l.slice(0, MAX_LINE)}…` : l));
+    },
+    size: () => buf.length,
+  };
+}
+
+module.exports = { createChildLog, redact, secretValues };

package/modules/npm-release/preview/commands.js

@@ -36,11 +36,49 @@ function installCommand({ cacheRoot, version }) {
   return { cmd: 'npm', args: ['install', '--prefix', versionDir(cacheRoot, version), `${PACKAGE}@${version}`, '--omit=dev'] };
 }
 
-/** Drop the copy if one is left over — used before a start and on every stop. */
+/**
+ * Drop the copy if one is left over — used before a start and on every stop. --force ends any
+ * connection still open on the copy (Postgres 13+), which is exactly the case that used to
+ * leave it behind: a child that had not finished exiting. The name is the fixed PREVIEW_DB
+ * and nothing is interpolated, so this can never name the live database.
+ */
 function dropDbCommand() {
+  return { cmd: 'dropdb', args: ['--if-exists', '--force', PREVIEW_DB] };
+}
+
+/** Plain drop, for a server whose dropdb has no --force (Postgres 12 and older). */
+function dropDbPlainCommand() {
   return { cmd: 'dropdb', args: ['--if-exists', PREVIEW_DB] };
 }
 
+/**
+ * End every connection to the copy and ONLY the copy: the database name is the literal
+ * PREVIEW_DB, never a parameter. Run on the maintenance database, since the copy cannot be
+ * connected to while it is being emptied.
+ */
+function terminateBackendsCommand() {
+  return {
+    cmd: 'psql',
+    args: ['-X', '-d', 'postgres', '-tAc',
+      `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${PREVIEW_DB}' AND pid <> pg_backend_pid()`],
+  };
+}
+
+/**
+ * Can this command only ever touch the copy? True for the drop commands above and nothing
+ * else. The supervisor checks it before running any drop, so a future edit that pointed one
+ * at another database is refused at run time, not discovered on the droplet.
+ */
+function targetsOnlyPreviewDb(command) {
+  if (!command || !Array.isArray(command.args)) return false;
+  if (command.cmd === 'dropdb') {
+    const last = command.args[command.args.length - 1];
+    return last === PREVIEW_DB && command.args.slice(0, -1).every((a) => a.startsWith('--'));
+  }
+  if (command.cmd === 'psql') return JSON.stringify(command.args) === JSON.stringify(terminateBackendsCommand().args);
+  return false;
+}
+
 /** An empty database for the copy. Deliberately no template (see the header). */
 function createDbCommand() {
   return { cmd: 'createdb', args: [PREVIEW_DB] };
@@ -65,17 +103,20 @@ function restoreCommand({ dumpFile }) {
  * DATABASE_URL is blanked because the pool prefers it to PGDATABASE, and PGDATABASE is
  * pinned to the copy so nothing can be applied to the live database by mistake.
  */
-function migrateCommand({ cacheRoot, version, env }) {
+function migrateCommand({ cacheRoot, version, env, instanceDir }) {
   const dir = versionDir(cacheRoot, version);
+  // With a scratch instance directory, THAT is the instance root (its config/ and
+  // migrations/instance/ are copies of the live instance's); without one, the install directory.
+  const root = instanceDir || dir;
   return {
     cmd: 'bash',
     args: [path.join(dir, 'node_modules', '@bongos', 'core', 'scripts', 'migrate.sh')],
-    cwd: dir,
-    env: { ...env, INIT_CWD: dir, PGDATABASE: PREVIEW_DB, DATABASE_URL: '' },
+    cwd: root,
+    env: { ...env, INIT_CWD: root, PGDATABASE: PREVIEW_DB, DATABASE_URL: '' },
   };
 }
 
 module.exports = {
   VERSION_RE, PACKAGE, versionDir, entryPath,
-  installCommand, dropDbCommand, createDbCommand, dumpCommand, restoreCommand, migrateCommand,
+  installCommand, dropDbCommand, dropDbPlainCommand, terminateBackendsCommand, targetsOnlyPreviewDb, createDbCommand, dumpCommand, restoreCommand, migrateCommand,
 };

package/modules/npm-release/preview/divert.js

@@ -44,9 +44,19 @@ function pathOf(req) {
   return q === -1 ? url : url.slice(0, q);
 }
 
-/** The Set-Cookie value that removes the steering cookie. */
-function clearCookie() {
-  return `${COOKIE_NAME}=; Path=/; Max-Age=0; HttpOnly; SameSite=Lax`;
+/** Path/HttpOnly/SameSite/Secure the steering cookie is set with; the clear repeats them. */
+function steerFlags(req) {
+  return `HttpOnly; SameSite=Lax${req && req.secure ? '; Secure' : ''}`;
+}
+
+/** The Set-Cookie value that sets the steering cookie. */
+function setCookie(version, req) {
+  return `${COOKIE_NAME}=${version}; Path=/; ${steerFlags(req)}`;
+}
+
+/** The Set-Cookie value that removes it: same Path, SameSite and Secure as the set (task 1004466). */
+function clearCookie(req) {
+  return `${COOKIE_NAME}=; Path=/; Max-Age=0; ${steerFlags(req)}`;
 }
 
 // Run an express-style gate to its verdict without letting it write a response of its own:
@@ -77,14 +87,14 @@ function createDivert({ supervisor, forward, requireBuilder, requirePermission }
     const st = supervisor.status();
     if (st.state !== 'running' || st.version !== wanted) {
       // The preview this cookie pointed at is gone: drop the cookie and serve the live hall.
-      res.append('Set-Cookie', clearCookie());
+      res.append('Set-Cookie', clearCookie(req));
       return next();
     }
     const p = pathOf(req);
     if (AUTH_PATH.test(p) || PREVIEW_PATH.test(p)) return next();
     if (req.headers.authorization) return next();
     if (!(await passes(requireBuilder, req)) || !(await passes(mayPreview, req))) {
-      res.append('Set-Cookie', clearCookie());
+      res.append('Set-Cookie', clearCookie(req));
       return next();
     }
     supervisor.touch();
@@ -92,4 +102,4 @@ function createDivert({ supervisor, forward, requireBuilder, requirePermission }
   };
 }
 
-module.exports = { createDivert, cookieVersion, clearCookie, AUTH_PATH, PREVIEW_PATH };
+module.exports = { createDivert, cookieVersion, clearCookie, setCookie, AUTH_PATH, PREVIEW_PATH };

package/modules/npm-release/preview/env.js

@@ -41,12 +41,15 @@ const POLLERS_OFF = Object.freeze([
 
 // Modules that talk to the outside world, off. npm-release is off too: a preview must not
 // offer previews of its own.
-const MODULES_OFF = Object.freeze([
-  'BONGOS_MODULE_DISCORD',
-  'BONGOS_MODULE_AGENTS',
-  'BONGOS_MODULE_PROVISIONING',
-  'BONGOS_MODULE_NPM_RELEASE',
-]);
+//
+// EVERY SPELLING OF THE PREFIX. The core reads <PREFIX>_MODULE_<KEY> with the PREFIX the
+// instance's branding pack names, then the legacy ones. The child now runs with a copy of the
+// live instance's config/ (task 1004465), so its prefix may be the live instance's own, and a
+// switch spelled only BONGOS_ would then be ignored — Discord and the agents would come back
+// on in the copy. So each switch is set under every prefix the core can resolve.
+const MODULE_KEYS = Object.freeze(['DISCORD', 'AGENTS', 'PROVISIONING', 'NPM_RELEASE']);
+const PREFIXES = Object.freeze(['BONGOS', 'CLOUDBONGOS', 'OTB', 'GDS', 'PMS']);
+const MODULES_OFF = Object.freeze(PREFIXES.flatMap((pfx) => MODULE_KEYS.map((k) => `${pfx}_MODULE_${k}`)));
 
 /**
  * Build the child's environment.
@@ -55,9 +58,12 @@ const MODULES_OFF = Object.freeze([
  * @param {object} [o.env]   the live process's environment (only CARRIED names are read)
  * @param {number|string} o.port   the loopback port the preview listens on
  * @param {string} o.scratchHome   an empty directory to serve as HOME
+ * @param {string} [o.envPrefix]   the env prefix of the copied instance's branding pack, if any
+ * @param {string} [o.instanceRoot]  the scratch directory shaped like an instance root (config/,
+ *                                   migrations/instance/); the child reads its host content from it
  * @returns {Record<string,string>} a fresh object, never `env` itself
  */
-function buildPreviewEnv({ env = process.env, port, scratchHome } = {}) {
+function buildPreviewEnv({ env = process.env, port, scratchHome, instanceRoot, envPrefix } = {}) {
   if (!port) throw new TypeError('buildPreviewEnv: port is required');
   if (!scratchHome) throw new TypeError('buildPreviewEnv: scratchHome is required');
   const out = {};
@@ -68,10 +74,15 @@ function buildPreviewEnv({ env = process.env, port, scratchHome } = {}) {
   out.PORT = String(port);
   out.HOST = '127.0.0.1';
   out.HOME = scratchHome;
+  // Named explicitly: without it the core takes the process's working directory as the instance
+  // root, and the install directory is not one (no config/, so no branding, no module switches).
+  if (instanceRoot) out.BONGOS_INSTANCE_ROOT = instanceRoot;
   out.PGDATABASE = PREVIEW_DB;
   out.CHAT_DRY_RUN = '1';
   for (const name of POLLERS_OFF) out[name] = '1';
   for (const name of MODULES_OFF) out[name] = '0';
+  // The copied instance's own prefix, when it is none of the above.
+  if (/^[A-Z][A-Z0-9_]*$/.test(String(envPrefix || ''))) for (const k of MODULE_KEYS) out[`${envPrefix}_MODULE_${k}`] = '0';
   return out;
 }
 

package/modules/npm-release/preview/proxy.js

@@ -74,7 +74,10 @@ function createProxy({ port, banner, exitHref, httpImpl = http } = {}) {
       { host: '127.0.0.1', port: port(), method: req.method, path: req.originalUrl || req.url, headers },
       (upRes) => {
         const out = {};
-        for (const [k, v] of Object.entries(upRes.headers)) if (!HOP.has(k)) out[k] = v;
+        // SET-COOKIE NEVER CROSSES (task 1004466). The preview shares the live hall's origin,
+        // so a cookie it set would overwrite the live hall's own (cb_stealth, the session,
+        // the steering cookie) and lock its owner out. Its cookies stay with the child.
+        for (const [k, v] of Object.entries(upRes.headers)) if (!HOP.has(k) && k.toLowerCase() !== 'set-cookie') out[k] = v;
         const isPage = /^text\/html\b/i.test(String(upRes.headers['content-type'] || ''));
         if (!isPage) {
           res.writeHead(upRes.statusCode, out);

package/modules/npm-release/preview/runtime.js

@@ -22,6 +22,7 @@ const REAP_EVERY_MS = 60 * 1000;
 const PROGRAMS = Object.freeze({
   npm: (args, o) => spawn('npm', args, o),
   dropdb: (args, o) => spawn('dropdb', args, o),
+  psql: (args, o) => spawn('psql', args, o),
   createdb: (args, o) => spawn('createdb', args, o),
   pg_dump: (args, o) => spawn('pg_dump', args, o),
   pg_restore: (args, o) => spawn('pg_restore', args, o),
@@ -49,27 +50,60 @@ function runCommand({ cmd, args, env, cwd }) {
   });
 }
 
-/** Does something answer /healthz on the loopback port? */
-function probeHealth(port) {
+/** GET one loopback path: { status, body } (body capped), or { error } when nothing answered. */
+function getLoopback(port, urlPath) {
   return new Promise((resolve) => {
-    const req = http.get({ host: '127.0.0.1', port, path: '/healthz', timeout: 2000 }, (res) => {
-      res.resume();
-      resolve(res.statusCode === 200);
+    const req = http.get({ host: '127.0.0.1', port, path: urlPath, timeout: 3000, headers: { accept: 'application/json' } }, (res) => {
+      let body = '';
+      res.on('data', (d) => { if (body.length < 16384) body += d; });
+      res.on('end', () => resolve({ status: res.statusCode, body }));
+      res.on('error', (e) => resolve({ error: e.message }));
     });
-    req.on('error', () => resolve(false));
-    req.on('timeout', () => { req.destroy(); resolve(false); });
+    req.on('error', (e) => resolve({ error: e.message }));
+    req.on('timeout', () => { req.destroy(); resolve({ error: 'timed out' }); });
   });
 }
 
+/**
+ * Can the preview SERVE DATA? Not just "does something answer": /healthz touches no database,
+ * and on the first live run a preview that answered it 200 returned 500 on every page.
+ *
+ *   1. /healthz answers 200 (the process is up).
+ *   2. /version's schemaPending is not null. It is computed from a read of schema_migrations,
+ *      and is null exactly when that read fails — so this proves the copy is reachable. A
+ *      version too old to carry the field at all is passed on to step 3 instead.
+ *   3. a database-backed route is not a server error: /api/bongos/me answers 401 or 200 when
+ *      the database and auth path work, and 500 when they do not.
+ *
+ * Returns { ok, reason }; the reason is shown to the owner, so it names the failing step.
+ */
+async function probeHealth(port, get = getLoopback) {
+  const h = await get(port, '/healthz');
+  if (h.error || h.status !== 200) return { ok: false, reason: h.error ? `/healthz: ${h.error}` : `/healthz answered ${h.status}` };
+  const v = await get(port, '/version');
+  if (v.error || v.status !== 200) return { ok: false, reason: v.error ? `/version: ${v.error}` : `/version answered ${v.status}` };
+  let info = null;
+  try { info = JSON.parse(v.body); } catch { /* not JSON: handled below */ }
+  if (!info || typeof info !== 'object') return { ok: false, reason: '/version did not answer with JSON' };
+  if ('schemaPending' in info && info.schemaPending === null) {
+    return { ok: false, reason: '/version reports schemaPending null: the database copy could not be read' };
+  }
+  const me = await get(port, '/api/bongos/me');
+  if (me.error || me.status >= 500) {
+    return { ok: false, reason: me.error ? `/api/bongos/me: ${me.error}` : `a database-backed read (/api/bongos/me) answered ${me.status}` };
+  }
+  return { ok: true };
+}
+
 /**
  * Build the production supervisor: the port from NPM_RELEASE_PREVIEW_PORT (default 3190,
  * loopback only), installs cached under ~/.cache/bongos-preview, and a timer that stops
  * an idle preview and a hook that stops it when this process does.
  */
-function createRuntime({ liveDb, log, env = process.env } = {}) {
+function createRuntime({ liveDb, log, env = process.env, instanceRoot = null } = {}) {
   const port = Number(env.NPM_RELEASE_PREVIEW_PORT) || DEFAULT_PORT;
   const supervisor = createSupervisor({
-    run: runCommand, spawn, probe: probeHealth, liveDb, log, port, env,
+    run: runCommand, spawn, probe: probeHealth, liveDb, log, port, env, instanceRoot,
     cacheRoot: path.join(os.homedir(), '.cache', 'bongos-preview'),
   });
   const timer = setInterval(() => {