memhouse 0.18.12 → 0.20.1

package/memhouse/house/schema-choice.js

@@ -0,0 +1,121 @@
+'use strict';
+// Which schema an install joins (memhouse 0.20). A NEW house is schema 3; an existing house is
+// followed — detected as the connecting credential sees it — never overridden by the default, and
+// an ambiguous one is refused with what to do. bin/memhouse.js `install` (and `onboard` and
+// `deploy --local`, which install) asks here before it writes anything.
+
+const sqlStr = (s) => `'${String(s).replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+
+// ClickHouse's AUTHENTICATION_FAILED, and REQUIRED_PASSWORD (a user that needs a password was
+// given none). Told by the code, which the HTTP wrappers carry as `chCode` from the
+// X-ClickHouse-Exception-Code header (bin/memhouse.js ch(), q-cli.js httpTransport), never by its
+// prose: a credential the server refused is not a house whose tables are unreadable.
+const AUTHENTICATION_FAILED = 516;
+const REQUIRED_PASSWORD = 194;
+const AUTH_CODES = new Set([AUTHENTICATION_FAILED, REQUIRED_PASSWORD]);
+function isAuthFailure(e) { return AUTH_CODES.has(Number(e && e.chCode)); }
+/** What every caller says when it is one; the caller adds what to set. */
+function authFailureText(user, url) {
+  return `could not authenticate${user ? ` as '${user}'` : ''}${url ? ` at ${url}` : ''}`;
+}
+
+/**
+ * Which schema an install joins (0.20: schema 3 for a NEW house; an existing house is followed,
+ * never overridden by the default). Schema 3 is harness-sessions-sql's profile (<m>_session_parts); a
+ * member still on the first schema-3 layout of the 0.20 development builds (<m>_turns) is named, so
+ * the install can refuse rather than lay the profile over it. Read as the connecting credential:
+ *   member  — this member's own tables: '3' (<m>_session_parts), '2' (<m>_messages), 'both', or null;
+ *   devLayout — <m>_turns: the development builds' layout, which the profile replaces;
+ *   devImports — on that layout, how many live sessions `memhouse import` wrote into <m>_turns (null: unreadable);
+ *   unreadable — the house's tables could not be listed: callers that guard a move refuse (fail closed).
+ *   authFailed — and that was the server refusing the credential (isAuthFailure): say so, not "could not list";
+ *   fenced  — its meta says min_writer_schema 3 (memhouse migrate --v3-prepare ran);
+ *   converted — its meta holds profile_converted (memhouse migrate --v3-convert ran to the end);
+ *   house   — the other tables this credential can see: '3', '2', 'mixed', or null (a new house).
+ */
+async function detectHouseSchema(rows, db, member) {
+  const out = { member: null, fenced: false, converted: false, house: null, devLayout: false, devImports: 0, unreadable: false, authFailed: false };
+  let names = [];
+  try { names = (await rows(`SELECT name FROM system.tables WHERE database = ${sqlStr(db)}`)).map((r) => r.name); }
+  catch (e) { out.unreadable = true; out.authFailed = isAuthFailure(e); return out; }
+  const has = (n) => names.includes(n);
+  const mine3 = has(`${member}_session_parts`); const mine2 = has(`${member}_messages`);
+  out.member = mine3 && mine2 ? 'both' : mine3 ? '3' : mine2 ? '2' : null;
+  out.devLayout = has(`${member}_turns`);
+  // Counted from <m>_turns itself, where the imported rows live: <m>_sessions may have been dropped
+  // or renamed for the move, or already be the profile's.
+  if (out.devLayout) {
+    try {
+      out.devImports = Number(((await rows(`SELECT uniqExact(harness, session_id) AS n FROM \`${db}\`.\`${member}_turns\` FINAL WHERE is_deleted = 0 AND origin = 'import'`))[0] || {}).n || 0);
+    } catch { out.devImports = null; }
+  }
+  if (has(`${member}_meta`)) {
+    try {
+      const r = await rows(`SELECT value FROM \`${db}\`.\`${member}_meta\` FINAL WHERE key = 'min_writer_schema'`);
+      out.fenced = r.length > 0 && Number(r[0].value) >= 3;
+      out.converted = (await rows(`SELECT value FROM \`${db}\`.\`${member}_meta\` FINAL WHERE key = 'profile_converted'`)).length > 0;
+    } catch { /* unreadable: no fence known */ }
+  }
+  const any3 = names.some((n) => n.endsWith('_session_parts')); const any2 = names.some((n) => n.endsWith('_messages'));
+  out.house = any3 && any2 ? 'mixed' : any3 ? '3' : any2 ? '2' : null;
+  return out;
+}
+/**
+ * Whether this build's `memhouse import` writes the profile tables. Until it does, a member whose
+ * imported sessions live only in <m>_turns is not moved: nothing in the move brings them back.
+ */
+function importWritesProfile() {
+  try { return require('../import/import').WRITES_LAYOUTS.includes('profile'); } catch { return false; }
+}
+
+/**
+ * What to say about a development-layout member's imported sessions, or '' when it has none.
+ * They live only in <m>_turns: a recapture reads the editors' files, and an import has none.
+ */
+function importsNotice(det, member) {
+  if (!det.devLayout || det.devImports === 0) return '';
+  const n = det.devImports === null ? 'an unknown number of' : String(det.devImports);
+  const how = importWritesProfile()
+    ? 'keep their export and import it again after the move, a dry run first'
+    : 'this build cannot import into the profile tables, so move this member with a build that can, then import the export again';
+  return `${n} imported session(s) need a re-import: '${member}' holds them in ${member}_turns (memhouse import), and a recapture cannot bring them back — ${how} (docs/guides/upgrade-to-0.20.md)`;
+}
+
+/** The schema to install with, or { refuse: message }. `stated`: '', '2' or '3' as given. */
+/** Why a member whose house holds <m>_turns may not be laid or captured on the profile, or ''. */
+function devLayoutRefusal(det, member) {
+  if (!det.devLayout) return '';
+  const notice = importsNotice(det, member);
+  // The profile tables already there and nothing imported left behind: <m>_turns is a leftover.
+  if (det.member === '3' && !(notice && !importWritesProfile())) return '';
+  return `the house holds ${member}_turns, the layout of memhouse 0.20's development builds, which harness-sessions-sql's profile replaces — its admin recaptures or converts '${member}' first (docs/guides/upgrade-to-0.20.md)${notice ? `. ${notice}` : ''}`;
+}
+
+/**
+ * @param {{user?:string, url?:string}} [who]  the credential the detection connected as, to name a refusal
+ */
+function chooseSchema(stated, det, member, who = {}) {
+  // A credential the server refused saw nothing: say that, rather than reading the house as new
+  // and letting the next step fail on the same credential.
+  if (det.authFailed) return { refuse: `${authFailureText(who.user, who.url)} — check the user and password, then run this again`, auth: true };
+  // Checked before the member's own tables: an --ensure-schema that laid the profile beside
+  // <m>_turns must not make the development layout look converted.
+  const dev = devLayoutRefusal(det, member);
+  if (dev) return { refuse: dev };
+  if (det.member === 'both') return { refuse: `the house holds both ${member}_messages (0.18) and ${member}_session_parts (schema 3) for '${member}' — ask its admin which is current (docs/guides/upgrade-to-0.20.md)` };
+  if (det.member && stated && det.member !== stated) {
+    return { refuse: `the house holds '${member}' on schema ${det.member}; --schema ${stated} contradicts it. Leave --schema out to follow the house` };
+  }
+  // The conversion lays the profile tables first: until it ran to the end, they are not the house yet.
+  if (det.member === '3' && det.fenced && !det.converted) {
+    return { refuse: `'${member}' is being converted to schema 3 and its conversion has not finished — install once its admin has run memhouse migrate --v3-convert to the end (a re-run skips the finished buckets)` };
+  }
+  if (det.member) return { schema: det.member };
+  if (det.fenced) return { refuse: `'${member}' is being converted to schema 3 (its 0.18 rooms are fenced off, its schema-3 tables not there yet) — install once its admin has run memhouse migrate --v3-convert` };
+  if (stated) return { schema: stated };
+  if (det.house === 'mixed') return { refuse: 'this house holds members on the 0.18 rooms and members on schema 3 — say which this install joins: --schema 2 or --schema 3' };
+  if (det.house) return { schema: det.house };
+  return { schema: '3' };   // a new house
+}
+
+module.exports = { detectHouseSchema, chooseSchema, importsNotice, importWritesProfile, devLayoutRefusal, isAuthFailure, authFailureText, AUTHENTICATION_FAILED, REQUIRED_PASSWORD };

package/memhouse/house/schema-profile.js

@@ -0,0 +1,290 @@
+// The harness-sessions-sql ClickHouse profile as memhouse's house tables (step B1): one member's tables,
+// in two databases.
+//
+//   <db>.<m>_session_parts, _sessions, _session_usage, _session_links, _session_lifecycle, _calls
+//                                       conversation layer, under the member's `<db>.<m>_*` grant
+//   <db>_archive.<m>_session_units, _session_sources
+//                                       capture layer: SELECT and INSERT for the member, no grant
+//                                       option, nothing in <db>_archive granted to anyone else
+//
+// The DDL lives in schema-profile.sql.tpl, rendered with {db} and {m}; every statement is
+// CREATE … IF NOT EXISTS. Transport-free like schema-v3.js: every entry point takes `q`.
+
+const fs = require('fs');
+const path = require('path');
+const { assertUsableName, legacyTextIndexDialect, isTextIndexGrammarRefusal, textIndexDialectFor } = require('./house');
+const { stripComments, qFromClient, DDL_SETTINGS } = require('./schema-v3');
+
+const TEMPLATE = path.join(__dirname, 'schema-profile.sql.tpl');
+
+// The columns retyped in harness-sessions-sql v1, which ensureProfileSchema brings an older table to:
+// adapter_ver UInt32 -> LowCardinality(String), child_stored Bool -> Nullable(Bool), orig_sha256
+// FixedString(64) -> Nullable(FixedString(64)).
+const TYPE_HEALS = new Set(['adapter_ver', 'child_stored', 'orig_sha256']);
+const CONVERSATION_TYPES = ['session_parts', 'sessions', 'session_usage', 'session_links', 'session_lifecycle', 'calls'];
+const CAPTURE_TYPES = ['session_units', 'session_sources'];
+const PROFILE_TYPES = [...CONVERSATION_TYPES, ...CAPTURE_TYPES];
+
+/** Each table's sorting key as system.tables.sorting_key reports it: harness-sessions-sql §3's, host first, no ts. */
+const S = 'host, harness, harness_path, session_id, incarnation';
+const PROFILE_KEYS = {
+  session_parts: `${S}, message_id, part_id`,
+  sessions: S,
+  session_usage: `${S}, usage_id`,
+  session_links: `${S}, owner, kind, other_host, other_harness_path, other_harness, other_session_id, other_incarnation, at`,
+  session_lifecycle: `${S}, lifecycle_id`,
+  calls: `${S}, call_id`,
+  session_units: `${S}, source_key, generation, unit_key, chunk, sha256`,
+  session_sources: `${S}, source_key`,
+};
+
+/**
+ * Columns an INSERT built from a row's mapped columns leaves to the server (insertColumns). The
+ * capture's archive rows are written as built and carry captured_at themselves: one value per
+ * archive write, the replay's pass id (profile-capture.js header: archive).
+ */
+const SERVER_COLUMNS = ['writer', 'ingested_at', 'search_ngram', 'search_word', 'ver', 'captured_at', 'committed_at'];
+
+function archiveDb(db) { return `${db}_archive`; }
+
+/** Every fully-qualified name of one member's profile tables. */
+function profileNames(db, member) {
+  assertUsableName(db, 'house');
+  assertUsableName(member, 'member');
+  const out = { db, archive: archiveDb(db), member };
+  for (const t of CONVERSATION_TYPES) out[t] = `${db}.${member}_${t}`;
+  for (const t of CAPTURE_TYPES) out[t] = `${archiveDb(db)}.${member}_${t}`;
+  out.meta = `${db}.${member}_meta`;
+  out.events = `${db}.${member}_events`;
+  return out;
+}
+
+function readTemplate() { return fs.readFileSync(TEMPLATE, 'utf-8'); }
+
+/** The CREATE statements for one member, in template order: [{ type, database, table, fq, sql }]. */
+function renderStatements(tpl, { db, member }) {
+  assertUsableName(db, 'house');
+  assertUsableName(member, 'member');
+  const body = stripComments(tpl).replaceAll('{db}', db).replaceAll('{m}', member);
+  const out = [];
+  for (const raw of body.split(';')) {
+    const sql = raw.trim();
+    if (!sql) continue;
+    const m = /^CREATE TABLE IF NOT EXISTS ([A-Za-z0-9_]+)\.([A-Za-z0-9_]+)\s*\(/.exec(sql);
+    if (!m) throw new Error(`schema-profile template: unexpected statement: ${sql.slice(0, 80)}`);
+    const type = m[2].startsWith(`${member}_`) ? m[2].slice(member.length + 1) : m[2];
+    if (!PROFILE_TYPES.includes(type)) throw new Error(`schema-profile template declares an unknown table '${m[2]}'`);
+    out.push({ type, database: m[1], table: m[2], fq: `${m[1]}.${m[2]}`, sql });
+  }
+  const missing = PROFILE_TYPES.filter((t) => !out.some((s) => s.type === t));
+  if (missing.length) throw new Error(`schema-profile template is missing: ${missing.join(', ')}`);
+  return out;
+}
+
+/** The columns each table declares, as [{ name, type }]; a Nested column is one entry. */
+function templateColumns(tpl) {
+  const out = {};
+  for (const s of renderStatements(tpl, { db: 'db', member: 'm' })) {
+    const inner = s.sql.slice(s.sql.indexOf('(') + 1, s.sql.search(/\)\s*ENGINE\b/));
+    const cols = [];
+    let depth = 0; let buf = '';
+    for (const ch of `${inner},`) {
+      if (ch === '(') depth++;
+      if (ch === ')') depth--;
+      if (ch === ',' && depth === 0) {
+        const decl = buf.replace(/\s+/g, ' ').trim();
+        buf = '';
+        if (!decl || /^(INDEX|CONSTRAINT|PROJECTION)\b/i.test(decl)) continue;
+        const sp = decl.indexOf(' ');
+        cols.push({ name: decl.slice(0, sp), type: decl.slice(sp + 1) });
+        continue;
+      }
+      buf += ch;
+    }
+    out[s.type] = cols;
+  }
+  return out;
+}
+function rowColumns(tpl, type) {
+  return (templateColumns(tpl)[type] || []).map((c) => c.name).filter((x) => !SERVER_COLUMNS.includes(x));
+}
+/** The columns an INSERT names, in template order: rowColumns with a Nested column as its dotted subcolumns. */
+function insertColumns(tpl, type) {
+  return (templateColumns(tpl)[type] || []).filter((c) => !SERVER_COLUMNS.includes(c.name)).flatMap((c) => {
+    const m = /^Nested\s*\(([\s\S]*)\)$/i.exec(c.type);
+    if (!m) return [c.name];
+    const subs = []; let depth = 0; let buf = '';
+    for (const ch of `${m[1]},`) {
+      if (ch === '(') depth++;
+      if (ch === ')') depth--;
+      if (ch === ',' && depth === 0) { if (buf.trim()) subs.push(`${c.name}.${buf.trim().split(' ')[0]}`); buf = ''; continue; }
+      buf += ch;
+    }
+    return subs;
+  });
+}
+/** The ORDER BY each table's template declares. */
+function templateKeys(tpl) {
+  const out = {};
+  for (const s of renderStatements(tpl, { db: 'db', member: 'm' })) {
+    const m = /ORDER BY \(([^)]*)\)/.exec(s.sql);
+    out[s.type] = m ? m[1].replace(/\s+/g, ' ').trim() : '';
+  }
+  return out;
+}
+
+/** What is wrong with a profile table's sorting key, or null. */
+function keyProblem(type, sortingKey) {
+  const want = PROFILE_KEYS[type];
+  if (!want) return `'${type}' is not a profile table`;
+  const have = String(sortingKey || '').trim();
+  if (!have) return 'not a MergeTree';
+  return have === want ? null : `(${have}) — expected (${want})`;
+}
+
+/** The capture layer's grants: SELECT and INSERT on the member's two archive tables, no grant option. */
+function captureGrants(db, member) {
+  const n = profileNames(db, member);
+  return CAPTURE_TYPES.map((t) => `GRANT SELECT, INSERT ON ${n[t]} TO ${member}`);
+}
+
+function isDenied(m) { return /Not enough privileges|ACCESS_DENIED/i.test(m); }
+function isExists(m) { return /ALREADY_EXISTS|already exists/i.test(m); }
+function msg(e) { return e && e.message ? e.message : String(e); }
+
+/**
+ * Create one member's profile tables, idempotently, then verify them against PROFILE_KEYS.
+ * As ensureV3Schema: a privilege denial is counted and skipped, a lost create race is success.
+ */
+async function ensureProfileSchema(q, { db, member, createDatabases = false, grantCapture = false, tpl = null } = {}, ui = null) {
+  const n = profileNames(db, member);
+  const say = (m) => ui && ui.ok && ui.ok(m);
+  const version = String(((await q.rows('SELECT version() AS v', { final: 0 }))[0] || {}).v || '');
+  const dialect = textIndexDialectFor(version);
+  const out = { version, dialect, created: [], denied: [], unhealed: [], grants: [], grant_failures: [], tables: {}, ok: false };
+  if (createDatabases) {
+    for (const d of [n.db, n.archive]) {
+      try { await q.sql(`CREATE DATABASE IF NOT EXISTS ${d}`, DDL_SETTINGS); } catch (e) {
+        const m = msg(e);
+        if (isDenied(m)) out.denied.push(`CREATE DATABASE ${d}`); else if (!isExists(m)) throw e;
+      }
+    }
+  }
+  for (const s of renderStatements(tpl || readTemplate(), { db, member })) {
+    const modern = s.sql;
+    const legacy = legacyTextIndexDialect(modern);
+    const [first, second] = dialect === 'legacy' ? [legacy, modern] : [modern, legacy];
+    try {
+      try { await q.sql(first, DDL_SETTINGS); } catch (e) {
+        if (!(first !== second && isTextIndexGrammarRefusal(msg(e)))) throw e;
+        await q.sql(second, DDL_SETTINGS);
+      }
+      out.created.push(s.fq);
+    } catch (e) {
+      const m = msg(e);
+      if (isDenied(m)) { out.denied.push(`CREATE TABLE ${s.fq}`); continue; }
+      if (isExists(m)) { out.created.push(s.fq); continue; }
+      throw new Error(`${s.fq}: ${m.split('\n')[0]}`);
+    }
+  }
+  // A table made before the template gained a column (session_links' r_* deployment columns, for
+  // one) gets it: ADD COLUMN IF NOT EXISTS for each column the template declares and it lacks.
+  out.added = [];
+  const lacking = new Map();   // table -> the columns a denial left it without
+  const want = templateColumns(tpl || readTemplate());
+  const dbOf = (t) => (CAPTURE_TYPES.includes(t) ? n.archive : n.db);
+  const have = new Map();
+  const types = new Map();   // db.table.column -> the server's type
+  const defaults = new Map(); // db.table.column -> the server's default expression ('' for none)
+  for (const r of await q.rows(`SELECT database, table, name, type, default_expression FROM system.columns
+    WHERE (database = '${n.db}' AND table IN (${CONVERSATION_TYPES.map((t) => `'${member}_${t}'`).join(', ')}))
+       OR (database = '${n.archive}' AND table IN (${CAPTURE_TYPES.map((t) => `'${member}_${t}'`).join(', ')}))`, { final: 0 })) {
+    const k = `${r.database}.${r.table}`;
+    if (!have.has(k)) have.set(k, new Set());
+    have.get(k).add(r.name);
+    types.set(`${k}.${r.name}`, String(r.type || ''));
+    defaults.set(`${k}.${r.name}`, String(r.default_expression || ''));
+  }
+  for (const t of PROFILE_TYPES) {
+    const got = have.get(`${dbOf(t)}.${member}_${t}`);
+    if (!got) continue;
+    for (const c of want[t] || []) {
+      // A Nested column is listed by its fields (usage.hour, usage.model, ...).
+      if (got.has(c.name) || [...got].some((x) => x.startsWith(`${c.name}.`))) continue;
+      try {
+        await q.sql(`ALTER TABLE ${n[t]} ADD COLUMN IF NOT EXISTS \`${c.name}\` ${c.type}`, DDL_SETTINGS);
+        out.added.push(`${n[t]}.${c.name}`);
+      } catch (e) {
+        const m = msg(e);
+        if (isDenied(m)) { out.denied.push(`ALTER TABLE ${n[t]} ADD COLUMN ${c.name}`); lacking.set(t, [...(lacking.get(t) || []), c.name]); continue; }
+        throw new Error(`${n[t]}: ${m.split('\n')[0]}`);
+      }
+    }
+  }
+  if (out.added.length) say(`profile tables for ${member}: added ${out.added.join(', ')}`);
+  // A column whose type the spec changed (TYPE_HEALS: harness-sessions-sql v1) is brought to the template's
+  // type: MODIFY COLUMN, which ClickHouse applies to the parts as a mutation. Only these columns: a
+  // type changed anywhere else is a template bug, not something to heal on a live house. A member
+  // may not ALTER its <db>_archive tables, so those wait for an admin; the capture writes both typings
+  // (ProfileHouse.insert), so an archive table left on the old ones is reported, not refused. A
+  // conversation table the member could not retype is a problem: session_links' old child_stored
+  // turns an unknown into true.
+  out.retyped = [];
+  for (const t of PROFILE_TYPES) {
+    const k = `${dbOf(t)}.${member}_${t}`;
+    if (!have.has(k)) continue;
+    for (const c of want[t] || []) {
+      const now = types.get(`${k}.${c.name}`);
+      // A type-only MODIFY keeps the column's old default (child_stored's true, orig_sha256's ''),
+      // so a default the template does not declare is removed too.
+      const stale = !/\bDEFAULT\b/.test(c.type) && !!defaults.get(`${k}.${c.name}`);
+      if (!TYPE_HEALS.has(c.name) || now == null || (now === c.type && !stale)) continue;
+      try {
+        if (now !== c.type) await q.sql(`ALTER TABLE ${n[t]} MODIFY COLUMN \`${c.name}\` ${c.type}`, DDL_SETTINGS);
+        if (stale) await q.sql(`ALTER TABLE ${n[t]} MODIFY COLUMN \`${c.name}\` REMOVE DEFAULT`, DDL_SETTINGS);
+        out.retyped.push(`${n[t]}.${c.name}: ${now} -> ${c.type}${stale ? ', its old default removed' : ''}`);
+      } catch (e) {
+        const m = msg(e);
+        if (isDenied(m)) {
+          out.denied.push(`ALTER TABLE ${n[t]} MODIFY COLUMN ${c.name}`);
+          const what = now === c.type ? `${c.name} (its old default kept)` : `${c.name} (${now}, not ${c.type})`;
+          if (CAPTURE_TYPES.includes(t)) out.unhealed.push(`${n[t]}.${what}`);
+          else lacking.set(t, [...(lacking.get(t) || []), what]);
+          continue;
+        }
+        throw new Error(`${n[t]}: ${m.split('\n')[0]}`);
+      }
+    }
+  }
+  if (out.retyped.length) say(`profile tables for ${member}: retyped ${out.retyped.join(', ')}`);
+  if (grantCapture) {
+    for (const g of captureGrants(db, member)) {
+      try { await q.sql(g, DDL_SETTINGS); out.grants.push(g); } catch (e) {
+        const m = msg(e);
+        if (isDenied(m)) out.denied.push(g); else out.grant_failures.push(`${g}: ${m.split('\n')[0]}`);
+      }
+    }
+  }
+  const rows = await q.rows(
+    `SELECT database, name, sorting_key FROM system.tables
+     WHERE (database = '${n.db}' AND name IN (${CONVERSATION_TYPES.map((t) => `'${member}_${t}'`).join(', ')}))
+        OR (database = '${n.archive}' AND name IN (${CAPTURE_TYPES.map((t) => `'${member}_${t}'`).join(', ')}))`, { final: 0 });
+  let ok = true;
+  for (const t of PROFILE_TYPES) {
+    const fq = n[t];
+    const r = rows.find((x) => `${x.database}.${x.name}` === fq);
+    // A table without a column the template declares cannot take the rows written to it.
+    const problem = r ? keyProblem(t, r.sorting_key) || (lacking.has(t) ? `columns not as the template has them: ${lacking.get(t).join(', ')}` : null) : 'missing';
+    if (problem) ok = false;
+    out.tables[t] = { fq, exists: Boolean(r), problem };
+  }
+  out.ok = ok;
+  if (ok) say(`profile tables for ${member}: ${PROFILE_TYPES.map((t) => n[t]).join(', ')}`);
+  return out;
+}
+
+module.exports = {
+  TEMPLATE, CONVERSATION_TYPES, CAPTURE_TYPES, PROFILE_TYPES, PROFILE_KEYS, SERVER_COLUMNS,
+  archiveDb, profileNames, readTemplate, renderStatements, templateColumns, templateKeys, rowColumns, insertColumns,
+  keyProblem, captureGrants, ensureProfileSchema, qFromClient,
+};