@dooer/dooer-test-env 1.0.0

package/lib/engine/dbbuild.js

@@ -0,0 +1,325 @@
+// dbbuild.js — the customer-free base DB. Two entry points:
+//   build(): runs IN-CLUSTER (k8s Job). The base contains NO organizations (Jimmy 2026-09-01). It
+//            schema-only-restores into its OWN scratch DB, then FULL-copies every NON-org table (reference/
+//            global data + users) — NEVER an org-scoped table, so nothing is imported-then-deleted and the
+//            scratch stays tiny. Then it shreds (anonymize users), runs the PII audit (fails on any
+//            uncovered PII column), pg_dumps the result and uploads it to the OBC bucket. The scratch is a
+//            SEPARATE instance — never the dooer-database volume (which lacks room for a full copy anyway).
+//   pull():  runs on a DEV machine. Downloads the newest base artifact from the OBC bucket and restores it
+//            into the LOCAL Postgres. Never touches a k8s DB.
+// See ENVIRONMENT-PLAN.md §5.
+const { spawnSync } = require('child_process')
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+const { Client } = require('pg')
+const copyFrom = require('pg-copy-streams').from
+const copyTo = require('pg-copy-streams').to
+const { pipeline } = require('stream/promises')
+const engine = require('./seed')
+const { purgeTables } = require('./purge')
+
+const { filterClause, qTable, qIdent } = engine
+
+// Connect directly. SSL defaults ON (the Zalando dooer-database rejects non-SSL via pg_hba); pass
+// ssl:false for the plain scratch postgres (which has no SSL → would error "server does not support SSL").
+async function connectDirect(params) {
+  const { ssl, ...rest } = params
+  const c = new Client({ ...rest, ssl: ssl === false ? undefined : { rejectUnauthorized: false } })
+  await c.connect()
+  return c
+}
+
+// default partner domains whose orgs are customers to remove (Jimmy 2026-09-01). Others are kept + shredded.
+const DEFAULT_PURGE_PARTNERS = ['dooer', 'dooersales']
+
+// Columns that mark a table as org-scoped (same OWNER set as gen-schema-map).
+const ORG_COLUMNS = [
+  'organizationId',
+  'company_id',
+  'organization_id',
+  'companyId',
+  'fk_companies_pk',
+  'companies_pk',
+  'taskOrganizationId',
+  'customerOrganizationId',
+]
+
+// Base-build exclusions BEYOND the automatic org-closure + history. These are tables with NO org column
+// that are nonetheless customer-derived (events/logs/caches/cursors/dispatch) or org-scoped only through a
+// reverse ref / a soft (unconstrained) FK the closure can't see — so they must be listed explicitly.
+// (Jimmy's rulings 2026-09-01; extend as the plan review finalizes — see scripts/base-build-plan.js.)
+const BASE_EXCLUDE_TABLES = new Set([
+  'service_file_storage.fileData', // reverse-ref: all orgs' file blobs (fileEntry is already org-scoped)
+  'service_event_stream.event',
+  'service_automatic_booking.bookingSuggestion',
+  'service_task_engine.customAttribute', // soft FK to task (org-scoped)
+])
+// name patterns that are customer-derived / operational, never reference data
+const BASE_EXCLUDE_PATTERNS = [
+  /event$/i,
+  /eventArchive$/i,
+  /\.cursor$/i,
+  /dispatch/i,
+  /requestLog$/i,
+  /logon_log/i,
+  /loginFailure/i,
+  /cachedData$/i,
+  /\.messages$/i,
+  /aiMatchSuggestion$/i,
+  /\.bdd20se_/i, // Swedish business-registry snapshot (~25M rows) — services repopulate it (Jimmy 2026-09-01)
+  /^service_institution_browser\./i, // bank-scraping + personnummer (skatteverket_sync_run.id_number) — security-sensitive (Jimmy 2026-09-01)
+]
+const isHistory = (schema, table) => /_history$/.test(schema) || /__history$/.test(table)
+
+// the set of org ids to purge: every org under the given partner domains, minus an allow-list.
+async function customerOrgIds(client, partnerDomains, allowList) {
+  const r = await client.query(
+    `SELECT po."organizationId" AS id
+       FROM service_accounts."partnerOrganization" po
+       JOIN service_accounts.partner p ON p.id = po."partnerId"
+      WHERE p."domain" = ANY($1)`,
+    [partnerDomains]
+  )
+  const skip = new Set(allowList || [])
+  return r.rows.map((x) => x.id).filter((id) => id && !skip.has(id))
+}
+
+// reuse the copy/purge filter, promoted from a single-org `=$1` to an org-SET `= ANY($1)`. Works for
+// owner-column, crossOrgRelation and (nested) fkFilter tables alike. extraFilter is dropped (full delete).
+function filterSetSql(t) {
+  const f = filterClause(t, null, { includeExtra: false })
+  if (!f) return null
+  return f.sql.replace(/[=]\$1/g, '= ANY($1)')
+}
+
+// delete all data for the customer org set across every org-scoped table (FK/triggers disabled).
+async function bulkPurge(
+  client,
+  { partnerDomains = DEFAULT_PURGE_PARTNERS, allowList = [], execute = false, mapPath } = {}
+) {
+  const map = require(mapPath || path.join(__dirname, 'schema-map.json'))
+  const orgIds = await customerOrgIds(client, partnerDomains, allowList)
+  console.log(
+    `customer orgs to purge (partners ${partnerDomains.join(', ')}): ${orgIds.length}${
+      allowList.length ? ` (allow-list spares ${allowList.length})` : ''
+    }`
+  )
+  if (!orgIds.length) return { orgs: 0, deleted: 0 }
+  const tables = purgeTables(map)
+  if (!execute) {
+    console.log(`DRY-RUN: would delete customer rows across up to ${tables.length} tables. Pass execute to run.`)
+    return { orgs: orgIds.length, deleted: 0 }
+  }
+  await client.query('BEGIN')
+  await client.query("SET session_replication_role = 'replica'")
+  let deleted = 0
+  for (const t of tables) {
+    const sql = filterSetSql(t)
+    if (!sql) continue
+    const exists = await client.query(
+      `SELECT 1 FROM information_schema.tables WHERE table_schema=$1 AND table_name=$2 AND table_type='BASE TABLE'`,
+      [t.schema, t.table]
+    )
+    if (!exists.rowCount) continue
+    const r = await client.query(`DELETE FROM ${qTable(t.schema, t.table)} WHERE ${sql}`, [orgIds])
+    deleted += r.rowCount
+  }
+  await client.query("SET session_replication_role = 'origin'")
+  await client.query('COMMIT')
+  console.log(`bulk purge: deleted ${deleted} rows for ${orgIds.length} customer orgs`)
+  return { orgs: orgIds.length, deleted }
+}
+
+// ── shell helpers for the in-cluster dump/restore ────────────────────────────
+function pg(cmd, args, env) {
+  const r = spawnSync(cmd, args, { stdio: 'inherit', env: { ...process.env, ...env } })
+  if (r.status !== 0) throw new Error(`${cmd} failed (exit ${r.status})`)
+}
+
+function dsn({ host, port = 5432, user, password, database }) {
+  return `postgresql://${user}:${encodeURIComponent(password)}@${host}:${port}/${database}`
+}
+
+// Compute the base INCLUDE set from the LIVE schema: every base table that is NOT org-scoped (no org
+// column and not FK-reachable into one), NOT history, and NOT in the configured excludes. Returns
+// [{schema, table}]. This is the authoritative list the base copies in full (reference/global data + users).
+async function computeIncludeSet(src) {
+  const orgColRows = (
+    await src.query(
+      `SELECT DISTINCT table_schema AS s, table_name AS t FROM information_schema.columns
+       WHERE column_name = ANY($1) AND table_schema NOT IN ('pg_catalog','information_schema') AND table_schema NOT LIKE 'pg_%'`,
+      [ORG_COLUMNS]
+    )
+  ).rows
+  const fks = (
+    await src.query(`SELECT ns.nspname AS cs, cl.relname AS ct, fns.nspname AS ps, fcl.relname AS pt
+       FROM pg_constraint con
+       JOIN pg_class cl ON cl.oid=con.conrelid JOIN pg_namespace ns ON ns.oid=cl.relnamespace
+       JOIN pg_class fcl ON fcl.oid=con.confrelid JOIN pg_namespace fns ON fns.oid=fcl.relnamespace
+       WHERE con.contype='f'`)
+  ).rows
+  const all = (
+    await src.query(`SELECT ns.nspname AS s, cl.relname AS t
+       FROM pg_class cl JOIN pg_namespace ns ON ns.oid=cl.relnamespace
+       WHERE cl.relkind='r' AND ns.nspname NOT IN ('pg_catalog','information_schema') AND ns.nspname NOT LIKE 'pg_%'`)
+  ).rows
+  const key = (s, t) => `${s}.${t}`
+  const orgScoped = new Set(orgColRows.map((r) => key(r.s, r.t)))
+  let grew = true
+  while (grew) {
+    grew = false
+    for (const e of fks) {
+      const child = key(e.cs, e.ct)
+      if (!orgScoped.has(child) && orgScoped.has(key(e.ps, e.pt))) {
+        orgScoped.add(child)
+        grew = true
+      }
+    }
+  }
+  const excluded = (s, t) =>
+    orgScoped.has(key(s, t)) ||
+    isHistory(s, t) ||
+    BASE_EXCLUDE_TABLES.has(key(s, t)) ||
+    BASE_EXCLUDE_PATTERNS.some((re) => re.test(key(s, t)))
+  return all.filter((r) => !excluded(r.s, r.t)).map((r) => ({ schema: r.s, table: r.t }))
+}
+
+// stream one table in full, source → scratch, via server-side COPY (constant memory).
+async function copyTableFull(src, scratch, schema, table) {
+  const rd = src.query(copyTo(`COPY ${qTable(schema, table)} TO STDOUT`))
+  const wr = scratch.query(copyFrom(`COPY ${qTable(schema, table)} FROM STDIN`))
+  await pipeline(rd, wr)
+}
+
+// build() runs IN-CLUSTER against its OWN scratch instance (never the dooer-database volume). It requires a
+// real cluster + bucket to run; computeIncludeSet is the pure, unit-testable core. `shredFn(client)` must
+// apply the shred rules and throw if the PII audit finds uncovered columns.
+async function build(opts) {
+  const {
+    source, // { host, user, password, database:'dooer' } — the live in-cluster DB (read-only here)
+    scratch, // { host, user, password } — a SEPARATE scratch postgres (NOT dooer-database); own volume
+    scratchDatabase = 'dooer_base',
+    artifactOut = path.join(os.tmpdir(), `dooer-base-db-${Date.now()}.dump`),
+    shredFn,
+    upload, // async (file) => void  — pushes the artifact to the OBC bucket
+    execute = false,
+  } = opts
+  if (!source || !source.host) throw new Error('build() needs source DB connection info')
+  if (source.database !== 'dooer') throw new Error('refusing: source.database must be the live `dooer`')
+  const scr = scratch || source // in a Job the scratch is a sidecar; must differ from dooer-database in prod
+  if (execute && scr.host === source.host && scr.database !== 'dooer') {
+    // allowed only if it's clearly a separate DB on a separate host; refuse same-host to protect staging
+    throw new Error(
+      'scratch must be a SEPARATE postgres instance (not the dooer-database host) — see the CronJob sidecar'
+    )
+  }
+  console.log(
+    `\n=== db build (in-cluster, no organizations) ===\nsource: ${source.host}/dooer · scratch: ${scr.host}/${scratchDatabase}`
+  )
+
+  const srcRo = await connectDirect(source)
+  const include = await computeIncludeSet(srcRo)
+  console.log(`include set: ${include.length} non-org tables (full copy) + users`)
+  if (!execute) {
+    console.log(
+      'DRY-RUN: would schema-restore, full-copy the include set, shred, audit, dump, upload. Pass --execute (in-cluster).'
+    )
+    await srcRo.end()
+    return { include: include.length }
+  }
+  try {
+    // 1. schema-only → scratch DB (create it fresh on the scratch instance)
+    const admin = await connectDirect({ ...scr, database: 'postgres', ssl: false })
+    // terminate any lingering connections (e.g. from a previously-killed run) so DROP can proceed
+    await admin.query(
+      `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname=$1 AND pid<>pg_backend_pid()`,
+      [scratchDatabase]
+    )
+    await admin.query(`DROP DATABASE IF EXISTS ${qIdent(scratchDatabase)}`)
+    await admin.query(`CREATE DATABASE ${qIdent(scratchDatabase)} TEMPLATE template0`)
+    await admin.end()
+    const schemaFile = `${artifactOut}.schema.sql`
+    // --no-privileges/--no-owner: the base doesn't need staging's ACLs/roles (a 537k-line dump of
+    // per-table GRANTs to dooer_reader/dooer_writer that don't exist in scratch); skip them.
+    pg('pg_dump', ['--schema-only', '--no-owner', '--no-privileges', '-d', dsn(source), '-f', schemaFile])
+    pg('psql', ['-v', 'ON_ERROR_STOP=0', '-d', dsn({ ...scr, database: scratchDatabase }), '-f', schemaFile])
+    fs.rmSync(schemaFile, { force: true })
+    // 2. full-copy the include set (FK/triggers disabled during load)
+    const tgt = await connectDirect({ ...scr, database: scratchDatabase, ssl: false })
+    try {
+      await tgt.query("SET session_replication_role = 'replica'")
+      let i = 0
+      for (const { schema, table } of include) {
+        i++
+        process.stderr.write(`\r[copy ${i}/${include.length}] ${schema}.${table}`.slice(0, 70).padEnd(70))
+        try {
+          await copyTableFull(srcRo, tgt, schema, table)
+        } catch (e) {
+          console.error(`\n  skip ${schema}.${table}: ${e.message}`)
+        }
+      }
+      process.stderr.write('\n')
+      await tgt.query("SET session_replication_role = 'origin'")
+      // 3. shred + audit
+      if (typeof shredFn !== 'function') throw new Error('build() requires a shredFn (shred + PII audit)')
+      await shredFn(tgt)
+    } finally {
+      await tgt.end()
+    }
+    // 4. dump artifact, 5. upload, 6. drop scratch
+    pg('pg_dump', ['-Fc', '-d', dsn({ ...scr, database: scratchDatabase }), '-f', artifactOut])
+    if (typeof upload === 'function') await upload(artifactOut)
+    const admin2 = await connectDirect({ ...scr, database: 'postgres', ssl: false })
+    await admin2.query(
+      `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname=$1 AND pid<>pg_backend_pid()`,
+      [scratchDatabase]
+    )
+    await admin2.query(`DROP DATABASE IF EXISTS ${qIdent(scratchDatabase)}`)
+    await admin2.end()
+    console.log(`\ndb build complete — artifact: ${artifactOut}`)
+  } finally {
+    await srcRo.end()
+  }
+}
+
+// pull() restores the newest base artifact into the LOCAL Postgres. Download is injected so the command
+// layer owns the OBC/kubectl plumbing and this stays testable.
+async function pull(opts) {
+  const {
+    download,
+    local = { host: 'localhost', port: 5432, user: 'dooer', password: 'dooer', database: 'dooer' },
+    execute = false,
+  } = opts
+  if (local.host !== 'localhost' && local.host !== '127.0.0.1')
+    throw new Error('db pull only restores into the LOCAL Postgres')
+  if (!execute) {
+    console.log('DRY-RUN: would download the newest base artifact and restore into local Postgres. Pass --execute.')
+    return
+  }
+  if (typeof download !== 'function')
+    throw new Error('pull() requires a download() that returns the artifact file path')
+  const file = await download()
+  console.log(`restoring ${file} into ${local.host}/${local.database} …`)
+  // pg_restore exits non-zero on ANY error, but the base has expected, ignorable ones: `public` already
+  // exists, and FK constraints referencing excluded tables (e.g. workflow.trigger → the excluded event
+  // table). Data + schema still restore — so we run it non-fatally and just report the ignored count.
+  const r = spawnSync('pg_restore', ['--no-owner', '--clean', '--if-exists', '-d', dsn(local), file], {
+    stdio: 'inherit',
+  })
+  if (r.status !== 0)
+    console.log(`(pg_restore reported ignorable errors — expected: 'public' exists + FKs into excluded tables)`)
+  console.log('db pull complete.')
+}
+
+module.exports = {
+  build,
+  pull,
+  computeIncludeSet,
+  bulkPurge,
+  customerOrgIds,
+  filterSetSql,
+  DEFAULT_PURGE_PARTNERS,
+  BASE_EXCLUDE_TABLES,
+  ORG_COLUMNS,
+}