@dooer/dooer-test-env 1.17.2 → 1.18.1

package/lib/command/customer.js

@@ -2,6 +2,28 @@ const fs = require('fs')
 const chalk = require('chalk')
 const remote = require('../remote')
 const serviceClient = require('../service-client')
+
+// Run the documented shredder over the local database. Emails are already anonymized by the copy itself;
+// this is the rest of the PII.
+async function shredLocal() {
+  console.log(chalk.bold('\nshredding the local database…'))
+  const { shred } = require('../shred')
+  const { Client } = require('pg')
+  const client = new Client({ host: 'localhost', port: 55432, user: 'dooer', password: 'dooer', database: 'dooer' })
+  await client.connect()
+  try {
+    const result = await shred(client, { execute: true })
+    const failed = (result && result.failed) || []
+    if (failed.length) {
+      // Say so rather than letting a partial shred read as a clean one.
+      console.log(chalk.yellow(`  ${failed.length} shred script(s) failed — local PII is NOT fully anonymized\n`))
+    } else {
+      console.log(chalk.green(`  shredded ${((result && result.scripts) || []).length} script(s)\n`))
+    }
+  } finally {
+    await client.end().catch(() => {})
+  }
+}
 const { createAccount, SUBSCRIPTION_TYPES } = require('../account')
 
 // `copy` now runs through service-dooer-test-env (export in the source, import in the target) instead of
@@ -65,6 +87,10 @@ const copyOptions = (y) =>
     .option('skip-users', { type: 'boolean', describe: 'skip copying referenced users missing from the target' })
     .option('confirm-production', { type: 'boolean', describe: 'REQUIRED to write to dooer-production' })
     .option('reason', { type: 'string', describe: 'recorded in the audit trail at both ends' })
+    .option('shred', {
+      type: 'boolean',
+      describe: 'after a LOCAL copy, run the shredder over the local database (localhost only)',
+    })
     .option('execute', { type: 'boolean', default: false, describe: 'actually write (default: dry-run)' })
 
 module.exports = {
@@ -82,6 +108,12 @@ module.exports = {
           const targetEnv = envFromFlags(argv.targetLocal, argv.targetNamespace, sourceEnv)
           const { copyOrganization } = require('../copy')
 
+          // Shredding is whole-database and localhost-only by design (see command/shred.js), so it can
+          // only follow a copy INTO local, and only once — not per org in a batch.
+          if (argv.shred && targetEnv !== 'local') {
+            throw new Error('--shred only applies to a copy into the local env (the shredder is localhost-only)')
+          }
+
           if (sources.length === 1) {
             await copyOrganization({
               organizationId: sources[0],
@@ -90,6 +122,7 @@ module.exports = {
               reason: argv.reason,
               dryRun: !argv.execute,
             })
+            if (argv.shred && argv.execute) await shredLocal()
             return
           }
           // Batch: each org is a full independent copy (the engine scans the source per org). One bad uuid
@@ -117,6 +150,8 @@ module.exports = {
             )
           )
           failed.forEach((f) => console.log(chalk.red(`  ${f.source}: ${f.error}`)))
+          // Once, after the whole batch — the shredder covers the entire database.
+          if (argv.shred && argv.execute && failed.length < sources.length) await shredLocal()
           if (failed.length) process.exitCode = 1
         },
       })

package/lib/command/env.js

@@ -15,15 +15,33 @@ const PROFILES = ['full', 'frontend', 'hq', 'booking']
 // `full` (everything staging runs) — compose profiles are additive, so we must name it explicitly.
 // See ENVIRONMENT-PLAN.md §6.
 
+// Regenerate when the generator is newer than the file it produced. It used to generate only when the
+// file was ABSENT, which meant a CLI upgrade never reached an existing environment: every generator fix
+// (a container flag, a new service, an image bump) silently applied to new installs only, and the way you
+// found out was a bug that was already fixed. An npm install rewrites lib/, so its mtime moves ahead of a
+// compose file generated by the previous version, and a local edit to the generator does the same.
+// The compose file is a build artifact — anything hand-edited there was already lost on the next `db pull`.
+function composeIsStale() {
+  if (!fs.existsSync(rt.COMPOSE_FILE)) return true
+  try {
+    const generator = require.resolve('../compose/generate')
+    return fs.statSync(generator).mtimeMs > fs.statSync(rt.COMPOSE_FILE).mtimeMs
+  } catch (_) {
+    return false // never block `up` on a stat that failed
+  }
+}
+
 function ensureCompose() {
   rt.ensureRunDir()
-  if (!fs.existsSync(rt.COMPOSE_FILE)) {
+  if (composeIsStale()) {
+    const existed = fs.existsSync(rt.COMPOSE_FILE)
     const r = generateCompose({
       servicesDir: rt.SERVICES_DIR,
       out: rt.COMPOSE_FILE,
       outputValidation: !!rt.readState().outputValidation,
     })
-    console.log(chalk.gray(`generated ${rt.COMPOSE_FILE} (${(r && r.serviceCount) || 'n'} services)`))
+    const what = existed ? 'regenerated (CLI is newer)' : 'generated'
+    console.log(chalk.gray(`${what} ${rt.COMPOSE_FILE} (${(r && r.serviceCount) || 'n'} services)`))
   }
   return rt.COMPOSE_FILE
 }

package/lib/compose/generate.js

@@ -242,6 +242,12 @@ function infraServices() {
         'ssl_cert_file=/etc/postgresql/server.crt',
         '-c',
         'ssl_key_file=/etc/postgresql/server.key',
+        // A full Dooer DB is ~163 schemas / 25k tables+sequences, and `db snapshot` runs pg_dump, which
+        // takes an ACCESS SHARE lock on every one of them inside a single transaction. The default 64
+        // gives ~64×(max_connections+prepared) lock slots and pg_dump dies partway with "out of shared
+        // memory / You might need to increase max_locks_per_transaction" — reliably, on a real base DB.
+        '-c',
+        'max_locks_per_transaction=1024',
       ],
       environment: {
         POSTGRES_DB: 'dooer',

package/lib/copy.js

@@ -93,6 +93,20 @@ async function copyOrganization({ organizationId, sourceEnv, targetEnv, reason,
       `  ${detail.rows} rows across ${detail.tables} tables`
   )
   console.log(`  new organization id: ${chalk.bold(detail.organizationId)}`)
+
+  // Rows whose foreign key points at a parent that did not come with them. The load runs with FK
+  // enforcement off (no ordering of 336 tables satisfies every constraint), and Postgres never re-checks,
+  // so these commit silently — they have to be told, or the copy looks clean when it is not.
+  if (detail.danglingRows) {
+    console.log(chalk.yellow(`\n  ${detail.danglingRows} row(s) reference a parent that was not copied:`))
+    for (const d of detail.dangling || []) {
+      console.log(`    ${String(d.rows).padStart(6)}  ${d.from} → ${d.to}  (${d.cause})`)
+    }
+    console.log(
+      '  The copy is complete and usable; these references dangle. `parent table not in schema map`\n' +
+        '  means the map does not carry that parent — it will recur on every copy until the map changes.'
+    )
+  }
   console.log(`  audit: export ${exported.id} (${source.name}), import ${imported.id} (${target.name})\n`)
 
   return { organizationId: detail.organizationId, rows: detail.rows, tables: detail.tables, skipped }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dooer/dooer-test-env",
-  "version": "1.17.2",
+  "version": "1.18.1",
   "description": "Run the whole Dooer backend locally (staging DB minus customers), copy/purge customers between environments, and shred — one CLI.",
   "license": "UNLICENSED",
   "repository": "Dooer/cli-dooer-test-env",

package/readme.md

@@ -254,10 +254,28 @@ npx @dooer/dooer-test-env@latest customer copy \
 # optional: shred the rest of the PII afterwards (localhost only)
 npx @dooer/dooer-test-env@latest shred --execute
 
-# undo: delete that org again (rows + its S3 objects), dry-run first
+# undo: delete that org's rows again — dry-run first (omit --execute)
 npx @dooer/dooer-test-env@latest customer purge --org <orgId> --local --execute
 ```
 
+**What the purge does not do:** it deletes rows, **not S3 objects**. A purged org leaves its uploaded
+documents in the bucket; the command says so in its own output. Removing them needs bucket credentials the
+service does not hold today.
+
+**Dangling references.** The load runs with foreign-key enforcement off — no ordering of 336 tables
+satisfies every constraint on the way in — and Postgres never re-checks afterwards, so a copied row whose
+parent is missing would otherwise commit silently. The import checks the rows it wrote and reports any
+that dangle:
+
+```
+  44 row(s) reference a parent that was not copied:
+      42  service_workflow.run.definitionId → service_workflow.definition  (parent table not in schema map)
+       2  service_accounts.systemMessageRecipient.systemMessageId → service_accounts.systemMessage
+```
+
+`parent table not in schema map` means the map does not carry that parent at all, so it recurs on every
+copy until the map changes. The copy is complete and usable; those references dangle.
+
 Naming: with **neither `--name` nor `--target`** the copy creates a new org that **keeps the source org's
 own name** (its `short_name` still gets a unique suffix). Pass `--name` to rename it, or `--target <uuid>`
 to copy into an org that already exists.