@dooer/dooer-test-env 1.18.0 → 1.19.0

package/lib/command/env.js

@@ -15,15 +15,33 @@ const PROFILES = ['full', 'frontend', 'hq', 'booking']
 // `full` (everything staging runs) — compose profiles are additive, so we must name it explicitly.
 // See ENVIRONMENT-PLAN.md §6.
 
+// Regenerate when the generator is newer than the file it produced. It used to generate only when the
+// file was ABSENT, which meant a CLI upgrade never reached an existing environment: every generator fix
+// (a container flag, a new service, an image bump) silently applied to new installs only, and the way you
+// found out was a bug that was already fixed. An npm install rewrites lib/, so its mtime moves ahead of a
+// compose file generated by the previous version, and a local edit to the generator does the same.
+// The compose file is a build artifact — anything hand-edited there was already lost on the next `db pull`.
+function composeIsStale() {
+  if (!fs.existsSync(rt.COMPOSE_FILE)) return true
+  try {
+    const generator = require.resolve('../compose/generate')
+    return fs.statSync(generator).mtimeMs > fs.statSync(rt.COMPOSE_FILE).mtimeMs
+  } catch (_) {
+    return false // never block `up` on a stat that failed
+  }
+}
+
 function ensureCompose() {
   rt.ensureRunDir()
-  if (!fs.existsSync(rt.COMPOSE_FILE)) {
+  if (composeIsStale()) {
+    const existed = fs.existsSync(rt.COMPOSE_FILE)
     const r = generateCompose({
       servicesDir: rt.SERVICES_DIR,
       out: rt.COMPOSE_FILE,
       outputValidation: !!rt.readState().outputValidation,
     })
-    console.log(chalk.gray(`generated ${rt.COMPOSE_FILE} (${(r && r.serviceCount) || 'n'} services)`))
+    const what = existed ? 'regenerated (CLI is newer)' : 'generated'
+    console.log(chalk.gray(`${what} ${rt.COMPOSE_FILE} (${(r && r.serviceCount) || 'n'} services)`))
   }
   return rt.COMPOSE_FILE
 }

package/lib/compose/generate.js

@@ -242,6 +242,12 @@ function infraServices() {
         'ssl_cert_file=/etc/postgresql/server.crt',
         '-c',
         'ssl_key_file=/etc/postgresql/server.key',
+        // A full Dooer DB is ~163 schemas / 25k tables+sequences, and `db snapshot` runs pg_dump, which
+        // takes an ACCESS SHARE lock on every one of them inside a single transaction. The default 64
+        // gives ~64×(max_connections+prepared) lock slots and pg_dump dies partway with "out of shared
+        // memory / You might need to increase max_locks_per_transaction" — reliably, on a real base DB.
+        '-c',
+        'max_locks_per_transaction=1024',
       ],
       environment: {
         POSTGRES_DB: 'dooer',

package/lib/copy.js

@@ -93,6 +93,32 @@ async function copyOrganization({ organizationId, sourceEnv, targetEnv, reason,
       `  ${detail.rows} rows across ${detail.tables} tables`
   )
   console.log(`  new organization id: ${chalk.bold(detail.organizationId)}`)
+
+  // Parent rows carried alongside the organization: global rows (workflow definitions, system messages)
+  // that its rows point at and the target may not have, because a target's base DB is a point-in-time
+  // snapshot while the source keeps moving. Only what was MISSING is inserted.
+  const refs = detail.references
+  if (refs && refs.rowsInArtifact) {
+    console.log(
+      `  referenced rows carried: ${refs.rowsInArtifact} from ${refs.tables.length} table(s)` +
+        `, ${refs.rowsInserted} inserted (the rest the target already had)`
+    )
+  }
+
+  // Rows whose foreign key points at a parent that did not come with them. The load runs with FK
+  // enforcement off (no ordering of 336 tables satisfies every constraint), and Postgres never re-checks,
+  // so these commit silently — they have to be told, or the copy looks clean when it is not.
+  if (detail.danglingRows) {
+    console.log(chalk.yellow(`\n  ${detail.danglingRows} row(s) still reference a parent that is not there:`))
+    for (const d of detail.dangling || []) {
+      console.log(`    ${String(d.rows).padStart(6)}  ${d.from} → ${d.to}  (${d.cause})`)
+    }
+    console.log(
+      '  The copy is complete and usable; these references dangle. Reference closure carries the parents\n' +
+        '  a copied row points at, so anything left here is a table it deliberately does not carry\n' +
+        '  (service_accounts.users / companies) or one it could not follow — see the warnings above.'
+    )
+  }
   console.log(`  audit: export ${exported.id} (${source.name}), import ${imported.id} (${target.name})\n`)
 
   return { organizationId: detail.organizationId, rows: detail.rows, tables: detail.tables, skipped }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dooer/dooer-test-env",
-  "version": "1.18.0",
+  "version": "1.19.0",
   "description": "Run the whole Dooer backend locally (staging DB minus customers), copy/purge customers between environments, and shred — one CLI.",
   "license": "UNLICENSED",
   "repository": "Dooer/cli-dooer-test-env",

package/readme.md

@@ -254,10 +254,44 @@ npx @dooer/dooer-test-env@latest customer copy \
 # optional: shred the rest of the PII afterwards (localhost only)
 npx @dooer/dooer-test-env@latest shred --execute
 
-# undo: delete that org again (rows + its S3 objects), dry-run first
+# undo: delete that org's rows again — dry-run first (omit --execute)
 npx @dooer/dooer-test-env@latest customer purge --org <orgId> --local --execute
 ```
 
+**What the purge does not do:** it deletes rows, **not S3 objects**. A purged org leaves its uploaded
+documents in the bucket; the command says so in its own output. Removing them needs bucket credentials the
+service does not hold today.
+
+**Referenced rows come with the org.** A copied row can point at a row in a table the schema map does not
+copy: `service_workflow.run` is org-scoped and copied, but the workflow `definition` it points at is
+global and is not. The target usually has those already — every environment's base DB descends from the
+same production dump — but a base DB is a point-in-time snapshot while the source keeps moving. Production
+has 819 workflow definitions; a base DB built in April 2023 has 689, so a 2026 org's runs pointed at
+versions that were not there.
+
+So the export carries the exact parent rows the org's rows reach, transitively (a carried `definition`
+points at a `workflow` in turn), and the import inserts only what is missing:
+
+```
+  referenced rows carried: 9 from 2 table(s), 8 inserted (the rest the target already had)
+```
+
+Their ids are **not** remapped — the copied rows reference them verbatim.
+
+Two tables are never carried: **`service_accounts.users`** (it would move real names and personnummer from
+the source environment into the target — exactly what a live→staging copy must not do) and
+**`service_accounts.companies`** (the organization itself). The closure is capped at 50,000 rows, and both
+the cap and any composite-key foreign key it declines to follow are reported as warnings.
+
+Anything that still dangles is reported after the load, because the bulk load runs with foreign-key
+enforcement off and Postgres never re-checks afterwards — so a dangling row would otherwise commit
+silently:
+
+```
+  44 row(s) still reference a parent that is not there:
+      42  service_workflow.run.definitionId → service_workflow.definition
+```
+
 Naming: with **neither `--name` nor `--target`** the copy creates a new org that **keeps the source org's
 own name** (its `short_name` still gets a unique suffix). Pass `--name` to rename it, or `--target <uuid>`
 to copy into an org that already exists.