@dooer/dooer-test-env 1.18.1 → 1.19.1

package/lib/command/env.js

@@ -5,6 +5,7 @@ const registry = require('../registry')
 const bankid = require('../bankid')
 const { generateCompose } = require('../compose/generate')
 const discovery = require('../discovery/client')
+const envSpec = require('../env-spec')
 
 // Every profile the generator emits (lib/compose/generate.js). `down` must name them all, or compose
 // silently leaves the profiled containers running.
@@ -21,11 +22,17 @@ const PROFILES = ['full', 'frontend', 'hq', 'booking']
 // found out was a bug that was already fixed. An npm install rewrites lib/, so its mtime moves ahead of a
 // compose file generated by the previous version, and a local edit to the generator does the same.
 // The compose file is a build artifact — anything hand-edited there was already lost on the next `db pull`.
+//
+// The environment spec counts as an input too. It carries the image tag of every service, so refreshing it
+// and NOT regenerating leaves the stack pinned to whatever versions it was born with — which is how the
+// local service-dooer-test-env sat at 3.1.1 while staging ran 3.2.1, and a purge failed with a foreign-key
+// error that had been fixed weeks earlier.
 function composeIsStale() {
   if (!fs.existsSync(rt.COMPOSE_FILE)) return true
   try {
-    const generator = require.resolve('../compose/generate')
-    return fs.statSync(generator).mtimeMs > fs.statSync(rt.COMPOSE_FILE).mtimeMs
+    const composedAt = fs.statSync(rt.COMPOSE_FILE).mtimeMs
+    const inputs = [require.resolve('../compose/generate'), require('../env-spec').SPEC_FILE]
+    return inputs.some((f) => fs.existsSync(f) && fs.statSync(f).mtimeMs > composedAt)
   } catch (_) {
     return false // never block `up` on a stat that failed
   }
@@ -153,6 +160,26 @@ module.exports = [
         for (const n of locals)
           console.log(`  ${n}  → host.docker.internal:${state.local[n].port}  (pid ${state.local[n].pid})`)
       }
+      // Version pins from `service deploy`. They live in docker-compose.override.yml and win over the
+      // generated compose file forever — so a pinned service silently stays behind while everything around
+      // it moves, and `up` looks like it worked. Flagged when the pin no longer matches the environment
+      // spec, because that is the case where it costs you: the local service-dooer-test-env sat on 3.1.1
+      // through a spec refresh, a regenerate and a --force-recreate, failing with a bug fixed weeks before.
+      const pins = state.deploy || {}
+      const names = Object.keys(pins)
+      if (names.length) {
+        console.log(chalk.yellow('\nPinned to a specific version (`service deploy`):'))
+        for (const n of names) {
+          const current = envSpec.imageFor(n)
+          const behind = current && current !== pins[n]
+          console.log(
+            `  ${n}  → ${pins[n]}` +
+              (behind ? chalk.yellow(`  ← the environment spec has ${current.split(':').pop()}`) : '')
+          )
+        }
+        console.log(chalk.gray('  clear a pin with: service deploy <name> <tag-from-the-spec>'))
+      }
+
       try {
         const reg = await discovery.list(rt.ROUTER_URL)
         const overrides = Object.keys((reg && reg.overrides) || {})

package/lib/copy.js

@@ -94,17 +94,32 @@ async function copyOrganization({ organizationId, sourceEnv, targetEnv, reason,
   )
   console.log(`  new organization id: ${chalk.bold(detail.organizationId)}`)
 
+  // Parent rows carried alongside the organization: global rows (workflow definitions, system messages)
+  // that its rows point at and the target may not have, because a target's base DB is a point-in-time
+  // snapshot while the source keeps moving. Only what was MISSING is inserted.
+  const refs = detail.references
+  if (refs && refs.rowsInArtifact) {
+    console.log(
+      `  referenced rows carried: ${refs.rowsInArtifact} from ${refs.tables.length} table(s)` +
+        `, ${refs.rowsInserted} inserted (the rest the target already had)`
+    )
+  }
+  // Places the closure could not reach. Printed even though the copy succeeded: each one is a reference
+  // that may dangle, and a limitation nobody is told about is the same as one nobody fixed.
+  for (const w of (refs && refs.warnings) || []) console.log(chalk.yellow(`  WARNING  ${w}`))
+
   // Rows whose foreign key points at a parent that did not come with them. The load runs with FK
   // enforcement off (no ordering of 336 tables satisfies every constraint), and Postgres never re-checks,
   // so these commit silently — they have to be told, or the copy looks clean when it is not.
   if (detail.danglingRows) {
-    console.log(chalk.yellow(`\n  ${detail.danglingRows} row(s) reference a parent that was not copied:`))
+    console.log(chalk.yellow(`\n  ${detail.danglingRows} row(s) still reference a parent that is not there:`))
     for (const d of detail.dangling || []) {
       console.log(`    ${String(d.rows).padStart(6)}  ${d.from} → ${d.to}  (${d.cause})`)
     }
     console.log(
-      '  The copy is complete and usable; these references dangle. `parent table not in schema map`\n' +
-        '  means the map does not carry that parent — it will recur on every copy until the map changes.'
+      '  The copy is complete and usable; these references dangle. Reference closure carries the parents\n' +
+        '  a copied row points at, so anything left here is a table it deliberately does not carry\n' +
+        '  (service_accounts.users / companies) or one it could not follow — see the warnings above.'
     )
   }
   console.log(`  audit: export ${exported.id} (${source.name}), import ${imported.id} (${target.name})\n`)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dooer/dooer-test-env",
-  "version": "1.18.1",
+  "version": "1.19.1",
   "description": "Run the whole Dooer backend locally (staging DB minus customers), copy/purge customers between environments, and shred — one CLI.",
   "license": "UNLICENSED",
   "repository": "Dooer/cli-dooer-test-env",

package/readme.md

@@ -262,19 +262,35 @@ npx @dooer/dooer-test-env@latest customer purge --org <orgId> --local --execute
 documents in the bucket; the command says so in its own output. Removing them needs bucket credentials the
 service does not hold today.
 
-**Dangling references.** The load runs with foreign-key enforcement off — no ordering of 336 tables
-satisfies every constraint on the way in — and Postgres never re-checks afterwards, so a copied row whose
-parent is missing would otherwise commit silently. The import checks the rows it wrote and reports any
-that dangle:
+**Referenced rows come with the org.** A copied row can point at a row in a table the schema map does not
+copy: `service_workflow.run` is org-scoped and copied, but the workflow `definition` it points at is
+global and is not. The target usually has those already — every environment's base DB descends from the
+same production dump — but a base DB is a point-in-time snapshot while the source keeps moving. Production
+has 819 workflow definitions; a base DB built in April 2023 has 689, so a 2026 org's runs pointed at
+versions that were not there.
+
+So the export carries the exact parent rows the org's rows reach, transitively (a carried `definition`
+points at a `workflow` in turn), and the import inserts only what is missing:
 
 ```
-  44 row(s) reference a parent that was not copied:
-      42  service_workflow.run.definitionId → service_workflow.definition  (parent table not in schema map)
-       2  service_accounts.systemMessageRecipient.systemMessageId → service_accounts.systemMessage
+  referenced rows carried: 9 from 2 table(s), 8 inserted (the rest the target already had)
 ```
 
-`parent table not in schema map` means the map does not carry that parent at all, so it recurs on every
-copy until the map changes. The copy is complete and usable; those references dangle.
+Their ids are **not** remapped — the copied rows reference them verbatim.
+
+Two tables are never carried: **`service_accounts.users`** (it would move real names and personnummer from
+the source environment into the target — exactly what a live→staging copy must not do) and
+**`service_accounts.companies`** (the organization itself). The closure is capped at 50,000 rows, and both
+the cap and any composite-key foreign key it declines to follow are reported as warnings.
+
+Anything that still dangles is reported after the load, because the bulk load runs with foreign-key
+enforcement off and Postgres never re-checks afterwards — so a dangling row would otherwise commit
+silently:
+
+```
+  44 row(s) still reference a parent that is not there:
+      42  service_workflow.run.definitionId → service_workflow.definition
+```
 
 Naming: with **neither `--name` nor `--target`** the copy creates a new org that **keeps the source org's
 own name** (its `short_name` still gets a unique suffix). Pass `--name` to rename it, or `--target <uuid>`