@dooer/dooer-test-env 1.8.1 → 1.9.0

package/discovery-router/server.js

@@ -79,7 +79,53 @@ function consulNode(name, { address, port }) {
 // we grant every session and every lock acquire. This is the difference between a static Consul stub and
 // a working local env (without it, all scheduled/background handlers stay dormant).
 const sessions = new Set()
-const kv = new Map() // key -> { value, session }
+const kv = new Map() // key -> { value, session, index }
+// Consul BLOCKING QUERIES. doode's leader election long-polls `GET /v1/kv/<lock>?index=N&wait=180s` from
+// every service. Consul holds that request open until the key's ModifyIndex exceeds N (or `wait` elapses);
+// our first shim answered instantly with a constant ModifyIndex, so ~90 services re-polled in a tight
+// loop — which produced a 177 GB container log and burned CPU. Track a real index and park waiters.
+let kvIndex = 1
+const kvWaiters = new Map() // key -> Set<{ resolve }>
+
+function kvBump(key) {
+  kvIndex += 1
+  const entry = kv.get(key)
+  if (entry) entry.index = kvIndex
+  const waiters = kvWaiters.get(key)
+  if (waiters) {
+    for (const w of waiters) w.resolve()
+    waiters.clear()
+  }
+}
+
+// Wait until `key` changes or `waitMs` elapses. Resolves either way (the caller re-reads current state).
+function kvWait(key, waitMs) {
+  return new Promise((resolve) => {
+    let done = false
+    const finish = () => {
+      if (done) return
+      done = true
+      clearTimeout(timer)
+      const set = kvWaiters.get(key)
+      if (set) set.delete(waiter)
+      resolve()
+    }
+    const waiter = { resolve: finish }
+    const timer = setTimeout(finish, waitMs)
+    if (!kvWaiters.has(key)) kvWaiters.set(key, new Set())
+    kvWaiters.get(key).add(waiter)
+  })
+}
+
+// Consul durations: "180s", "5m", "1000ms". Capped so a client cannot park a socket forever.
+function parseWait(raw) {
+  if (!raw) return 0
+  const m = String(raw).match(/^(\d+)(ms|s|m)?$/)
+  if (!m) return 0
+  const n = Number(m[1])
+  const ms = m[2] === 'ms' ? n : m[2] === 'm' ? n * 60000 : n * 1000
+  return Math.min(ms, 600000)
+}
 
 function readRaw(req) {
   return new Promise((resolve, reject) => {
@@ -121,36 +167,52 @@ async function handleConsulLock(req, res, url) {
       const release = searchParams.get('release')
       if (release) {
         const cur = kv.get(key)
-        if (cur && cur.session === release) kv.set(key, { value: cur.value, session: null })
+        if (cur && cur.session === release) kv.set(key, { value: cur.value, session: null, index: kvIndex })
+        kvBump(key)
         sendJson(res, 200, true)
       } else {
         // single-node: always grant the lock (or plain write)
-        kv.set(key, { value, session: acquire || null })
+        kv.set(key, { value, session: acquire || null, index: kvIndex })
+        kvBump(key)
         sendJson(res, 200, true)
       }
       return true
     }
     if (method === 'GET') {
+      const askedIndex = Number(searchParams.get('index') || 0)
+      const waitMs = parseWait(searchParams.get('wait'))
+      // Blocking query: hold the request while the caller's index is current (this is what stops the
+      // leader-election long-poll from becoming a tight loop).
+      if (askedIndex > 0 && waitMs > 0) {
+        const seen = kv.get(key)
+        if ((seen ? seen.index : kvIndex) <= askedIndex) await kvWait(key, waitMs)
+      }
       const cur = kv.get(key)
+      const idx = cur ? cur.index || kvIndex : kvIndex
       if (!cur) {
-        sendJson(res, 404, [])
+        res.writeHead(404, { 'content-type': 'application/json', 'x-consul-index': String(idx) })
+        res.end('[]')
         return true
       }
-      sendJson(res, 200, [
-        {
-          Key: key,
-          Value: Buffer.from(cur.value || '').toString('base64'),
-          Session: cur.session || undefined,
-          CreateIndex: 1,
-          ModifyIndex: 1,
-          LockIndex: 1,
-          Flags: 0,
-        },
-      ])
+      res.writeHead(200, { 'content-type': 'application/json', 'x-consul-index': String(idx) })
+      res.end(
+        JSON.stringify([
+          {
+            Key: key,
+            Value: Buffer.from(cur.value || '').toString('base64'),
+            Session: cur.session || undefined,
+            CreateIndex: 1,
+            ModifyIndex: idx,
+            LockIndex: 1,
+            Flags: 0,
+          },
+        ])
+      )
       return true
     }
     if (method === 'DELETE') {
       kv.delete(key)
+      kvBump(key)
       sendJson(res, 200, true)
       return true
     }
@@ -257,9 +319,14 @@ async function handle(req, res) {
 
 const server = http.createServer((req, res) => {
   const start = Date.now()
-  res.on('finish', () =>
-    log('request', { method: req.method, url: req.url, status: res.statusCode, ms: Date.now() - start })
-  )
+  res.on('finish', () => {
+    // Skip the leader-election/discovery poll traffic: ~90 services poll continuously, and logging every
+    // hit is what grew this container's log to 177 GB. Errors and everything else still log.
+    const noisy = /^\/v1\/(kv|session|health)\//.test(req.url || '')
+    if (!noisy || res.statusCode >= 400) {
+      log('request', { method: req.method, url: req.url, status: res.statusCode, ms: Date.now() - start })
+    }
+  })
   handle(req, res).catch((error) => {
     log('handler-error', { url: req.url, error: error.message })
     if (!res.headersSent) sendJson(res, 500, { error: 'internal error' })

package/lib/command/env.js

@@ -63,6 +63,23 @@ module.exports = [
       if (bankidEnv.SECRET_DOOER_BANK_ID_PFX)
         console.log(chalk.gray('bankid: injecting production certs from keychain'))
       const code = rt.dc(['up', '-d'], { profile: argv.profile, env: bankidEnv })
+      // `up` starts EVERY service in the profile, including ones currently served by a `service local`
+      // host process — the resurrected container then competes with (and for peers, wins over) the local
+      // code. Stop those again so local mode survives an `up`.
+      const localEntries = Object.entries(rt.readState().local || {}).filter(([, info]) => rt.isAlive(info.pid))
+      const localNames = localEntries.map(([name]) => name)
+      if (localNames.length) {
+        rt.dc(['stop', ...localNames])
+        // The router keeps overrides in MEMORY, so a docker (or router) restart loses them and discovery
+        // falls back to the container name — which is stopped for a local service, making every call to it
+        // fail. Re-apply them here so `up` heals that instead of leaving a silently broken stack.
+        for (const [name, info] of localEntries) {
+          await discovery
+            .setTarget(rt.ROUTER_URL, name, { address: 'host.docker.internal', port: info.port })
+            .catch(() => {})
+        }
+        console.log(chalk.gray(`kept local: ${localNames.join(', ')} (containers stopped, router repointed)`))
+      }
       if (code === 0) {
         console.log(chalk.green(`\nenv up (profile: ${argv.profile}).`))
         console.log('Next: `dooer-test-env db pull` to load the customer-free base DB, then `dooer-test-env status`.')

package/lib/command/service.js

@@ -6,7 +6,8 @@ const chalk = require('chalk')
 const rt = require('../runtime')
 const discovery = require('../discovery/client')
 const bankid = require('../bankid')
-const { imageFor } = require('../compose/manifests')
+const { imageFor, listServices } = require('../compose/manifests')
+const { serviceHostPortMap } = require('../compose/generate')
 
 // Per-service control in a running env. `local`/`unlocal`/`deploy` update the discovery router so peers
 // pick up the change with no restarts (see ENVIRONMENT-PLAN.md §4/§6). Local host-process ports are
@@ -52,16 +53,45 @@ function localEnv(name, port) {
   env.DOOER_SERVICE_NAME = env.DOOER_SERVICE_NAME || name
   env.DOOER_ENVIRONMENT_NAME = env.DOOER_ENVIRONMENT_NAME || 'local'
   env.NODE_ENV = env.NODE_ENV || 'development'
+  // PEER calls: a host process cannot resolve container names (`service-core-objects:3000`) — nor container
+  // IPs, which Docker Desktop doesn't route on macOS. Every service is published on a deterministic host
+  // port (see generate.js serviceHostPortMap), so point @dooer/service at those, and at THIS service's own
+  // local port for self-calls. Without this a locally-run service can only talk to itself.
+  const hostPorts = serviceHostPortMap(listServices(rt.SERVICES_DIR))
+  const localPeers = rt.readState().local || {}
+  for (const [svcName, hostPort] of Object.entries(hostPorts)) {
+    // A peer that is ALSO running locally must be reached on ITS host process port, not the port its
+    // (stopped) container would publish — otherwise a local service-graphql calls the container copy of
+    // service-accounts and never sees your local changes.
+    const peer = localPeers[svcName]
+    const target = peer && rt.isAlive(peer.pid) ? peer.port : hostPort
+    env[`DOOER_HOST_${svcName.toUpperCase().replace(/-/g, '_')}`] = `localhost:${target}`
+  }
   // SELF-calls: the router advertises this service as host.docker.internal:<port> so CONTAINERS can reach
-  // the host process — but that name does not resolve ON the host, so the process fails calling itself
-  // (service-accounts → /api/v1/token-invalidation/all → ENOTFOUND, which broke every HQ navigation query).
-  // @dooer/service checks DOOER_HOST_<NAME> before Consul, so point the service at its own localhost port.
+  // the host process, but that name does not resolve ON the host (service-accounts →
+  // /api/v1/token-invalidation/all → ENOTFOUND, which broke every HQ navigation query).
   env[`DOOER_HOST_${name.toUpperCase().replace(/-/g, '_')}`] = `localhost:${port}`
+  // @dooer/logging: `logMode = DOOER_LOG || (process.stdout.isTTY ? 'pretty' : 'none')`. We redirect the
+  // host process to a log FILE, so stdout is not a TTY and it would log NOTHING — the images bake
+  // DOOER_LOG=json, which the manifests don't carry, so set it here too or `service local` runs blind.
+  env.DOOER_LOG = env.DOOER_LOG || 'json'
   return env
 }
 
+// Where a service's checked-out code lives. Nothing here is machine-specific: an explicit path wins, then
+// the ~/dooer/<service> convention, then the CURRENT directory if you happen to be standing in that
+// service's checkout (so `cd ~/src/service-ledger && … service local service-ledger` works too).
 function repoPath(name, given) {
-  return given || path.join(os.homedir(), 'dooer', name)
+  if (given) return path.resolve(given)
+  const conventional = path.join(os.homedir(), 'dooer', name)
+  if (fs.existsSync(path.join(conventional, 'package.json'))) return conventional
+  try {
+    const pkgName = require(path.join(process.cwd(), 'package.json')).name || ''
+    if (pkgName.replace(/^@[^/]+\//, '') === name) return process.cwd()
+  } catch (_) {
+    /* no readable package.json in cwd */
+  }
+  return conventional
 }
 
 module.exports = {

package/lib/compose/generate.js

@@ -126,6 +126,11 @@ const SERVICE_ENV_OVERRIDES = {
   },
 }
 
+// Cap EVERY container's log. The json-file driver is unbounded by default: the discovery-router alone grew
+// a 177 GB log file and filled the host disk (Jimmy 2026-09-03). 10 MB x 3 files per container bounds the
+// whole stack to a few GB worst case, and `logs`/`docker logs` still have plenty of history.
+const LOGGING = { driver: 'json-file', options: { 'max-size': '10m', 'max-file': '3' } }
+
 // Marker: this secret-backed env var must NOT be emitted at all (see below).
 const OMIT = Symbol('omit-env')
 
@@ -249,6 +254,7 @@ function infraServices() {
         timeout: '5s',
         retries: 20,
       },
+      logging: LOGGING,
       restart: 'unless-stopped',
     },
     redis: {
@@ -256,6 +262,7 @@ function infraServices() {
       // password-protected like staging — see DEV.redisPassword for why it matters (GraphQL subscriptions).
       command: ['redis-server', '--requirepass', DEV.redisPassword],
       ports: ['6379:6379'],
+      logging: LOGGING,
       restart: 'unless-stopped',
     },
     // Single-node Redpanda: Kafka-API compatible, no ZooKeeper needed — much
@@ -284,6 +291,7 @@ function infraServices() {
       ],
       ports: ['9092:9092'],
       volumes: ['kafka_data:/var/lib/redpanda/data'],
+      logging: LOGGING,
       restart: 'unless-stopped',
     },
     minio: {
@@ -296,6 +304,7 @@ function infraServices() {
       },
       ports: ['9000:9000', '9001:9001'],
       volumes: ['minio_data:/data'],
+      logging: LOGGING,
       restart: 'unless-stopped',
     },
     // One-shot: wait for MinIO, then create the buckets the services expect (idempotent). Runs on every `up`.
@@ -317,6 +326,7 @@ function infraServices() {
     'discovery-router': {
       build: path.resolve(__dirname, '..', '..', 'discovery-router'),
       ports: ['8500:8500'],
+      logging: LOGGING,
       restart: 'unless-stopped',
     },
   }
@@ -328,6 +338,25 @@ function infraServices() {
 // an internal backend it fronts — NOT the facade — so it gets its own port for direct dev access.
 const FACADE_HOST_PORT = 4000 // public-facade  → DOOER_PUBLIC_FACADE_HOST / DOOER_GRAPHQL_HOST
 const GRAPHQL_HOST_PORT = 4001 // service-graphql (direct access; frontends go through the facade)
+// EVERY backend service is also published on a deterministic host port. Containers reach each other by
+// container name, but a `service local` HOST process cannot resolve those names (nor container IPs, which
+// Docker Desktop doesn't route on macOS) — so without this a locally-run service can call itself but not
+// its peers. `service local` turns this map into DOOER_HOST_<NAME> overrides. (Jimmy 2026-09-03.)
+const SERVICE_HOST_PORT_BASE = 21000
+
+// {service name → published host port}, stable because listServices() is sorted by filename.
+function serviceHostPortMap(services) {
+  const map = {}
+  ;(services || []).forEach((svc, i) => {
+    map[svc.name] =
+      svc.name === 'public-facade'
+        ? FACADE_HOST_PORT
+        : svc.name === 'service-graphql'
+        ? GRAPHQL_HOST_PORT
+        : SERVICE_HOST_PORT_BASE + i
+  })
+  return map
+}
 const HQ_HOST_PORT = 8080 // frontend-hq
 
 // host port for a frontend: HQ is pinned to 8080; the rest get 8081+ in sorted order.
@@ -382,6 +411,7 @@ const SERVICE_ALIASES = {
 function buildCompose(services, frontends, { profile, outputValidation = false } = {}) {
   const wantedSvc = profile ? services.filter((s) => profilesFor(s.name).includes(profile)) : services
   const portMap = frontendPortMap(frontends)
+  const svcPortMap = serviceHostPortMap(services)
 
   const composeServices = infraServices()
   wantedSvc.forEach((svc) => {
@@ -398,13 +428,13 @@ function buildCompose(services, frontends, { profile, outputValidation = false }
         portMap
       ),
       // publish the browser-facing facade (4000) and service-graphql (4001) on the host
-      ...(svc.name === 'public-facade' ? { ports: [`${FACADE_HOST_PORT}:3000`] } : {}),
-      ...(svc.name === 'service-graphql' ? { ports: [`${GRAPHQL_HOST_PORT}:3000`] } : {}),
+      ports: [`${svcPortMap[svc.name]}:3000`],
       // legacy-name DNS aliases (e.g. service-periods → service-closing), so callers resolving the old name
       // reach this container on the compose network.
       ...(SERVICE_ALIASES[svc.name] ? { networks: { default: { aliases: SERVICE_ALIASES[svc.name] } } } : {}),
       depends_on: ['postgres', 'redis', 'discovery-router'],
       profiles: profilesFor(svc.name),
+      logging: LOGGING,
       restart: 'unless-stopped',
     }
   })
@@ -433,6 +463,7 @@ function buildCompose(services, frontends, { profile, outputValidation = false }
       ports: [`${hostPort}:3000`],
       depends_on: ['discovery-router'],
       profiles: ['full', 'frontend', ...(fe.name === 'frontend-hq' ? ['hq'] : [])],
+      logging: LOGGING,
       restart: 'unless-stopped',
     }
   })
@@ -479,6 +510,7 @@ function generateCompose({ profile, servicesDir, out, outputValidation = false }
 
 module.exports = {
   BOOKING_PROFILE_SERVICES,
+  serviceHostPortMap,
   VALIDATION_ENV,
   GLOBAL_OVERRIDES,
   secretDefault,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dooer/dooer-test-env",
-  "version": "1.8.1",
+  "version": "1.9.0",
   "description": "Run the whole Dooer backend locally (staging DB minus customers), copy/purge customers between environments, and shred — one CLI.",
   "license": "UNLICENSED",
   "repository": "Dooer/cli-dooer-test-env",

package/readme.md

@@ -51,19 +51,147 @@ service <start|stop|restart|deploy|local|unlocal|version> <name>   per-service c
 
 Everything that writes is **dry-run by default**; add `--execute`. Run any command with `--help`.
 
-### Examples
+## Use cases
+
+Everything below runs from **any directory** (`npx` resolves the published CLI). Nothing is tied to a
+particular machine: checkouts default to `~/dooer/<service>`, state lives in `~/.dooer-test-env`.
+
+### Where things are served
+
+| | |
+| --- | --- |
+| `http://localhost:4000` | **public-facade** — the browser-facing entrypoint (GraphQL + REST passthrough) |
+| `http://localhost:4001` | service-graphql (direct; frontends go via the facade) |
+| `http://localhost:8080` | frontend-hq |
+| `http://localhost:8082` | frontend-back-office-neue (GraphiQL at `/tools/graphiql`) |
+| `http://localhost:8089` | frontend-sumify-neue |
+| `localhost:55432` | Postgres (`dooer`/`dooer`) · `localhost:9001` MinIO console |
+| `localhost:21000+` | every backend service, one host port each (so host processes can reach them) |
+
+### Spin up an empty, ready-to-use customer
+
+```bash
+npx @dooer/dooer-test-env@latest customer new "Nordvik Handel AB" \
+  --owner <users_pk-of-a-CUSTOMER-user> --execute
+```
+
+Provisions through the product APIs (so `company-created` / `fiscal-year-created` fire and service-closing
+generates the accounting periods): company + Owner + address, the Ghost-Inspector subscription set, partner
+`dooer`, Terms of Service pre-accepted, the current-year fiscal year, and the temporal keys the platform
+reads (`fiscalYear`, `hasVatRegistration`, `hasCompanyTax`, `hasEmployeeRegistration`, `bookkeepingMethod`,
+`vatPeriod`/`vatDue`). `--no-fiscal-year` leaves all of that blank.
+
+The owner must be a **`customer`** user — a `hi`/admin owner cannot accept Terms of Service and the command
+warns you. Find one with:
+
+```sql
+SELECT users_pk, email FROM service_accounts.users
+WHERE fk_user_roles_at_dooer_pk = 'customer' AND inactivated_at IS NULL LIMIT 5;
+```
+
+### Copy a real customer into the local env
+
+`--target-namespace local` addresses this stack (Postgres on 55432 + MinIO); source can be any k8s
+namespace. Reading production needs no confirmation — only *writing* to it does. Emails are always
+anonymized. Dry-run by default; add `--execute`.
+
+```bash
+# copy Ghost Inspector out of LIVE into the local env, as a new org owned by a local user
+npx @dooer/dooer-test-env@latest customer copy \
+  --source e45a3bc4-1b61-4f55-9bd6-de2705420cc2 \
+  --source-namespace dooer-production \
+  --target-namespace local \
+  --name "Ghost inspector (live)" \
+  --owner-user <local-customer-users_pk> \
+  --execute
+
+# optional: shred the rest of the PII afterwards (localhost only)
+npx @dooer/dooer-test-env@latest shred --execute
+
+# undo: delete that org again (rows + its S3 objects), dry-run first
+npx @dooer/dooer-test-env@latest customer purge --org <orgId> --namespace local --execute
+```
+
+It scans all ~336 tables in the source before writing, so expect a few minutes on a production source.
+
+### Run your own code — one service, or several at once
+
+`service local` stops that service's container, starts `yarn start` from your checkout, and repoints the
+discovery router so **containers call your process**. Your process reaches the containerized services
+through their published host ports, so traffic flows **both ways**.
+
+The checkout is resolved as: explicit path → `~/dooer/<service>` → the current directory if you are standing
+in that service's checkout. So you do **not** need to `cd` anywhere:
 
 ```bash
-# an empty test account owned by a user, subscriptions like Ghost Inspector, under partner dooer
-npx @dooer/dooer-test-env customer new "Test Bolag AB" --owner 203b9010-abb9-4f7c-9a5e-8bbf6f75ca7b --execute
+npx @dooer/dooer-test-env@latest service local service-ledger              # uses ~/dooer/service-ledger
+npx @dooer/dooer-test-env@latest service local service-ledger ~/src/ledger # or an explicit path
+npx @dooer/dooer-test-env@latest service version service-ledger            # LOCAL code or which image?
+npx @dooer/dooer-test-env@latest service unlocal service-ledger            # back to the container image
+```
 
-# find where a transactionId showed up across the whole stack
-npx @dooer/dooer-test-env logs 1374f61b-5b42-48e6-9b28-ac086b55ea53
+Logs go to `~/.dooer-test-env/<service>.local.log`.
 
-# run one service from your checked-out code; peers follow via the discovery router
-npx @dooer/dooer-test-env service local service-ledger
+**Several services at once** — e.g. test a gateway change together with a backend change, no image builds:
+
+```bash
+npx @dooer/dooer-test-env@latest service local service-accounts   # start the backend first…
+npx @dooer/dooer-test-env@latest service local service-graphql    # …then the gateway
 ```
 
+Order matters: a local process learns its peers' addresses at startup, so start the service **being called**
+before the one calling it (or restart the caller afterwards).
+
+Now a single query through the facade spans everything — browser → public-facade (Docker) → service-graphql
+(your code) → service-accounts (your code) **and** service-core-objects (Docker):
+
+```graphql
+query VerifyLocalCode {
+  organization(id: "<orgId>") {
+    id
+    documents(first: 1) { totalCount }   # service-core-objects, in Docker
+  }
+}
+```
+
+Try it in back-office's GraphiQL at `http://localhost:8082/tools/graphiql`, or:
+
+```bash
+curl -s -X POST http://localhost:4000/graphql -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer <token>" \
+  --data '{"query":"query { organization(id:\"<orgId>\") { id documents(first:1){ totalCount } } }"}'
+```
+
+### Test a published build before it ships
+
+```bash
+npx @dooer/dooer-test-env@latest service deploy service-graphql 66.34.2   # pin an image tag
+```
+
+Pins it in `docker-compose.override.yml` and keeps `DOOER_SERVICE_VERSION` in sync so logs report the
+version actually running.
+
+### Find out what broke
+
+```bash
+npx @dooer/dooer-test-env@latest logs <transactionId>          # across ALL services at once
+npx @dooer/dooer-test-env@latest logs "error" --service ledger --since 30m -C 2
+```
+
+### Output-schema validation (off by default)
+
+Off means the local env behaves like staging/live, where these checks are silent. Turn it **on** to hunt
+stale service definitions — a consumer whose bundled definition predates a producer's added fields returns
+500 on perfectly valid data:
+
+```bash
+npx @dooer/dooer-test-env@latest validation on      # then reproduce; drift shows up as 500s
+npx @dooer/dooer-test-env@latest validation status
+npx @dooer/dooer-test-env@latest validation off
+```
+
+Inbound *input* validation always runs — this only affects response validation.
+
 ## How it works
 
 - **Compose generator** (`lib/compose/`): translates the s-e032 k8s manifests → a docker-compose spec —
@@ -82,6 +210,9 @@ npx @dooer/dooer-test-env service local service-ledger
   PII-audits, and uploads the artifact to an OBC bucket; devs only `db pull`. The CronJob + OBC manifests
   live with the other staging yaml in `new-infrastructure/kubernetes/environments/s-e032-onprem/`.
 
+- **Log caps**: every container is capped at `10m x 3` json-file logs. Unbounded logging once grew the
+  discovery-router's log to 179 GB and filled the host disk.
+
 ## Safety
 
 - `shred` from a dev machine is **localhost-only**.