@dooer/dooer-test-env 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/index.js +7 -0
- package/discovery-router/Dockerfile +18 -0
- package/discovery-router/README.md +99 -0
- package/discovery-router/package.json +13 -0
- package/discovery-router/registry.example.json +5 -0
- package/discovery-router/server.js +272 -0
- package/lib/account.js +120 -0
- package/lib/auth-dev-keys.js +12 -0
- package/lib/bankid.js +130 -0
- package/lib/cli.js +27 -0
- package/lib/command/bankid.js +45 -0
- package/lib/command/customer.js +108 -0
- package/lib/command/db.js +156 -0
- package/lib/command/env.js +114 -0
- package/lib/command/logs.js +143 -0
- package/lib/command/measure.js +81 -0
- package/lib/command/service.js +166 -0
- package/lib/command/setup.js +92 -0
- package/lib/command/shred.js +60 -0
- package/lib/compose/README.md +98 -0
- package/lib/compose/generate.js +375 -0
- package/lib/compose/manifests.js +108 -0
- package/lib/db/roles.js +118 -0
- package/lib/discovery/client.js +40 -0
- package/lib/engine/GUIDE.md +176 -0
- package/lib/engine/PROCESS.md +571 -0
- package/lib/engine/dbbuild.js +325 -0
- package/lib/engine/gen-schema-map.js +479 -0
- package/lib/engine/purge.js +137 -0
- package/lib/engine/schema-map.json +11016 -0
- package/lib/engine/seed.js +1045 -0
- package/lib/obc.js +72 -0
- package/lib/registry.js +123 -0
- package/lib/runtime.js +101 -0
- package/lib/service-token.js +40 -0
- package/lib/shred/README.md +118 -0
- package/lib/shred/audit.js +128 -0
- package/lib/shred/faker.js +545 -0
- package/lib/shred/index.js +126 -0
- package/lib/shred/scripts/base-partner-emails.sql +9 -0
- package/lib/shred/scripts/dev-accounts.sql +195 -0
- package/lib/shred/scripts/emails.sql +48 -0
- package/lib/shred/scripts/institution-browser.sql +3 -0
- package/lib/shred/scripts/notification-targets.sql +5 -0
- package/lib/shred/scripts/partners.sql +2 -0
- package/lib/shred/scripts/passwords.sql +8 -0
- package/lib/shred/scripts/personal-numbers.sql +177 -0
- package/lib/shred/scripts/phone-numbers.sql +22 -0
- package/lib/shred/scripts/salary-spec-reports.sql +5 -0
- package/lib/shred/scripts/service-activity-tracker-data.sql +4 -0
- package/lib/shred/scripts/service-core-objects.sql +19 -0
- package/lib/shred/scripts/service-event-stream.sql +2 -0
- package/lib/shred/scripts/service-integrations.sql +4 -0
- package/lib/shred/scripts/template.sql +4 -0
- package/lib/shred/scripts/x-service-billing.sql +34 -0
- package/lib/shred/scripts/xxx-history-tables.sql +25 -0
- package/lib/stub.js +8 -0
- package/local-postgres/Dockerfile +11 -0
- package/package.json +46 -0
- package/readme.md +92 -0
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
// Generate a docker-compose spec that runs the whole Dooer backend locally,
|
|
2
|
+
// wired together, from the s-e032 (staging) k8s manifests.
|
|
3
|
+
//
|
|
4
|
+
// Design notes live in ./README.md. In short: images are taken VERBATIM from the
|
|
5
|
+
// manifests (two registries, each service pulls from the right one); every
|
|
6
|
+
// cluster-specific env var (DB host, Consul, Redis, Kafka, S3, secrets) is
|
|
7
|
+
// overridden to point at the local infra containers this file also defines.
|
|
8
|
+
|
|
9
|
+
const fs = require('fs')
|
|
10
|
+
const path = require('path')
|
|
11
|
+
const yaml = require('js-yaml')
|
|
12
|
+
const { listServices, listFrontends } = require('./manifests')
|
|
13
|
+
const { AUTH_PUBLIC_KEY_B64, AUTH_PRIVATE_KEY_B64 } = require('../auth-dev-keys')
|
|
14
|
+
const { mint } = require('../service-token')
|
|
15
|
+
|
|
16
|
+
// PEM of the local dev private key, decoded once, to sign fresh per-service tokens (see below).
|
|
17
|
+
const AUTH_PRIVATE_KEY_PEM = Buffer.from(AUTH_PRIVATE_KEY_B64, 'base64').toString('utf8')
|
|
18
|
+
|
|
19
|
+
// The narrower `booking` profile — the minimum set to drive a booking end to end.
|
|
20
|
+
const BOOKING_PROFILE_SERVICES = [
|
|
21
|
+
'service-graphql',
|
|
22
|
+
'service-core-objects',
|
|
23
|
+
'service-accounts',
|
|
24
|
+
'service-authorization',
|
|
25
|
+
'service-ledger',
|
|
26
|
+
'service-task-engine',
|
|
27
|
+
'service-file-storage',
|
|
28
|
+
'service-ai-agents',
|
|
29
|
+
'service-booking-engine',
|
|
30
|
+
'service-report-generator',
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
// Dev credentials for the local stack. Throwaway — this env never holds real
|
|
34
|
+
// customer data or real secrets.
|
|
35
|
+
const DEV = {
|
|
36
|
+
sqlUser: 'dooer',
|
|
37
|
+
sqlPassword: 'dooer',
|
|
38
|
+
minioUser: 'minioadmin',
|
|
39
|
+
minioPassword: 'minioadmin',
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Applied to EVERY dooer service, last, so they win over the manifest values.
|
|
43
|
+
// These repoint the service at the local infra containers.
|
|
44
|
+
const GLOBAL_OVERRIDES = {
|
|
45
|
+
// Database → local postgres container, static-password mode.
|
|
46
|
+
DOOER_SQL_HOST: 'postgres',
|
|
47
|
+
DOOER_SQL_PORT: '5432',
|
|
48
|
+
DOOER_SQL_DATABASE: 'dooer',
|
|
49
|
+
// NOTE: we do NOT override DOOER_SQL_APPLICATION_USER — each service keeps its OWN manifest user (e.g.
|
|
50
|
+
// "service-notifications"). @dooer/database stores a service's schema as a ROLE-LEVEL default search_path
|
|
51
|
+
// (ALTER USER <appUser> SET search_path TO <schema>,public — see lib/db/roles.js), so collapsing every
|
|
52
|
+
// service onto one shared user breaks every UNqualified query (background/doode tasks) with 42P01. The
|
|
53
|
+
// matching roles are created locally by `db roles` (run automatically after `db pull`). Only the password
|
|
54
|
+
// is forced to the shared dev secret; a service with no manifest user falls back to `dooer` (below).
|
|
55
|
+
DOOER_SQL_APPLICATION_PASSWORD: DEV.sqlPassword,
|
|
56
|
+
DOOER_SQL_MIGRATION_USER: DEV.sqlUser, // migrations run as the dooer superuser; db-migrate sets the schema itself
|
|
57
|
+
DOOER_SQL_MIGRATION_PASSWORD: DEV.sqlPassword,
|
|
58
|
+
DOOER_IS_ONEPLATFORMER: 'false',
|
|
59
|
+
// clear the RDS CA baked into the image, and disable node TLS verification: @dooer/database always
|
|
60
|
+
// connects with SSL and verifies against a baked CA, which rejects the local postgres self-signed cert
|
|
61
|
+
// (DEPTH_ZERO_SELF_SIGNED_CERT). Safe locally — everything is on the loopback compose network.
|
|
62
|
+
DOOER_SQL_SSL_CA: '',
|
|
63
|
+
NODE_TLS_REJECT_UNAUTHORIZED: '0',
|
|
64
|
+
// Discovery → the local mutable router (speaks the Consul health API).
|
|
65
|
+
CONSUL_HTTP_ADDR: 'discovery-router:8500',
|
|
66
|
+
DOOER_CONSUL_HOST: 'discovery-router',
|
|
67
|
+
// Redis → a plain single-node container (no sentinel, no auth locally).
|
|
68
|
+
DOOER_HOST_REDIS: 'redis:6379',
|
|
69
|
+
DOOER_REDIS_USE_TLS: 'false',
|
|
70
|
+
// Kafka / event-stream → the local single-node broker.
|
|
71
|
+
DOOER_KAFKA_BOOTSTRAP_BROKER: 'kafka:9092',
|
|
72
|
+
DOOER_STREAM_NAME: 'local-s01',
|
|
73
|
+
// S3 → local MinIO.
|
|
74
|
+
AWS_ENDPOINT_URL_S3: 'http://minio:9000',
|
|
75
|
+
// Boot / identity.
|
|
76
|
+
// NOTE: never set DOOER_MICROSERVICE_PORT — no staging manifest sets it, so every image defaults to 3000
|
|
77
|
+
// on its own. service-ai (python) reads it as a raw string and does `bind(('', port))`, which throws
|
|
78
|
+
// "an integer is required (got type str)" the moment we inject it. Let the images default.
|
|
79
|
+
NODE_ENV: 'development',
|
|
80
|
+
DOOER_ENVIRONMENT_NAME: 'local',
|
|
81
|
+
DOOER_LOG_LEVEL: 'informational',
|
|
82
|
+
// Auth: force the shared @dooer LOCAL dev PUBLIC key on every verifier. The manifests carry the STAGING
|
|
83
|
+
// public key as a plain value (which would win over config's `local` default), but the only signer —
|
|
84
|
+
// service-accounts — signs with the LOCAL private key, so verifiers must trust the matching LOCAL public
|
|
85
|
+
// key or every authenticated call 401s "invalid signature". service-accounts gets the matching private key
|
|
86
|
+
// via SERVICE_ENV_OVERRIDES. (authIssuer is https://example.com/ everywhere already.)
|
|
87
|
+
DOOER_AUTH_PUBLIC_KEY: AUTH_PUBLIC_KEY_B64,
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Manifest env vars that are meaningless (or actively wrong) locally and are
|
|
91
|
+
// removed rather than overridden.
|
|
92
|
+
const DROP_ENV_KEYS = [
|
|
93
|
+
'DOOER_REDIS_SENTINEL_MASTER', // no sentinel with a plain redis container
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
// Per-service env we add on top of the manifest. Values written as compose ${VAR} interpolation are filled
|
|
97
|
+
// in at `up` time from the child env (never stored in the yaml).
|
|
98
|
+
const SERVICE_ENV_OVERRIDES = {
|
|
99
|
+
// BankID: `dooer-test-env bankid pull` copies the staging PFX/passphrase into the keychain and `up`
|
|
100
|
+
// injects them here. Absent → the vars resolve empty and service-accounts falls back to the bankid
|
|
101
|
+
// library's built-in TEST cert (appapi2.test.bankid.com). Present → `up` also sets IS_PRODUCTION=true, so
|
|
102
|
+
// local login uses PRODUCTION BankID exactly like staging (@dooer/config env override beats the
|
|
103
|
+
// per-environment `local:false`). See lib/bankid.js.
|
|
104
|
+
'service-accounts': {
|
|
105
|
+
// service-accounts is the only token SIGNER — give it the LOCAL dev private key matching the public key
|
|
106
|
+
// forced on every verifier (GLOBAL_OVERRIDES.DOOER_AUTH_PUBLIC_KEY). Non-secret dev material.
|
|
107
|
+
SECRET_DOOER_AUTH_PRIVATE_KEY: AUTH_PRIVATE_KEY_B64,
|
|
108
|
+
// NB: the ${...} strings below are docker-compose interpolation refs, not JS template literals.
|
|
109
|
+
/* eslint-disable no-template-curly-in-string */
|
|
110
|
+
SECRET_DOOER_BANK_ID_PFX: '${SECRET_DOOER_BANK_ID_PFX:-}',
|
|
111
|
+
SECRET_DOOER_BANK_ID_PFX_PASSPHRASE: '${SECRET_DOOER_BANK_ID_PFX_PASSPHRASE:-}',
|
|
112
|
+
SECRET_DOOER_BANK_ID_CA_CERTIFICATE: '${SECRET_DOOER_BANK_ID_CA_CERTIFICATE:-}',
|
|
113
|
+
DOOER_BANK_ID_IS_PRODUCTION: '${DOOER_BANK_ID_IS_PRODUCTION:-false}',
|
|
114
|
+
/* eslint-enable no-template-curly-in-string */
|
|
115
|
+
},
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// Marker: this secret-backed env var must NOT be emitted at all (see below).
|
|
119
|
+
const OMIT = Symbol('omit-env')
|
|
120
|
+
|
|
121
|
+
// Dev value for a secret-backed (valueFrom) env var.
|
|
122
|
+
//
|
|
123
|
+
// Anything we can't map to a local infra credential is OMITTED entirely rather than emitted as an empty
|
|
124
|
+
// string. This matters: many services read a secret through @dooer/config, which supplies its OWN local
|
|
125
|
+
// default when the env var is UNSET (e.g. service-notifications' twilioAuthToken defaults to "notset"), but
|
|
126
|
+
// an empty-string env var COUNTS as set and CLOBBERS that default — so injecting `SECRET_...=""` turns a
|
|
127
|
+
// harmless "notset" into `new Twilio(sid, "")` → "password is required" at boot. Omitting lets the image's
|
|
128
|
+
// own default stand (Jimmy's rule 2026-09-01: a required var with no default gets a value; a var WITH a
|
|
129
|
+
// default is left alone). A dev who needs a real value exports it and we pass it through (`${NAME:-}` only
|
|
130
|
+
// when the host actually has it — handled in buildServiceEnv).
|
|
131
|
+
function secretDefault(name) {
|
|
132
|
+
if (name === 'DOOER_SQL_APPLICATION_PASSWORD') return DEV.sqlPassword
|
|
133
|
+
if (name === 'DOOER_SQL_MIGRATION_PASSWORD') return DEV.sqlPassword
|
|
134
|
+
if (name === 'SECRET_DOOER_REDIS_PASSWORD') return '' // local redis has no auth; empty is the correct value
|
|
135
|
+
if (/ACCESS_KEY_ID$/.test(name)) return DEV.minioUser
|
|
136
|
+
if (/SECRET_ACCESS_KEY$/.test(name)) return DEV.minioPassword
|
|
137
|
+
return OMIT
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Build the final env map for one dooer service.
|
|
141
|
+
function buildServiceEnv(svc) {
|
|
142
|
+
const env = {}
|
|
143
|
+
// 1. plain manifest values (verbatim).
|
|
144
|
+
Object.assign(env, svc.env)
|
|
145
|
+
// 2. secret-backed vars → dev substitutions (or OMIT to leave the image's own default in place).
|
|
146
|
+
svc.secretEnv.forEach((name) => {
|
|
147
|
+
const v = secretDefault(name)
|
|
148
|
+
if (v === OMIT) return
|
|
149
|
+
env[name] = v
|
|
150
|
+
})
|
|
151
|
+
// 3. global local overrides win.
|
|
152
|
+
Object.assign(env, GLOBAL_OVERRIDES)
|
|
153
|
+
// 3b. stateless services (service-graphql, service-pdf, …) declare no application user; keep the shared
|
|
154
|
+
// dooer superuser for them so they still connect. Services WITH a manifest user keep it (see GLOBAL_OVERRIDES).
|
|
155
|
+
if (!env.DOOER_SQL_APPLICATION_USER) env.DOOER_SQL_APPLICATION_USER = DEV.sqlUser
|
|
156
|
+
// 3c. services that use a service token get a FRESH one signed with the local dev key (the baked config
|
|
157
|
+
// default expired in 2019). service_id = the service's own name, so authz that keys on the caller works.
|
|
158
|
+
if (svc.secretEnv && svc.secretEnv.includes('SECRET_DOOER_SERVICE_TOKEN')) {
|
|
159
|
+
env.SECRET_DOOER_SERVICE_TOKEN = mint({
|
|
160
|
+
serviceId: env.DOOER_SERVICE_NAME || svc.name,
|
|
161
|
+
privateKeyPem: AUTH_PRIVATE_KEY_PEM,
|
|
162
|
+
})
|
|
163
|
+
}
|
|
164
|
+
// 4. drop the cluster-only leftovers.
|
|
165
|
+
DROP_ENV_KEYS.forEach((key) => {
|
|
166
|
+
delete env[key]
|
|
167
|
+
})
|
|
168
|
+
return env
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Which profiles a dooer service belongs to. Everything is in `full` (the
|
|
172
|
+
// default set the CLI's `up` activates); the booking subset is additionally in
|
|
173
|
+
// `booking`. See README for why bare `docker compose up` is not the "all" path.
|
|
174
|
+
function profilesFor(name) {
|
|
175
|
+
// the `hq` profile = the booking backend core (which service-graphql + HQ need) so HQ is functional
|
|
176
|
+
// without starting all ~85 images. frontend-hq adds itself to `hq` in buildCompose.
|
|
177
|
+
return BOOKING_PROFILE_SERVICES.includes(name) ? ['full', 'booking', 'hq'] : ['full']
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// The infra containers every profile needs (no `profiles:` → always started).
|
|
181
|
+
function infraServices() {
|
|
182
|
+
return {
|
|
183
|
+
postgres: {
|
|
184
|
+
// built from ../local-postgres — postgres:14 + a baked self-signed cert, started with ssl=on, because
|
|
185
|
+
// @dooer/database always connects with SSL (a plain postgres image → "server does not support SSL").
|
|
186
|
+
build: path.resolve(__dirname, '..', '..', 'local-postgres'),
|
|
187
|
+
command: [
|
|
188
|
+
'postgres',
|
|
189
|
+
'-c',
|
|
190
|
+
'ssl=on',
|
|
191
|
+
'-c',
|
|
192
|
+
'ssl_cert_file=/etc/postgresql/server.crt',
|
|
193
|
+
'-c',
|
|
194
|
+
'ssl_key_file=/etc/postgresql/server.key',
|
|
195
|
+
],
|
|
196
|
+
environment: {
|
|
197
|
+
POSTGRES_DB: 'dooer',
|
|
198
|
+
POSTGRES_USER: DEV.sqlUser,
|
|
199
|
+
POSTGRES_PASSWORD: DEV.sqlPassword,
|
|
200
|
+
},
|
|
201
|
+
// publish on 55432 to avoid a dev's local (brew) postgres squatting host 5432. Services still reach
|
|
202
|
+
// it as `postgres:5432` on the compose network; only host-side tools (db pull/shred) use 55432.
|
|
203
|
+
ports: ['55432:5432'],
|
|
204
|
+
volumes: ['postgres_data:/var/lib/postgresql/data'],
|
|
205
|
+
healthcheck: {
|
|
206
|
+
test: ['CMD-SHELL', `pg_isready -U ${DEV.sqlUser} -d dooer`],
|
|
207
|
+
interval: '5s',
|
|
208
|
+
timeout: '5s',
|
|
209
|
+
retries: 20,
|
|
210
|
+
},
|
|
211
|
+
restart: 'unless-stopped',
|
|
212
|
+
},
|
|
213
|
+
redis: {
|
|
214
|
+
image: 'redis:7-alpine',
|
|
215
|
+
ports: ['6379:6379'],
|
|
216
|
+
restart: 'unless-stopped',
|
|
217
|
+
},
|
|
218
|
+
// Single-node Redpanda: Kafka-API compatible, no ZooKeeper needed — much
|
|
219
|
+
// lighter than a Strimzi/Kafka + ZK pair on a laptop. Advertises as
|
|
220
|
+
// `kafka:9092` so services reach it under the overridden bootstrap broker.
|
|
221
|
+
kafka: {
|
|
222
|
+
image: 'docker.redpanda.com/redpandadata/redpanda:v24.1.7',
|
|
223
|
+
platform: 'linux/amd64',
|
|
224
|
+
command: [
|
|
225
|
+
'redpanda',
|
|
226
|
+
'start',
|
|
227
|
+
'--overprovisioned',
|
|
228
|
+
'--smp',
|
|
229
|
+
'1',
|
|
230
|
+
'--memory',
|
|
231
|
+
'1G',
|
|
232
|
+
'--reserve-memory',
|
|
233
|
+
'0M',
|
|
234
|
+
'--node-id',
|
|
235
|
+
'0',
|
|
236
|
+
'--check=false',
|
|
237
|
+
'--kafka-addr',
|
|
238
|
+
'PLAINTEXT://0.0.0.0:9092',
|
|
239
|
+
'--advertise-kafka-addr',
|
|
240
|
+
'PLAINTEXT://kafka:9092',
|
|
241
|
+
],
|
|
242
|
+
ports: ['9092:9092'],
|
|
243
|
+
volumes: ['kafka_data:/var/lib/redpanda/data'],
|
|
244
|
+
restart: 'unless-stopped',
|
|
245
|
+
},
|
|
246
|
+
minio: {
|
|
247
|
+
image: 'minio/minio:latest',
|
|
248
|
+
platform: 'linux/amd64',
|
|
249
|
+
command: ['server', '/data', '--console-address', ':9001'],
|
|
250
|
+
environment: {
|
|
251
|
+
MINIO_ROOT_USER: DEV.minioUser,
|
|
252
|
+
MINIO_ROOT_PASSWORD: DEV.minioPassword,
|
|
253
|
+
},
|
|
254
|
+
ports: ['9000:9000', '9001:9001'],
|
|
255
|
+
volumes: ['minio_data:/data'],
|
|
256
|
+
restart: 'unless-stopped',
|
|
257
|
+
},
|
|
258
|
+
// Mutable name→endpoint registry that speaks the Consul health API. Build context is an ABSOLUTE
|
|
259
|
+
// path to the repo's discovery-router, so it resolves no matter where the compose file is written
|
|
260
|
+
// (it lives in ~/.dooer-test-env, not the repo).
|
|
261
|
+
'discovery-router': {
|
|
262
|
+
build: path.resolve(__dirname, '..', '..', 'discovery-router'),
|
|
263
|
+
ports: ['8500:8500'],
|
|
264
|
+
restart: 'unless-stopped',
|
|
265
|
+
},
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// Host ports published so the BROWSER (and dev) can reach things on localhost.
|
|
270
|
+
const GATEWAY_HOST_PORT = 4000 // service-graphql (the public GraphQL facade the frontends' browsers call)
|
|
271
|
+
const HQ_HOST_PORT = 8080 // frontend-hq
|
|
272
|
+
|
|
273
|
+
// host port for a frontend: HQ is pinned to 8080; the rest get 8081+ in sorted order.
|
|
274
|
+
function frontendPort(name, index) {
|
|
275
|
+
return name === 'frontend-hq' ? HQ_HOST_PORT : 8081 + index
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// Turn the parsed manifests into the compose object.
|
|
279
|
+
function buildCompose(services, frontends, { profile } = {}) {
|
|
280
|
+
const wantedSvc = profile ? services.filter((s) => profilesFor(s.name).includes(profile)) : services
|
|
281
|
+
|
|
282
|
+
const composeServices = infraServices()
|
|
283
|
+
wantedSvc.forEach((svc) => {
|
|
284
|
+
composeServices[svc.name] = {
|
|
285
|
+
image: svc.image, // VERBATIM — registry preserved.
|
|
286
|
+
platform: 'linux/amd64', // cluster images are amd64-only; run under emulation on arm64 (Apple Silicon)
|
|
287
|
+
// Use each service's OWN manifest command (most are the OnePlatformer ./CMD-OPF.sh entrypoint —
|
|
288
|
+
// no Consul/Vault, runs db-migrate then starts; a few differ, e.g. service-ai=python, service-pdf=node).
|
|
289
|
+
// Fall back to ./CMD-OPF.sh when a manifest sets none, so we never hit the image default ./CMD.sh
|
|
290
|
+
// (Consul path) which hangs on consul-template.
|
|
291
|
+
command: svc.command || ['./CMD-OPF.sh'],
|
|
292
|
+
environment: { ...buildServiceEnv(svc), ...(SERVICE_ENV_OVERRIDES[svc.name] || {}) },
|
|
293
|
+
// publish the GraphQL gateway on the host so the frontends' browsers can reach it at localhost
|
|
294
|
+
...(svc.name === 'service-graphql' ? { ports: [`${GATEWAY_HOST_PORT}:3000`] } : {}),
|
|
295
|
+
depends_on: ['postgres', 'redis', 'discovery-router'],
|
|
296
|
+
profiles: profilesFor(svc.name),
|
|
297
|
+
restart: 'unless-stopped',
|
|
298
|
+
}
|
|
299
|
+
})
|
|
300
|
+
|
|
301
|
+
// frontends: served (SSR/static) on :3000, published to a host port. The BROWSER loads them from
|
|
302
|
+
// localhost and calls the GraphQL facade — repointed to the local gateway. Cross-frontend base URLs are
|
|
303
|
+
// repointed to each frontend's own localhost port where we run it (others stay as-is).
|
|
304
|
+
;(frontends || []).forEach((fe, i) => {
|
|
305
|
+
const hostPort = frontendPort(fe.name, i)
|
|
306
|
+
const env = buildServiceEnv(fe)
|
|
307
|
+
env.DOOER_PUBLIC_FACADE_HOST = `http://localhost:${GATEWAY_HOST_PORT}`
|
|
308
|
+
// dooer-frontend's config validation rejects an EMPTY sentryEndpoint for the `local` environment
|
|
309
|
+
// ("No value specified for [sentryEndpoint] ... (ENV: 'DOOER_SENTRY_ENDPOINT')"), even though staging
|
|
310
|
+
// ships it as "". We don't use Sentry (Jimmy 2026-09-01) — inject a well-formed but inert DSN so the
|
|
311
|
+
// client SDK initialises and any error POST just fails silently against the loopback host.
|
|
312
|
+
env.DOOER_SENTRY_ENDPOINT = 'https://local@localhost/0'
|
|
313
|
+
if (env.DOOER_PUBLIC_FRONTEND_BASE_URL) env.DOOER_PUBLIC_FRONTEND_BASE_URL = `http://localhost:${hostPort}`
|
|
314
|
+
composeServices[fe.name] = {
|
|
315
|
+
image: fe.image,
|
|
316
|
+
platform: 'linux/amd64',
|
|
317
|
+
...(fe.command ? { command: fe.command } : {}),
|
|
318
|
+
environment: env,
|
|
319
|
+
ports: [`${hostPort}:3000`],
|
|
320
|
+
depends_on: ['discovery-router'],
|
|
321
|
+
profiles: ['full', 'frontend', ...(fe.name === 'frontend-hq' ? ['hq'] : [])],
|
|
322
|
+
restart: 'unless-stopped',
|
|
323
|
+
}
|
|
324
|
+
})
|
|
325
|
+
|
|
326
|
+
return {
|
|
327
|
+
// no top-level `version:` — obsolete in Compose v2 and it prints a warning on every command
|
|
328
|
+
volumes: {
|
|
329
|
+
postgres_data: {},
|
|
330
|
+
minio_data: {},
|
|
331
|
+
kafka_data: {},
|
|
332
|
+
},
|
|
333
|
+
services: composeServices,
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// Public entry point. Reads every `service-*.yaml` + `frontend-*.yaml` in `servicesDir`, builds the
|
|
338
|
+
// compose object, writes it as YAML to `out`, and returns a small summary.
|
|
339
|
+
function generateCompose({ profile, servicesDir, out } = {}) {
|
|
340
|
+
const services = listServices(servicesDir)
|
|
341
|
+
const frontends = listFrontends(servicesDir)
|
|
342
|
+
const compose = buildCompose(services, frontends, { profile })
|
|
343
|
+
|
|
344
|
+
const yamlText = yaml.safeDump(compose, {
|
|
345
|
+
lineWidth: -1, // never wrap (long base64 public keys must stay one line)
|
|
346
|
+
noRefs: true,
|
|
347
|
+
sortKeys: false,
|
|
348
|
+
})
|
|
349
|
+
|
|
350
|
+
if (out) {
|
|
351
|
+
fs.mkdirSync(path.dirname(out), { recursive: true })
|
|
352
|
+
fs.writeFileSync(out, yamlText, 'utf8')
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
const dooerServiceCount = Object.keys(compose.services).length - Object.keys(infraServices()).length
|
|
356
|
+
return {
|
|
357
|
+
compose,
|
|
358
|
+
yaml: yamlText,
|
|
359
|
+
out: out || null,
|
|
360
|
+
dooerServiceCount,
|
|
361
|
+
infraServiceCount: Object.keys(infraServices()).length,
|
|
362
|
+
bookingProfileServices: BOOKING_PROFILE_SERVICES,
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
module.exports = {
|
|
367
|
+
BOOKING_PROFILE_SERVICES,
|
|
368
|
+
GLOBAL_OVERRIDES,
|
|
369
|
+
secretDefault,
|
|
370
|
+
buildServiceEnv,
|
|
371
|
+
profilesFor,
|
|
372
|
+
infraServices,
|
|
373
|
+
buildCompose,
|
|
374
|
+
generateCompose,
|
|
375
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// Read the staging k8s Deployment manifests and expose the bits the compose
|
|
2
|
+
// generator needs. Pure/deterministic: given a directory of `service-*.yaml`
|
|
3
|
+
// files it returns a stable, sorted list of { name, image, env, secretEnv, port }.
|
|
4
|
+
//
|
|
5
|
+
// A `service-*.yaml` file is a multi-document manifest (Deployment + Service, and
|
|
6
|
+
// sometimes an ObjectBucketClaim / CronJob / RBAC). We only care about the
|
|
7
|
+
// Deployment's single application container.
|
|
8
|
+
|
|
9
|
+
const fs = require('fs')
|
|
10
|
+
const path = require('path')
|
|
11
|
+
const yaml = require('js-yaml')
|
|
12
|
+
|
|
13
|
+
// Where the s-e032 (staging) manifests live by default. Callers should pass an
|
|
14
|
+
// explicit `servicesDir`; this is only the fallback for `imageFor(name)`.
|
|
15
|
+
const DEFAULT_SERVICES_DIR = path.resolve(
|
|
16
|
+
process.env.HOME || '',
|
|
17
|
+
'dooer/new-infrastructure/kubernetes/environments/s-e032-onprem'
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
const SERVICE_FILE_RE = /^service-.*\.yaml$/
|
|
21
|
+
const FRONTEND_FILE_RE = /^frontend-.*\.yaml$/
|
|
22
|
+
|
|
23
|
+
// Pull the application container out of a Deployment doc. Containers are named
|
|
24
|
+
// `node` across the fleet; fall back to the first container defensively.
|
|
25
|
+
function applicationContainer(deployment) {
|
|
26
|
+
const podSpec = deployment && deployment.spec && deployment.spec.template && deployment.spec.template.spec
|
|
27
|
+
const containers = (podSpec && podSpec.containers) || []
|
|
28
|
+
return containers.find((c) => c && c.name === 'node') || containers[0] || null
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Parse one `service-*.yaml`. Returns null for files that carry no Deployment
|
|
32
|
+
// (e.g. the e2e-nightly and token CronJobs) — they are not runnable services.
|
|
33
|
+
function parseManifestFile(filePath) {
|
|
34
|
+
const raw = fs.readFileSync(filePath, 'utf8')
|
|
35
|
+
const docs = yaml.safeLoadAll(raw).filter(Boolean)
|
|
36
|
+
const deployment = docs.find((doc) => doc && doc.kind === 'Deployment')
|
|
37
|
+
if (!deployment) return null
|
|
38
|
+
|
|
39
|
+
const container = applicationContainer(deployment)
|
|
40
|
+
if (!container || !container.image) return null
|
|
41
|
+
|
|
42
|
+
const name = (deployment.metadata && deployment.metadata.name) || path.basename(filePath, '.yaml')
|
|
43
|
+
|
|
44
|
+
// Split the env block into plain values (reproduced verbatim) and
|
|
45
|
+
// secret-backed vars (valueFrom → the generator substitutes dev values).
|
|
46
|
+
const env = {}
|
|
47
|
+
const secretEnv = []
|
|
48
|
+
const envEntries = container.env || []
|
|
49
|
+
envEntries.forEach((entry) => {
|
|
50
|
+
if (!entry || !entry.name) return
|
|
51
|
+
if (Object.prototype.hasOwnProperty.call(entry, 'value')) {
|
|
52
|
+
env[entry.name] = entry.value === null ? '' : String(entry.value)
|
|
53
|
+
} else if (entry.valueFrom) {
|
|
54
|
+
secretEnv.push(entry.name)
|
|
55
|
+
}
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
const ports = container.ports || []
|
|
59
|
+
const port = (ports[0] && Number(ports[0].containerPort)) || 3000
|
|
60
|
+
|
|
61
|
+
return { name, image: container.image, env, secretEnv, port, command: container.command || null }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// List every dooer service defined by a `service-*.yaml` Deployment in `dir`,
|
|
65
|
+
// sorted by filename for deterministic output.
|
|
66
|
+
function listServices(servicesDir) {
|
|
67
|
+
const dir = servicesDir || DEFAULT_SERVICES_DIR
|
|
68
|
+
return fs
|
|
69
|
+
.readdirSync(dir)
|
|
70
|
+
.filter((f) => SERVICE_FILE_RE.test(f))
|
|
71
|
+
.sort()
|
|
72
|
+
.map((f) => parseManifestFile(path.join(dir, f)))
|
|
73
|
+
.filter(Boolean)
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// List every frontend defined by a `frontend-*.yaml` Deployment in `dir`.
|
|
77
|
+
function listFrontends(servicesDir) {
|
|
78
|
+
const dir = servicesDir || DEFAULT_SERVICES_DIR
|
|
79
|
+
return fs
|
|
80
|
+
.readdirSync(dir)
|
|
81
|
+
.filter((f) => FRONTEND_FILE_RE.test(f))
|
|
82
|
+
.sort()
|
|
83
|
+
.map((f) => parseManifestFile(path.join(dir, f)))
|
|
84
|
+
.filter(Boolean)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Return the verbatim image ref for one service name (registry preserved — there
|
|
88
|
+
// are two registries in the fleet). Reads the single matching file when present,
|
|
89
|
+
// otherwise scans the directory.
|
|
90
|
+
function imageFor(name, servicesDir) {
|
|
91
|
+
const dir = servicesDir || DEFAULT_SERVICES_DIR
|
|
92
|
+
const direct = path.join(dir, `${name}.yaml`)
|
|
93
|
+
if (fs.existsSync(direct)) {
|
|
94
|
+
const parsed = parseManifestFile(direct)
|
|
95
|
+
if (parsed) return parsed.image
|
|
96
|
+
}
|
|
97
|
+
const match = listServices(dir).find((s) => s.name === name)
|
|
98
|
+
return match ? match.image : null
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
module.exports = {
|
|
102
|
+
DEFAULT_SERVICES_DIR,
|
|
103
|
+
SERVICE_FILE_RE,
|
|
104
|
+
parseManifestFile,
|
|
105
|
+
listServices,
|
|
106
|
+
listFrontends,
|
|
107
|
+
imageFor,
|
|
108
|
+
}
|
package/lib/db/roles.js
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
// Per-service Postgres roles + schema grants for the local env — a faithful copy of how the fleet is
|
|
2
|
+
// provisioned on staging (cli-cowboy's lib/service-db/setup-db-schema.js).
|
|
3
|
+
//
|
|
4
|
+
// Why this is needed: @dooer/database stores a service's schema as a ROLE-LEVEL default search_path
|
|
5
|
+
// ALTER USER <applicationUser> SET search_path TO <schema>,"public"
|
|
6
|
+
// (schema = DOOER_SERVICE_NAME with dashes→underscores). Staging runs the setup per service so each app
|
|
7
|
+
// user carries that default and the right per-schema GRANTs. The base dump we restore locally is a single
|
|
8
|
+
// database dump; roles are cluster-global and are NOT in it. If every service shares the throwaway `dooer`
|
|
9
|
+
// superuser (search_path `dooer,public`), unqualified queries (background/doode tasks, raw postgres-lib
|
|
10
|
+
// pools) fail with 42P01. So we recreate the fleet's role model here.
|
|
11
|
+
//
|
|
12
|
+
// The model (from cli-cowboy): a NOLOGIN group role `<schema>_access` owns the privileges; a shared
|
|
13
|
+
// `read-only` group gets SELECT; the service's LOGIN user gets the schema as its default search_path and
|
|
14
|
+
// membership in `<schema>_access`. We deliberately mirror this rather than granting SUPERUSER, so the local
|
|
15
|
+
// env exercises the same privilege boundaries as staging.
|
|
16
|
+
|
|
17
|
+
const { Client } = require('pg')
|
|
18
|
+
const { listServices, DEFAULT_SERVICES_DIR } = require('../compose/manifests')
|
|
19
|
+
|
|
20
|
+
const pgFriendly = (serviceName) => serviceName.replace(/-/g, '_')
|
|
21
|
+
const ident = (name) => `"${String(name).replace(/"/g, '""')}"`
|
|
22
|
+
const literal = (val) => `'${String(val).replace(/'/g, "''")}'`
|
|
23
|
+
|
|
24
|
+
// Derive [{ user, schema }] from the manifests: every service that declares an application user. Stateless
|
|
25
|
+
// services (service-graphql, service-pdf, …) declare none and keep the shared dooer user.
|
|
26
|
+
function roleSpecs(servicesDir) {
|
|
27
|
+
const seen = new Map()
|
|
28
|
+
listServices(servicesDir || DEFAULT_SERVICES_DIR).forEach((svc) => {
|
|
29
|
+
const user = svc.env.DOOER_SQL_APPLICATION_USER
|
|
30
|
+
if (!user || user === 'dooer') return
|
|
31
|
+
seen.set(user, pgFriendly(svc.env.DOOER_SERVICE_NAME || svc.name))
|
|
32
|
+
})
|
|
33
|
+
return [...seen].map(([user, schema]) => ({ user, schema }))
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// The schema + group-role setup, byte-for-byte the shape cli-cowboy applies (idempotent). Creates the
|
|
37
|
+
// schema (+_history), the `<schema>_access` and `read-only` group roles, and all the USAGE/DML/sequence
|
|
38
|
+
// grants + default privileges. Runs as the connecting superuser (dooer), which also owns the migrated
|
|
39
|
+
// tables, so the ALTER DEFAULT PRIVILEGES cover future dooer-run migrations too.
|
|
40
|
+
function schemaSetupSql(schema) {
|
|
41
|
+
return `
|
|
42
|
+
BEGIN TRANSACTION;
|
|
43
|
+
CREATE SCHEMA IF NOT EXISTS ${ident(schema)};
|
|
44
|
+
CREATE SCHEMA IF NOT EXISTS ${ident(`${schema}_history`)};
|
|
45
|
+
DO $body$ BEGIN
|
|
46
|
+
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = ${literal(`${schema}_access`)}) THEN
|
|
47
|
+
CREATE ROLE ${ident(`${schema}_access`)};
|
|
48
|
+
END IF;
|
|
49
|
+
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'read-only') THEN
|
|
50
|
+
CREATE ROLE "read-only";
|
|
51
|
+
END IF;
|
|
52
|
+
END $body$;
|
|
53
|
+
GRANT USAGE ON SCHEMA ${ident(schema)} TO "read-only";
|
|
54
|
+
GRANT USAGE ON SCHEMA ${ident(`${schema}_history`)} TO "read-only";
|
|
55
|
+
ALTER DEFAULT PRIVILEGES IN SCHEMA ${ident(schema)} GRANT SELECT ON TABLES TO "read-only";
|
|
56
|
+
ALTER DEFAULT PRIVILEGES IN SCHEMA ${ident(`${schema}_history`)} GRANT SELECT ON TABLES TO "read-only";
|
|
57
|
+
GRANT SELECT ON ALL TABLES IN SCHEMA ${ident(schema)} TO "read-only";
|
|
58
|
+
GRANT SELECT ON ALL TABLES IN SCHEMA ${ident(`${schema}_history`)} TO "read-only";
|
|
59
|
+
GRANT USAGE ON SCHEMA ${ident(schema)} TO ${ident(`${schema}_access`)};
|
|
60
|
+
GRANT USAGE ON SCHEMA ${ident(`${schema}_history`)} TO ${ident(`${schema}_access`)};
|
|
61
|
+
ALTER DEFAULT PRIVILEGES IN SCHEMA ${ident(schema)} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ${ident(
|
|
62
|
+
`${schema}_access`
|
|
63
|
+
)};
|
|
64
|
+
ALTER DEFAULT PRIVILEGES IN SCHEMA ${ident(
|
|
65
|
+
`${schema}_history`
|
|
66
|
+
)} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ${ident(`${schema}_access`)};
|
|
67
|
+
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ${ident(schema)} TO ${ident(`${schema}_access`)};
|
|
68
|
+
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ${ident(`${schema}_history`)} TO ${ident(
|
|
69
|
+
`${schema}_access`
|
|
70
|
+
)};
|
|
71
|
+
ALTER DEFAULT PRIVILEGES IN SCHEMA ${ident(schema)} GRANT USAGE, SELECT ON SEQUENCES TO ${ident(
|
|
72
|
+
`${schema}_access`
|
|
73
|
+
)};
|
|
74
|
+
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA ${ident(schema)} TO ${ident(`${schema}_access`)};
|
|
75
|
+
COMMIT TRANSACTION;
|
|
76
|
+
`
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// The LOGIN user, mirroring cli-cowboy's Vault creation_statements — but with a static name/password (the
|
|
80
|
+
// local env runs @dooer/database with isOneplatformer=false, i.e. plain user+password, not IAM tokens).
|
|
81
|
+
function loginUserSql(user, schema, password) {
|
|
82
|
+
return `
|
|
83
|
+
DO $body$ BEGIN
|
|
84
|
+
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = ${literal(user)}) THEN
|
|
85
|
+
CREATE ROLE ${ident(user)} WITH LOGIN PASSWORD ${literal(password)};
|
|
86
|
+
END IF;
|
|
87
|
+
END $body$;
|
|
88
|
+
ALTER ROLE ${ident(user)} WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE PASSWORD ${literal(password)};
|
|
89
|
+
ALTER USER ${ident(user)} SET "search_path" TO ${ident(schema)},"public";
|
|
90
|
+
GRANT ${ident(`${schema}_access`)} TO ${ident(user)};
|
|
91
|
+
`
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Create/repair every service's schema grants + login role on a connected client. Idempotent.
|
|
95
|
+
async function applyRoles(client, specs, { password = 'dooer' } = {}) {
|
|
96
|
+
const applied = []
|
|
97
|
+
for (const { user, schema } of specs) {
|
|
98
|
+
await client.query(schemaSetupSql(schema))
|
|
99
|
+
await client.query(loginUserSql(user, schema, password))
|
|
100
|
+
applied.push(user)
|
|
101
|
+
}
|
|
102
|
+
return { applied }
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Connect to the local Postgres and apply all derived roles. Used by `db pull` (post-restore) and `db roles`.
|
|
106
|
+
async function ensureRoles({ local, servicesDir, password } = {}) {
|
|
107
|
+
const specs = roleSpecs(servicesDir)
|
|
108
|
+
const client = new Client({ ...local, ssl: local.ssl === false ? undefined : { rejectUnauthorized: false } })
|
|
109
|
+
await client.connect()
|
|
110
|
+
try {
|
|
111
|
+
const res = await applyRoles(client, specs, { password: password || local.password || 'dooer' })
|
|
112
|
+
return { total: specs.length, ...res }
|
|
113
|
+
} finally {
|
|
114
|
+
await client.end()
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
module.exports = { pgFriendly, roleSpecs, schemaSetupSql, loginUserSql, applyRoles, ensureRoles }
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// Thin client for the local discovery-router admin API (see discovery-router/). The CLI uses this to
|
|
2
|
+
// repoint a service name as it starts / stops / goes local / switches version. Uses global fetch
|
|
3
|
+
// (Node 18+). `routerUrl` is the router base, e.g. 'http://localhost:8500'.
|
|
4
|
+
/* global fetch */
|
|
5
|
+
|
|
6
|
+
function base(routerUrl) {
|
|
7
|
+
return String(routerUrl).replace(/\/+$/, '')
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
async function asJson(response, action) {
|
|
11
|
+
if (!response.ok) {
|
|
12
|
+
const text = await response.text().catch(() => '')
|
|
13
|
+
throw new Error(`discovery-router ${action} failed: ${response.status} ${response.statusText} ${text}`.trim())
|
|
14
|
+
}
|
|
15
|
+
return response.json()
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// Set an override so peers resolve `name` to { address, port }.
|
|
19
|
+
async function setTarget(routerUrl, name, { address, port }) {
|
|
20
|
+
const response = await fetch(`${base(routerUrl)}/registry/${encodeURIComponent(name)}`, {
|
|
21
|
+
method: 'PUT',
|
|
22
|
+
headers: { 'content-type': 'application/json' },
|
|
23
|
+
body: JSON.stringify({ address, port }),
|
|
24
|
+
})
|
|
25
|
+
return asJson(response, `setTarget(${name})`)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Remove an override, reverting `name` to the router's default template.
|
|
29
|
+
async function clearTarget(routerUrl, name) {
|
|
30
|
+
const response = await fetch(`${base(routerUrl)}/registry/${encodeURIComponent(name)}`, { method: 'DELETE' })
|
|
31
|
+
return asJson(response, `clearTarget(${name})`)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// List the current registry: { overrides, effective, defaults }.
|
|
35
|
+
async function list(routerUrl) {
|
|
36
|
+
const response = await fetch(`${base(routerUrl)}/registry`)
|
|
37
|
+
return asJson(response, 'list')
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
module.exports = { setTarget, clearTarget, list }
|