@dooer/dooer-test-env 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/index.js +7 -0
- package/discovery-router/Dockerfile +18 -0
- package/discovery-router/README.md +99 -0
- package/discovery-router/package.json +13 -0
- package/discovery-router/registry.example.json +5 -0
- package/discovery-router/server.js +272 -0
- package/lib/account.js +120 -0
- package/lib/auth-dev-keys.js +12 -0
- package/lib/bankid.js +130 -0
- package/lib/cli.js +27 -0
- package/lib/command/bankid.js +45 -0
- package/lib/command/customer.js +108 -0
- package/lib/command/db.js +156 -0
- package/lib/command/env.js +114 -0
- package/lib/command/logs.js +143 -0
- package/lib/command/measure.js +81 -0
- package/lib/command/service.js +166 -0
- package/lib/command/setup.js +92 -0
- package/lib/command/shred.js +60 -0
- package/lib/compose/README.md +98 -0
- package/lib/compose/generate.js +375 -0
- package/lib/compose/manifests.js +108 -0
- package/lib/db/roles.js +118 -0
- package/lib/discovery/client.js +40 -0
- package/lib/engine/GUIDE.md +176 -0
- package/lib/engine/PROCESS.md +571 -0
- package/lib/engine/dbbuild.js +325 -0
- package/lib/engine/gen-schema-map.js +479 -0
- package/lib/engine/purge.js +137 -0
- package/lib/engine/schema-map.json +11016 -0
- package/lib/engine/seed.js +1045 -0
- package/lib/obc.js +72 -0
- package/lib/registry.js +123 -0
- package/lib/runtime.js +101 -0
- package/lib/service-token.js +40 -0
- package/lib/shred/README.md +118 -0
- package/lib/shred/audit.js +128 -0
- package/lib/shred/faker.js +545 -0
- package/lib/shred/index.js +126 -0
- package/lib/shred/scripts/base-partner-emails.sql +9 -0
- package/lib/shred/scripts/dev-accounts.sql +195 -0
- package/lib/shred/scripts/emails.sql +48 -0
- package/lib/shred/scripts/institution-browser.sql +3 -0
- package/lib/shred/scripts/notification-targets.sql +5 -0
- package/lib/shred/scripts/partners.sql +2 -0
- package/lib/shred/scripts/passwords.sql +8 -0
- package/lib/shred/scripts/personal-numbers.sql +177 -0
- package/lib/shred/scripts/phone-numbers.sql +22 -0
- package/lib/shred/scripts/salary-spec-reports.sql +5 -0
- package/lib/shred/scripts/service-activity-tracker-data.sql +4 -0
- package/lib/shred/scripts/service-core-objects.sql +19 -0
- package/lib/shred/scripts/service-event-stream.sql +2 -0
- package/lib/shred/scripts/service-integrations.sql +4 -0
- package/lib/shred/scripts/template.sql +4 -0
- package/lib/shred/scripts/x-service-billing.sql +34 -0
- package/lib/shred/scripts/xxx-history-tables.sql +25 -0
- package/lib/stub.js +8 -0
- package/local-postgres/Dockerfile +11 -0
- package/package.json +46 -0
- package/readme.md +92 -0
package/lib/bankid.js
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
// BankID config for the local env — pulled from staging, stored in the macOS keychain, injected into the
|
|
2
|
+
// service-accounts container at `up` time.
|
|
3
|
+
//
|
|
4
|
+
// Why: service-accounts runs @dooer/config, where `bankIdIsProduction` defaults to true but is forced
|
|
5
|
+
// FALSE for the `local` environment — so with DOOER_ENVIRONMENT_NAME=local the bankid library falls back to
|
|
6
|
+
// its built-in TEST cert + https://appapi2.test.bankid.com. Staging (s-e032, no override) runs PRODUCTION
|
|
7
|
+
// BankID with the real client PFX from the `s-e032-service-accounts` k8s secret. To log in on the local HQ
|
|
8
|
+
// exactly like staging we need that PFX/passphrase AND the env override DOOER_BANK_ID_IS_PRODUCTION=true
|
|
9
|
+
// (a @dooer/config env override wins over the per-environment value).
|
|
10
|
+
//
|
|
11
|
+
// Storage: the secrets live in the macOS keychain (never in the generated compose file or a plaintext env
|
|
12
|
+
// file). At `up` the values are read from the keychain into the docker-compose child env and interpolated
|
|
13
|
+
// into service-accounts' `${SECRET_DOOER_BANK_ID_*}` / `${DOOER_BANK_ID_IS_PRODUCTION}` references only.
|
|
14
|
+
|
|
15
|
+
const { spawnSync } = require('child_process')
|
|
16
|
+
|
|
17
|
+
// keychain coordinates
|
|
18
|
+
const KEYCHAIN_ACCOUNT = 'dooer-test-env'
|
|
19
|
+
// the three secret keys service-accounts reads (names = @dooer/config's derived env vars, matching the
|
|
20
|
+
// staging manifest). CA cert is unused by the current login code path but pulled for parity with staging.
|
|
21
|
+
const SECRET_KEYS = [
|
|
22
|
+
'SECRET_DOOER_BANK_ID_PFX',
|
|
23
|
+
'SECRET_DOOER_BANK_ID_PFX_PASSPHRASE',
|
|
24
|
+
'SECRET_DOOER_BANK_ID_CA_CERTIFICATE',
|
|
25
|
+
]
|
|
26
|
+
const DEFAULT_NAMESPACE = 'dooer-staging'
|
|
27
|
+
const DEFAULT_SECRET = 's-e032-service-accounts'
|
|
28
|
+
|
|
29
|
+
function sh(cmd, args, input) {
|
|
30
|
+
const r = spawnSync(cmd, args, { encoding: 'utf8', input })
|
|
31
|
+
return { status: r.status, stdout: (r.stdout || '').trim(), stderr: (r.stderr || '').trim() }
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function keychainService(key) {
|
|
35
|
+
return `dooer-test-env-bankid:${key}`
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Read one secret from the keychain (null if absent). `security -w` prints the password as a lowercase hex
|
|
39
|
+
// string for longer values (observed for the multi-KB PFX + CA cert) and verbatim for short ones (the
|
|
40
|
+
// passphrase). Our real values are base64/PEM/text — none are pure lowercase hex — so when the output IS
|
|
41
|
+
// pure even-length lowercase hex we decode it back. (Missing this returned the hex text as the PFX →
|
|
42
|
+
// createSecureContext "header too long".)
|
|
43
|
+
function keychainGet(key) {
|
|
44
|
+
const r = sh('security', ['find-generic-password', '-a', KEYCHAIN_ACCOUNT, '-s', keychainService(key), '-w'])
|
|
45
|
+
if (r.status !== 0) return null
|
|
46
|
+
const out = r.stdout
|
|
47
|
+
if (out && out.length % 2 === 0 && /^[0-9a-f]+$/.test(out)) return Buffer.from(out, 'hex').toString('utf8')
|
|
48
|
+
return out
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Store one secret (upsert: -U replaces an existing item). Value passed via -w (argv) — acceptable for a
|
|
52
|
+
// dev tool on a single-user machine; the keychain is the durable store.
|
|
53
|
+
function keychainSet(key, value) {
|
|
54
|
+
const r = sh('security', [
|
|
55
|
+
'add-generic-password',
|
|
56
|
+
'-U',
|
|
57
|
+
'-a',
|
|
58
|
+
KEYCHAIN_ACCOUNT,
|
|
59
|
+
'-s',
|
|
60
|
+
keychainService(key),
|
|
61
|
+
'-w',
|
|
62
|
+
value,
|
|
63
|
+
])
|
|
64
|
+
if (r.status !== 0) throw new Error(`keychain store failed for ${key}: ${r.stderr}`)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function keychainDelete(key) {
|
|
68
|
+
return sh('security', ['delete-generic-password', '-a', KEYCHAIN_ACCOUNT, '-s', keychainService(key)]).status === 0
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Pull the bankid secrets out of the staging k8s secret via kubectl. k8s stores each value base64-encoded
|
|
72
|
+
// in `.data`; we decode to the raw value the staging container would receive (for the PFX that raw value is
|
|
73
|
+
// itself the base64 PFX string, exactly what @dooer/config's `Buffer.from(pfx,'base64')` expects).
|
|
74
|
+
function pullFromStaging({ namespace = DEFAULT_NAMESPACE, secret = DEFAULT_SECRET } = {}) {
|
|
75
|
+
const r = sh('kubectl', ['-n', namespace, 'get', 'secret', secret, '-o', 'json'])
|
|
76
|
+
if (r.status !== 0) throw new Error(`kubectl get secret ${namespace}/${secret} failed: ${r.stderr || r.stdout}`)
|
|
77
|
+
const data = JSON.parse(r.stdout).data || {}
|
|
78
|
+
const out = {}
|
|
79
|
+
SECRET_KEYS.forEach((key) => {
|
|
80
|
+
if (data[key] == null) return // CA cert may be absent in some envs — skip quietly
|
|
81
|
+
out[key] = Buffer.from(data[key], 'base64').toString('utf8')
|
|
82
|
+
})
|
|
83
|
+
if (!out.SECRET_DOOER_BANK_ID_PFX) {
|
|
84
|
+
throw new Error(`secret ${namespace}/${secret} has no SECRET_DOOER_BANK_ID_PFX — wrong secret?`)
|
|
85
|
+
}
|
|
86
|
+
return out
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Pull + store in the keychain. Returns the list of keys stored (never the values).
|
|
90
|
+
function pullAndStore(opts = {}) {
|
|
91
|
+
const values = pullFromStaging(opts)
|
|
92
|
+
const stored = []
|
|
93
|
+
Object.entries(values).forEach(([key, value]) => {
|
|
94
|
+
keychainSet(key, value)
|
|
95
|
+
stored.push(key)
|
|
96
|
+
})
|
|
97
|
+
return stored
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Which secrets are present in the keychain.
|
|
101
|
+
function status() {
|
|
102
|
+
return SECRET_KEYS.map((key) => ({ key, present: keychainGet(key) != null }))
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function clear() {
|
|
106
|
+
return SECRET_KEYS.filter((key) => keychainDelete(key))
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// The env map to inject into the service-accounts container at `up`. Empty (→ test-mode fallback) when the
|
|
110
|
+
// PFX isn't in the keychain; otherwise the certs + DOOER_BANK_ID_IS_PRODUCTION=true (matching staging).
|
|
111
|
+
function loadEnv() {
|
|
112
|
+
const env = {}
|
|
113
|
+
SECRET_KEYS.forEach((key) => {
|
|
114
|
+
const v = keychainGet(key)
|
|
115
|
+
if (v != null) env[key] = v
|
|
116
|
+
})
|
|
117
|
+
if (env.SECRET_DOOER_BANK_ID_PFX) env.DOOER_BANK_ID_IS_PRODUCTION = 'true'
|
|
118
|
+
return env
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
module.exports = {
|
|
122
|
+
SECRET_KEYS,
|
|
123
|
+
DEFAULT_NAMESPACE,
|
|
124
|
+
DEFAULT_SECRET,
|
|
125
|
+
pullFromStaging,
|
|
126
|
+
pullAndStore,
|
|
127
|
+
status,
|
|
128
|
+
clear,
|
|
129
|
+
loadEnv,
|
|
130
|
+
}
|
package/lib/cli.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
const yargs = require('yargs')
|
|
2
|
+
const { hideBin } = require('yargs/helpers')
|
|
3
|
+
const pkg = require('../package.json')
|
|
4
|
+
|
|
5
|
+
// One CLI for the local test environment: bring up the full backend locally (staging DB minus
|
|
6
|
+
// customers), copy/purge customers between environments, and shred — see ENVIRONMENT-PLAN.md.
|
|
7
|
+
// Command groups are wired as modules under ./command.
|
|
8
|
+
module.exports = async function cli() {
|
|
9
|
+
return yargs(hideBin(process.argv))
|
|
10
|
+
.scriptName('dooer-test-env')
|
|
11
|
+
.usage('$0 <group> <command> [options]')
|
|
12
|
+
.version(pkg.version)
|
|
13
|
+
.command(require('./command/setup')) // prerequisites + registry login + compose generation
|
|
14
|
+
.command(require('./command/env')) // up / down / start / stop / status
|
|
15
|
+
.command(require('./command/service')) // service start|stop|restart|deploy|local|unlocal|version
|
|
16
|
+
.command(require('./command/db')) // db build|pull|snapshot|rollback
|
|
17
|
+
.command(require('./command/customer')) // customer copy|purge
|
|
18
|
+
.command(require('./command/shred')) // shred (localhost only)
|
|
19
|
+
.command(require('./command/bankid')) // bankid pull|status|clear (staging certs → keychain)
|
|
20
|
+
.command(require('./command/logs')) // search logs across all local services (transactionId, errors, …)
|
|
21
|
+
.command(require('./command/measure')) // resource-usage report for the running env
|
|
22
|
+
.demandCommand(1, 'Pick a command group. Try --help.')
|
|
23
|
+
.strict()
|
|
24
|
+
.help()
|
|
25
|
+
.wrap(Math.min(120, yargs().terminalWidth()))
|
|
26
|
+
.parseAsync()
|
|
27
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
const chalk = require('chalk')
|
|
2
|
+
const bankid = require('../bankid')
|
|
3
|
+
|
|
4
|
+
// BankID config for the local env. `pull` copies the client PFX/passphrase (+ CA cert) out of the staging
|
|
5
|
+
// k8s secret into the macOS keychain; `up` then injects them into service-accounts so the local BankID flow
|
|
6
|
+
// behaves like staging (production BankID) instead of the library's built-in test cert. See lib/bankid.js.
|
|
7
|
+
module.exports = {
|
|
8
|
+
command: 'bankid <command>',
|
|
9
|
+
describe: 'BankID certs for local login: pull from staging → keychain (used by service-accounts)',
|
|
10
|
+
builder: (y) =>
|
|
11
|
+
y
|
|
12
|
+
.command({
|
|
13
|
+
command: 'pull',
|
|
14
|
+
describe: 'pull the BankID PFX/passphrase from the staging secret into the keychain',
|
|
15
|
+
builder: (y2) =>
|
|
16
|
+
y2
|
|
17
|
+
.option('namespace', { type: 'string', default: bankid.DEFAULT_NAMESPACE })
|
|
18
|
+
.option('secret', { type: 'string', default: bankid.DEFAULT_SECRET }),
|
|
19
|
+
handler: (argv) => {
|
|
20
|
+
const stored = bankid.pullAndStore({ namespace: argv.namespace, secret: argv.secret })
|
|
21
|
+
console.log(chalk.green(`bankid: stored ${stored.length} secret(s) in the keychain: ${stored.join(', ')}`))
|
|
22
|
+
console.log(chalk.gray('run `dooer-test-env up` (or restart service-accounts) to apply — production BankID.'))
|
|
23
|
+
},
|
|
24
|
+
})
|
|
25
|
+
.command({
|
|
26
|
+
command: 'status',
|
|
27
|
+
describe: 'show which BankID secrets are in the keychain',
|
|
28
|
+
handler: () => {
|
|
29
|
+
bankid.status().forEach((s) => {
|
|
30
|
+
console.log(`${s.present ? chalk.green('✓') : chalk.gray('·')} ${s.key}`)
|
|
31
|
+
})
|
|
32
|
+
},
|
|
33
|
+
})
|
|
34
|
+
.command({
|
|
35
|
+
command: 'clear',
|
|
36
|
+
describe: 'remove the BankID secrets from the keychain (reverts to test mode)',
|
|
37
|
+
handler: () => {
|
|
38
|
+
const removed = bankid.clear()
|
|
39
|
+
console.log(`bankid: removed ${removed.length} secret(s) from the keychain`)
|
|
40
|
+
},
|
|
41
|
+
})
|
|
42
|
+
.demandCommand(1)
|
|
43
|
+
.strict(),
|
|
44
|
+
handler: () => {},
|
|
45
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
const chalk = require('chalk')
|
|
2
|
+
const engine = require('../engine/seed')
|
|
3
|
+
const { purge } = require('../engine/purge')
|
|
4
|
+
const { createAccount, SUBSCRIPTION_TYPES } = require('../account')
|
|
5
|
+
|
|
6
|
+
// Map yargs values back to the engine's own flag argv, then let the engine's proven parser apply its
|
|
7
|
+
// defaults + validation. Keeps ONE source of truth for option semantics (lib/engine/seed.js).
|
|
8
|
+
function toEngineArgv(argv) {
|
|
9
|
+
const a = []
|
|
10
|
+
const flag = (name, val) => {
|
|
11
|
+
if (val !== undefined && val !== null && val !== false) a.push(`--${name}`, String(val))
|
|
12
|
+
}
|
|
13
|
+
flag('source', argv.source)
|
|
14
|
+
flag('target', argv.target)
|
|
15
|
+
flag('name', argv.name)
|
|
16
|
+
flag('owner-user', argv.ownerUser)
|
|
17
|
+
flag('source-namespace', argv.sourceNamespace)
|
|
18
|
+
flag('target-namespace', argv.targetNamespace)
|
|
19
|
+
flag('email', argv.email)
|
|
20
|
+
flag('if-target-nonempty', argv.ifTargetNonempty)
|
|
21
|
+
flag('salt', argv.salt)
|
|
22
|
+
if (argv.only) a.push('--only', [].concat(argv.only).join(','))
|
|
23
|
+
if (argv.skipTables) a.push('--skip-tables', [].concat(argv.skipTables).join(','))
|
|
24
|
+
if (argv.execute) a.push('--execute')
|
|
25
|
+
if (argv.skipFiles) a.push('--skip-files')
|
|
26
|
+
if (argv.skipUsers) a.push('--skip-users')
|
|
27
|
+
if (argv.confirmProduction) a.push('--confirm-production')
|
|
28
|
+
return a
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const copyOptions = (y) =>
|
|
32
|
+
y
|
|
33
|
+
.option('source', { type: 'string', demandOption: true, describe: 'org (companies_pk) to copy FROM' })
|
|
34
|
+
.option('target', { type: 'string', describe: 'existing org to copy INTO' })
|
|
35
|
+
.option('name', { type: 'string', describe: 'create a NEW target org with this name (needs --owner-user)' })
|
|
36
|
+
.option('owner-user', { type: 'string', describe: 'existing user (users_pk) to own a newly-created org' })
|
|
37
|
+
.option('source-namespace', { type: 'string', default: 'dooer-staging', describe: 'k8s namespace to read from' })
|
|
38
|
+
.option('target-namespace', { type: 'string', describe: 'k8s namespace to write to (default = source)' })
|
|
39
|
+
.option('email', { type: 'string', describe: 'address to scrub emails to (default testcustomer@dooer.com)' })
|
|
40
|
+
.option('if-target-nonempty', {
|
|
41
|
+
choices: ['refuse', 'insert'],
|
|
42
|
+
describe: 'behaviour if a copy table already has target rows (default refuse)',
|
|
43
|
+
})
|
|
44
|
+
.option('only', { type: 'string', describe: 'copy ONLY these tables (comma-separated); pass-1 still scans all' })
|
|
45
|
+
.option('skip-tables', { type: 'string', describe: 'copy all tables EXCEPT these (comma-separated)' })
|
|
46
|
+
.option('salt', { type: 'string', describe: 'extra entropy for the deterministic remap salt (rarely needed)' })
|
|
47
|
+
.option('skip-files', { type: 'boolean', describe: 'skip the S3 file copy' })
|
|
48
|
+
.option('skip-users', { type: 'boolean', describe: 'skip copying referenced users missing from the target' })
|
|
49
|
+
.option('confirm-production', { type: 'boolean', describe: 'REQUIRED to write to dooer-production' })
|
|
50
|
+
.option('execute', { type: 'boolean', default: false, describe: 'actually write (default: dry-run)' })
|
|
51
|
+
|
|
52
|
+
module.exports = {
|
|
53
|
+
command: 'customer <command>',
|
|
54
|
+
describe: 'copy or purge a customer organization between environments',
|
|
55
|
+
builder: (y) =>
|
|
56
|
+
y
|
|
57
|
+
.command({
|
|
58
|
+
command: 'copy',
|
|
59
|
+
describe: 'copy one org into another (dry-run by default; emails anonymized)',
|
|
60
|
+
builder: copyOptions,
|
|
61
|
+
handler: (argv) => engine.main(engine.parseArgs(toEngineArgv(argv))),
|
|
62
|
+
})
|
|
63
|
+
.command({
|
|
64
|
+
command: 'purge',
|
|
65
|
+
describe: 'delete a single org’s data (inverse of copy; files included). Dry-run by default.',
|
|
66
|
+
builder: (y2) =>
|
|
67
|
+
y2
|
|
68
|
+
.option('org', { type: 'string', demandOption: true, describe: 'org (companies_pk) to delete' })
|
|
69
|
+
.option('namespace', { type: 'string', default: 'dooer-staging', describe: 'k8s namespace' })
|
|
70
|
+
.option('skip-files', { type: 'boolean', describe: 'do not delete the org’s S3 objects' })
|
|
71
|
+
.option('confirm-production', { type: 'boolean', describe: 'REQUIRED to delete from dooer-production' })
|
|
72
|
+
.option('execute', { type: 'boolean', default: false, describe: 'actually delete (default: dry-run)' }),
|
|
73
|
+
handler: (argv) =>
|
|
74
|
+
purge({
|
|
75
|
+
org: argv.org,
|
|
76
|
+
namespace: argv.namespace,
|
|
77
|
+
skipFiles: argv.skipFiles,
|
|
78
|
+
confirmProduction: argv.confirmProduction,
|
|
79
|
+
execute: argv.execute,
|
|
80
|
+
}),
|
|
81
|
+
})
|
|
82
|
+
.command({
|
|
83
|
+
command: 'new <name>',
|
|
84
|
+
describe:
|
|
85
|
+
'create an empty functional test account (company + Owner + Ghost-Inspector subscriptions) in the LOCAL DB',
|
|
86
|
+
builder: (y2) =>
|
|
87
|
+
y2
|
|
88
|
+
.positional('name', { type: 'string', describe: 'company name for the new account' })
|
|
89
|
+
.option('owner', { type: 'string', demandOption: true, describe: 'user (users_pk) to set as Owner' })
|
|
90
|
+
.option('port', { type: 'number', default: 55432, describe: 'local Postgres port' })
|
|
91
|
+
.option('execute', { type: 'boolean', default: false, describe: 'actually create (default: dry-run)' }),
|
|
92
|
+
handler: async (argv) => {
|
|
93
|
+
const local = { host: 'localhost', port: argv.port, user: 'dooer', password: 'dooer', database: 'dooer' }
|
|
94
|
+
const res = await createAccount({ local, name: argv.name, ownerUserId: argv.owner, execute: argv.execute })
|
|
95
|
+
if (argv.execute) {
|
|
96
|
+
console.log(chalk.green(`\ncreated account "${res.name}"`))
|
|
97
|
+
console.log(` org id: ${res.orgId}`)
|
|
98
|
+
console.log(` short name: ${res.shortName}`)
|
|
99
|
+
console.log(` owner: ${argv.owner} (${res.owner}) — role Owner`)
|
|
100
|
+
console.log(` partner: ${res.partner}`)
|
|
101
|
+
console.log(` subscriptions: ${SUBSCRIPTION_TYPES.join(', ')} (active)`)
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
})
|
|
105
|
+
.demandCommand(1, 'Use: customer copy | customer purge | customer new')
|
|
106
|
+
.strict(),
|
|
107
|
+
handler: () => {},
|
|
108
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
const os = require('os')
|
|
2
|
+
const path = require('path')
|
|
3
|
+
const chalk = require('chalk')
|
|
4
|
+
const dbbuild = require('../engine/dbbuild')
|
|
5
|
+
const obc = require('../obc')
|
|
6
|
+
const { shred } = require('../shred')
|
|
7
|
+
const { auditPii } = require('../shred/audit')
|
|
8
|
+
const { ensureRoles } = require('../db/roles')
|
|
9
|
+
|
|
10
|
+
// Base DB: the customer-free, anonymized artifact and local restore points. `db build` runs ONLY as an
|
|
11
|
+
// in-cluster k8s Job (devs never run it); devs `db pull`. See ENVIRONMENT-PLAN.md §5.
|
|
12
|
+
|
|
13
|
+
// shredFn injected into build(): shred the scratch DB, then FAIL the build if the PII audit finds any
|
|
14
|
+
// uncovered PII column (the drift backstop).
|
|
15
|
+
async function shredAndAudit(client) {
|
|
16
|
+
await shred(client, { execute: true, verbose: true })
|
|
17
|
+
const a = await auditPii(client)
|
|
18
|
+
if (a.uncovered.length) {
|
|
19
|
+
a.uncovered.forEach((u) => console.error(` UNCOVERED ${u.category}: ${u.schema}.${u.table}.${u.column}`))
|
|
20
|
+
throw new Error(
|
|
21
|
+
`PII audit failed: ${a.uncovered.length} uncovered column(s) — add rules before shipping a base artifact`
|
|
22
|
+
)
|
|
23
|
+
}
|
|
24
|
+
console.log(`PII audit clean (covered ${a.covered}, scanned ${a.scanned}).`)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
module.exports = {
|
|
28
|
+
command: 'db <command>',
|
|
29
|
+
describe: 'base DB: build (in-cluster), pull, and local snapshots',
|
|
30
|
+
builder: (y) =>
|
|
31
|
+
y
|
|
32
|
+
.command({
|
|
33
|
+
command: 'build',
|
|
34
|
+
describe: 'in-cluster: copy → purge customers → shred → audit → publish base artifact (OBC)',
|
|
35
|
+
builder: (y2) =>
|
|
36
|
+
y2
|
|
37
|
+
.option('execute', { type: 'boolean', default: false })
|
|
38
|
+
.option('obc-namespace', {
|
|
39
|
+
type: 'string',
|
|
40
|
+
describe: 'namespace of the OBC (for upload); omit in-cluster (env-provided)',
|
|
41
|
+
})
|
|
42
|
+
.option('allow', { type: 'string', describe: 'comma-separated org uuids to spare from the purge' }),
|
|
43
|
+
handler: async (argv) => {
|
|
44
|
+
const src = {
|
|
45
|
+
host: process.env.DOOER_SQL_HOST,
|
|
46
|
+
port: Number(process.env.DOOER_SQL_PORT || 5432),
|
|
47
|
+
user: process.env.DOOER_SQL_MIGRATION_USER || process.env.DOOER_SQL_APPLICATION_USER || 'postgres',
|
|
48
|
+
password:
|
|
49
|
+
process.env.SECRET_DOOER_SQL_MIGRATION_PASSWORD ||
|
|
50
|
+
process.env.DOOER_SQL_MIGRATION_PASSWORD ||
|
|
51
|
+
process.env.DOOER_SQL_APPLICATION_PASSWORD,
|
|
52
|
+
database: 'dooer',
|
|
53
|
+
}
|
|
54
|
+
if (argv.execute && !src.host) {
|
|
55
|
+
throw new Error(
|
|
56
|
+
'db build runs in-cluster: DOOER_SQL_HOST (+ creds) must be set. Devs use `db pull`, not `db build`.'
|
|
57
|
+
)
|
|
58
|
+
}
|
|
59
|
+
// scratch = a SEPARATE postgres (the CronJob's sidecar at localhost), never dooer-database
|
|
60
|
+
const scratch = {
|
|
61
|
+
host: process.env.DOOER_SCRATCH_HOST || 'localhost',
|
|
62
|
+
port: Number(process.env.DOOER_SCRATCH_PORT || 5432),
|
|
63
|
+
user: process.env.DOOER_SCRATCH_USER || 'postgres',
|
|
64
|
+
password: process.env.DOOER_SCRATCH_PASSWORD || 'postgres',
|
|
65
|
+
}
|
|
66
|
+
const artifactOut = path.join(os.tmpdir(), `dooer-base-db-${new Date().toISOString().slice(0, 10)}.dump`)
|
|
67
|
+
await dbbuild.build({
|
|
68
|
+
source: src,
|
|
69
|
+
scratch,
|
|
70
|
+
artifactOut,
|
|
71
|
+
shredFn: shredAndAudit,
|
|
72
|
+
upload: async (file) => {
|
|
73
|
+
const cfg = obc.obcConfig({ namespace: argv.obcNamespace })
|
|
74
|
+
const key = await obc.upload(cfg, file)
|
|
75
|
+
console.log(`uploaded base artifact → ${cfg.bucket}/${key}`)
|
|
76
|
+
},
|
|
77
|
+
execute: argv.execute,
|
|
78
|
+
})
|
|
79
|
+
},
|
|
80
|
+
})
|
|
81
|
+
.command({
|
|
82
|
+
command: 'pull',
|
|
83
|
+
aliases: ['bootstrap'],
|
|
84
|
+
describe: 'fetch the newest base artifact and restore it into the LOCAL Postgres',
|
|
85
|
+
builder: (y2) =>
|
|
86
|
+
y2
|
|
87
|
+
.option('execute', { type: 'boolean', default: false })
|
|
88
|
+
.option('obc-namespace', {
|
|
89
|
+
type: 'string',
|
|
90
|
+
default: 'dooer-staging',
|
|
91
|
+
describe: 'namespace of the OBC to read the artifact from',
|
|
92
|
+
})
|
|
93
|
+
.option('port', { type: 'number', default: 55432 }),
|
|
94
|
+
handler: async (argv) => {
|
|
95
|
+
const local = { host: 'localhost', port: argv.port, user: 'dooer', password: 'dooer', database: 'dooer' }
|
|
96
|
+
await dbbuild.pull({
|
|
97
|
+
local,
|
|
98
|
+
download: async () => {
|
|
99
|
+
const cfg = obc.obcConfig({ namespace: argv.obcNamespace })
|
|
100
|
+
const out = path.join(os.tmpdir(), 'dooer-base-db.pulled.dump')
|
|
101
|
+
const { file, key } = await obc.download(cfg, out)
|
|
102
|
+
console.log(chalk.gray(`downloaded ${cfg.bucket}/${key}`))
|
|
103
|
+
return file
|
|
104
|
+
},
|
|
105
|
+
execute: argv.execute,
|
|
106
|
+
})
|
|
107
|
+
if (argv.execute) {
|
|
108
|
+
// Recreate the per-service schema grants + login roles + default search_path that staging has
|
|
109
|
+
// (the dump carries no cluster-global roles). Without them unqualified queries 42P01. See db/roles.js.
|
|
110
|
+
const { total, applied } = await ensureRoles({ local })
|
|
111
|
+
console.log(chalk.gray(`db roles: ${applied.length}/${total} service roles provisioned`))
|
|
112
|
+
}
|
|
113
|
+
},
|
|
114
|
+
})
|
|
115
|
+
.command({
|
|
116
|
+
command: 'roles',
|
|
117
|
+
describe: 'create/repair per-service schema grants + login roles + search_path (re-runnable)',
|
|
118
|
+
builder: (y2) => y2.option('port', { type: 'number', default: 55432 }),
|
|
119
|
+
handler: async (argv) => {
|
|
120
|
+
const local = { host: 'localhost', port: argv.port, user: 'dooer', password: 'dooer', database: 'dooer' }
|
|
121
|
+
const { total, applied } = await ensureRoles({ local })
|
|
122
|
+
console.log(`db roles: ${applied.length}/${total} service roles provisioned`)
|
|
123
|
+
},
|
|
124
|
+
})
|
|
125
|
+
.command({
|
|
126
|
+
command: 'snapshot <name>',
|
|
127
|
+
describe: 'save a local restore point (pg_dump of the local DB)',
|
|
128
|
+
handler: (argv) => {
|
|
129
|
+
const { spawnSync } = require('child_process')
|
|
130
|
+
const file = path.join(os.homedir(), '.dooer-test-env', `snap-${argv.name}.dump`)
|
|
131
|
+
require('fs').mkdirSync(path.dirname(file), { recursive: true })
|
|
132
|
+
const r = spawnSync('pg_dump', ['-Fc', '-d', 'postgresql://dooer:dooer@localhost:55432/dooer', '-f', file], {
|
|
133
|
+
stdio: 'inherit',
|
|
134
|
+
})
|
|
135
|
+
process.exitCode = r.status || 0
|
|
136
|
+
if (!r.status) console.log(`saved ${file}`)
|
|
137
|
+
},
|
|
138
|
+
})
|
|
139
|
+
.command({
|
|
140
|
+
command: 'rollback <name>',
|
|
141
|
+
describe: 'restore a local snapshot (or the base) into the local DB',
|
|
142
|
+
handler: (argv) => {
|
|
143
|
+
const { spawnSync } = require('child_process')
|
|
144
|
+
const file = path.join(os.homedir(), '.dooer-test-env', `snap-${argv.name}.dump`)
|
|
145
|
+
const r = spawnSync(
|
|
146
|
+
'pg_restore',
|
|
147
|
+
['--no-owner', '--clean', '--if-exists', '-d', 'postgresql://dooer:dooer@localhost:55432/dooer', file],
|
|
148
|
+
{ stdio: 'inherit' }
|
|
149
|
+
)
|
|
150
|
+
process.exitCode = r.status || 0
|
|
151
|
+
},
|
|
152
|
+
})
|
|
153
|
+
.demandCommand(1)
|
|
154
|
+
.strict(),
|
|
155
|
+
handler: () => {},
|
|
156
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
const fs = require('fs')
|
|
2
|
+
const chalk = require('chalk')
|
|
3
|
+
const rt = require('../runtime')
|
|
4
|
+
const registry = require('../registry')
|
|
5
|
+
const bankid = require('../bankid')
|
|
6
|
+
const { generateCompose } = require('../compose/generate')
|
|
7
|
+
const discovery = require('../discovery/client')
|
|
8
|
+
|
|
9
|
+
// Environment lifecycle for the whole local stack. The compose file is generated from the s-e032 k8s
|
|
10
|
+
// manifests (lib/compose) on first `up`; the discovery-router wires services together. Default profile is
|
|
11
|
+
// `full` (everything staging runs) — compose profiles are additive, so we must name it explicitly.
|
|
12
|
+
// See ENVIRONMENT-PLAN.md §6.
|
|
13
|
+
|
|
14
|
+
function ensureCompose() {
|
|
15
|
+
rt.ensureRunDir()
|
|
16
|
+
if (!fs.existsSync(rt.COMPOSE_FILE)) {
|
|
17
|
+
const r = generateCompose({ servicesDir: rt.SERVICES_DIR, out: rt.COMPOSE_FILE })
|
|
18
|
+
console.log(chalk.gray(`generated ${rt.COMPOSE_FILE} (${(r && r.serviceCount) || 'n'} services)`))
|
|
19
|
+
}
|
|
20
|
+
return rt.COMPOSE_FILE
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
module.exports = [
|
|
24
|
+
{
|
|
25
|
+
command: 'up',
|
|
26
|
+
describe: 'create + start the local stack (Postgres, Redis, Kafka, MinIO, discovery router, services)',
|
|
27
|
+
builder: (y) =>
|
|
28
|
+
y
|
|
29
|
+
.option('profile', {
|
|
30
|
+
type: 'string',
|
|
31
|
+
default: 'full',
|
|
32
|
+
describe: 'compose profile (default: full = all staging services; e.g. booking)',
|
|
33
|
+
})
|
|
34
|
+
.option('namespace', {
|
|
35
|
+
type: 'string',
|
|
36
|
+
default: 'dooer-staging',
|
|
37
|
+
describe: 'namespace to read registry pull secrets from',
|
|
38
|
+
})
|
|
39
|
+
.option('regenerate', { type: 'boolean', describe: 'regenerate the compose file from the current manifests' }),
|
|
40
|
+
handler: async (argv) => {
|
|
41
|
+
if (argv.regenerate && fs.existsSync(rt.COMPOSE_FILE)) fs.rmSync(rt.COMPOSE_FILE)
|
|
42
|
+
ensureCompose()
|
|
43
|
+
// ensure docker can pull the private images (no manual login needed); non-fatal so an already
|
|
44
|
+
// logged-in / offline-images run still proceeds.
|
|
45
|
+
try {
|
|
46
|
+
const logins = await registry.loginAll({ namespace: argv.namespace, interactive: false })
|
|
47
|
+
if (logins.some((l) => !l.ok))
|
|
48
|
+
console.log(chalk.yellow('warning: a registry login failed; `dooer-test-env setup` can fix it'))
|
|
49
|
+
} catch (e) {
|
|
50
|
+
console.log(
|
|
51
|
+
chalk.yellow(
|
|
52
|
+
`warning: registry login skipped (${e.message}). Run \`dooer-test-env setup\` if image pulls fail.`
|
|
53
|
+
)
|
|
54
|
+
)
|
|
55
|
+
}
|
|
56
|
+
// Inject BankID certs from the keychain (if pulled) so service-accounts runs production BankID like
|
|
57
|
+
// staging; absent → service-accounts falls back to the library's test cert. Never written to disk.
|
|
58
|
+
const bankidEnv = bankid.loadEnv()
|
|
59
|
+
if (bankidEnv.SECRET_DOOER_BANK_ID_PFX)
|
|
60
|
+
console.log(chalk.gray('bankid: injecting production certs from keychain'))
|
|
61
|
+
const code = rt.dc(['up', '-d'], { profile: argv.profile, env: bankidEnv })
|
|
62
|
+
if (code === 0) {
|
|
63
|
+
console.log(chalk.green(`\nenv up (profile: ${argv.profile}).`))
|
|
64
|
+
console.log('Next: `dooer-test-env db pull` to load the customer-free base DB, then `dooer-test-env status`.')
|
|
65
|
+
}
|
|
66
|
+
process.exitCode = code
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
command: 'stop',
|
|
71
|
+
describe: 'stop the running env (preserve containers/volumes)',
|
|
72
|
+
handler: () => {
|
|
73
|
+
process.exitCode = rt.dc(['stop'])
|
|
74
|
+
},
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
command: 'start',
|
|
78
|
+
describe: 'start a previously-stopped env (not recreate)',
|
|
79
|
+
handler: () => {
|
|
80
|
+
process.exitCode = rt.dc(['start'])
|
|
81
|
+
},
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
command: 'down',
|
|
85
|
+
describe: 'stop and remove the env',
|
|
86
|
+
builder: (y) => y.option('volumes', { type: 'boolean', describe: 'also remove volumes (wipes the local DB)' }),
|
|
87
|
+
handler: (argv) => {
|
|
88
|
+
process.exitCode = rt.dc(['down', ...(argv.volumes ? ['-v'] : [])])
|
|
89
|
+
},
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
command: 'status',
|
|
93
|
+
aliases: ['ps'],
|
|
94
|
+
describe: 'list services: running/stopped, and published-image vs local checked-out code',
|
|
95
|
+
handler: async () => {
|
|
96
|
+
rt.dc(['ps'])
|
|
97
|
+
// annotate which services are currently served from local checked-out code (router overrides)
|
|
98
|
+
const state = rt.readState()
|
|
99
|
+
const locals = Object.keys(state.local || {}).filter((n) => rt.isAlive(state.local[n].pid))
|
|
100
|
+
if (locals.length) {
|
|
101
|
+
console.log(chalk.yellow('\nRunning from LOCAL checked-out code (via discovery router):'))
|
|
102
|
+
for (const n of locals)
|
|
103
|
+
console.log(` ${n} → host.docker.internal:${state.local[n].port} (pid ${state.local[n].pid})`)
|
|
104
|
+
}
|
|
105
|
+
try {
|
|
106
|
+
const reg = await discovery.list(rt.ROUTER_URL)
|
|
107
|
+
const overrides = Object.keys((reg && reg.overrides) || {})
|
|
108
|
+
if (overrides.length) console.log(chalk.gray(`\nrouter overrides: ${overrides.join(', ')}`))
|
|
109
|
+
} catch (_) {
|
|
110
|
+
/* router not up */
|
|
111
|
+
}
|
|
112
|
+
},
|
|
113
|
+
},
|
|
114
|
+
]
|