@dooer/dooer-test-env 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/index.js +7 -0
- package/discovery-router/Dockerfile +18 -0
- package/discovery-router/README.md +99 -0
- package/discovery-router/package.json +13 -0
- package/discovery-router/registry.example.json +5 -0
- package/discovery-router/server.js +272 -0
- package/lib/account.js +120 -0
- package/lib/auth-dev-keys.js +12 -0
- package/lib/bankid.js +130 -0
- package/lib/cli.js +27 -0
- package/lib/command/bankid.js +45 -0
- package/lib/command/customer.js +108 -0
- package/lib/command/db.js +156 -0
- package/lib/command/env.js +114 -0
- package/lib/command/logs.js +143 -0
- package/lib/command/measure.js +81 -0
- package/lib/command/service.js +166 -0
- package/lib/command/setup.js +92 -0
- package/lib/command/shred.js +60 -0
- package/lib/compose/README.md +98 -0
- package/lib/compose/generate.js +375 -0
- package/lib/compose/manifests.js +108 -0
- package/lib/db/roles.js +118 -0
- package/lib/discovery/client.js +40 -0
- package/lib/engine/GUIDE.md +176 -0
- package/lib/engine/PROCESS.md +571 -0
- package/lib/engine/dbbuild.js +325 -0
- package/lib/engine/gen-schema-map.js +479 -0
- package/lib/engine/purge.js +137 -0
- package/lib/engine/schema-map.json +11016 -0
- package/lib/engine/seed.js +1045 -0
- package/lib/obc.js +72 -0
- package/lib/registry.js +123 -0
- package/lib/runtime.js +101 -0
- package/lib/service-token.js +40 -0
- package/lib/shred/README.md +118 -0
- package/lib/shred/audit.js +128 -0
- package/lib/shred/faker.js +545 -0
- package/lib/shred/index.js +126 -0
- package/lib/shred/scripts/base-partner-emails.sql +9 -0
- package/lib/shred/scripts/dev-accounts.sql +195 -0
- package/lib/shred/scripts/emails.sql +48 -0
- package/lib/shred/scripts/institution-browser.sql +3 -0
- package/lib/shred/scripts/notification-targets.sql +5 -0
- package/lib/shred/scripts/partners.sql +2 -0
- package/lib/shred/scripts/passwords.sql +8 -0
- package/lib/shred/scripts/personal-numbers.sql +177 -0
- package/lib/shred/scripts/phone-numbers.sql +22 -0
- package/lib/shred/scripts/salary-spec-reports.sql +5 -0
- package/lib/shred/scripts/service-activity-tracker-data.sql +4 -0
- package/lib/shred/scripts/service-core-objects.sql +19 -0
- package/lib/shred/scripts/service-event-stream.sql +2 -0
- package/lib/shred/scripts/service-integrations.sql +4 -0
- package/lib/shred/scripts/template.sql +4 -0
- package/lib/shred/scripts/x-service-billing.sql +34 -0
- package/lib/shred/scripts/xxx-history-tables.sql +25 -0
- package/lib/stub.js +8 -0
- package/local-postgres/Dockerfile +11 -0
- package/package.json +46 -0
- package/readme.md +92 -0
|
@@ -0,0 +1,1045 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/*
|
|
3
|
+
* seed-test-account.js — copy ALL customer-specific data for one Dooer organization into another.
|
|
4
|
+
*
|
|
5
|
+
* WHAT IT DOES
|
|
6
|
+
* Reads every org-scoped table (schema-map.json, action:"copy") for the SOURCE org and inserts the
|
|
7
|
+
* rows into the TARGET org, in the SAME database or ACROSS namespaces (e.g. read dooer-production,
|
|
8
|
+
* write dooer-staging). The org identity is one uuid (organizationId == companies.companies_pk); the
|
|
9
|
+
* source org uuid is remapped to the target org uuid, and every other copied uuid PK is remapped to a
|
|
10
|
+
* fresh uuid so intra-org references rewire automatically. References to non-copied rows (users,
|
|
11
|
+
* global templates) are left untouched. Emails are scrubbed so the source customer gets no mail.
|
|
12
|
+
*
|
|
13
|
+
* SAFETY
|
|
14
|
+
* - DRY-RUN BY DEFAULT ("dummy" mode): prints the full plan and writes NOTHING. Pass --execute to write.
|
|
15
|
+
* - Writing to the dooer-production namespace additionally requires --confirm-production.
|
|
16
|
+
* - All writes run inside ONE transaction on the target; any error rolls the whole copy back.
|
|
17
|
+
* - Referenced users missing from the target ARE copied (customer emails anonymized); existing users and
|
|
18
|
+
* the target org's own account/owners are left as-is. Never creates brand-new login users.
|
|
19
|
+
*
|
|
20
|
+
* RESILIENCE (flaky VPN)
|
|
21
|
+
* - Reads reconnect + retry on a dropped connection (pass-1, discovery, strategy/guard reads, file-id read).
|
|
22
|
+
* - The remap salt is derived from (source, target), so RE-RUNNING the same copy yields identical ids.
|
|
23
|
+
* - --only / --skip-tables copy a subset (pass-1 still scans all → cross-refs remap), so a big org can be
|
|
24
|
+
* copied in survivable chunks; re-run with the same --target and --if-target-nonempty insert. See GUIDE.md.
|
|
25
|
+
*
|
|
26
|
+
* CONNECTION
|
|
27
|
+
* Per namespace it reads the dooer-database ClusterIP and the postgres password from kubectl
|
|
28
|
+
* (requires VPN access to the ClusterIP). Nothing is written to disk.
|
|
29
|
+
*
|
|
30
|
+
* USAGE (see GUIDE.md for full examples)
|
|
31
|
+
* node seed-test-account.js --source <uuid> --target <uuid> \
|
|
32
|
+
* [--source-namespace dooer-staging] [--target-namespace dooer-staging] \
|
|
33
|
+
* [--execute] [--confirm-production] [--email testcustomer@dooer.com]
|
|
34
|
+
* node seed-test-account.js --source <uuid> --name "Copy of X" --owner-user <uuid> --execute
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
const { execFileSync } = require('child_process')
|
|
38
|
+
const crypto = require('crypto')
|
|
39
|
+
const path = require('path')
|
|
40
|
+
const { Client, types } = require('pg')
|
|
41
|
+
const copyFrom = require('pg-copy-streams').from
|
|
42
|
+
const Cursor = require('pg-cursor')
|
|
43
|
+
const { Readable } = require('stream')
|
|
44
|
+
const { pipeline } = require('stream/promises')
|
|
45
|
+
const { S3Client, GetObjectCommand, PutObjectCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3')
|
|
46
|
+
|
|
47
|
+
// Read date/timestamp columns as their raw Postgres text (no JS Date round-trip), so COPY writes them
|
|
48
|
+
// back verbatim with no timezone drift. numeric/int8/uuid already come back as strings.
|
|
49
|
+
types.setTypeParser(1082, (v) => v) // date
|
|
50
|
+
types.setTypeParser(1114, (v) => v) // timestamp
|
|
51
|
+
types.setTypeParser(1184, (v) => v) // timestamptz
|
|
52
|
+
// Read json/jsonb as RAW TEXT too: node-pg parses a jsonb value of `null` into JS null, which is
|
|
53
|
+
// indistinguishable from a SQL NULL — writing it back as SQL NULL breaks NOT-NULL jsonb columns.
|
|
54
|
+
// Raw text keeps JSON null as the string "null" (distinct from a real SQL NULL, which stays JS null).
|
|
55
|
+
types.setTypeParser(114, (v) => v) // json
|
|
56
|
+
types.setTypeParser(3802, (v) => v) // jsonb
|
|
57
|
+
|
|
58
|
+
// ── args ─────────────────────────────────────────────────────────────────────
|
|
59
|
+
function parseArgs(argv) {
|
|
60
|
+
const a = {
|
|
61
|
+
emailScrub: 'testcustomer@dooer.com',
|
|
62
|
+
sourceNamespace: 'dooer-staging',
|
|
63
|
+
mapPath: path.join(__dirname, 'schema-map.json'),
|
|
64
|
+
}
|
|
65
|
+
for (let i = 0; i < argv.length; i++) {
|
|
66
|
+
const k = argv[i]
|
|
67
|
+
const val = () => argv[++i]
|
|
68
|
+
if (k === '--source') a.source = val()
|
|
69
|
+
else if (k === '--target') a.target = val()
|
|
70
|
+
else if (k === '--name') a.name = val()
|
|
71
|
+
else if (k === '--owner-user') a.ownerUser = val()
|
|
72
|
+
else if (k === '--source-namespace') a.sourceNamespace = val()
|
|
73
|
+
else if (k === '--target-namespace') a.targetNamespace = val()
|
|
74
|
+
else if (k === '--email') a.emailScrub = val()
|
|
75
|
+
else if (k === '--map') a.mapPath = val()
|
|
76
|
+
else if (k === '--execute') a.execute = true
|
|
77
|
+
else if (k === '--skip-files') a.skipFiles = true
|
|
78
|
+
else if (k === '--skip-users') a.skipUsers = true
|
|
79
|
+
else if (k === '--confirm-production') a.confirmProduction = true
|
|
80
|
+
else if (k === '--if-target-nonempty') a.ifTargetNonempty = val()
|
|
81
|
+
// refuse (default) | insert
|
|
82
|
+
else if (k === '--salt') a.salt = val()
|
|
83
|
+
// extra entropy folded into the deterministic remap salt
|
|
84
|
+
else if (k === '--only')
|
|
85
|
+
a.only = val()
|
|
86
|
+
.split(',')
|
|
87
|
+
.map((s) => s.trim())
|
|
88
|
+
.filter(Boolean)
|
|
89
|
+
// copy ONLY these tables (pass-1 still scans all)
|
|
90
|
+
else if (k === '--skip-tables')
|
|
91
|
+
a.skipTables = val()
|
|
92
|
+
.split(',')
|
|
93
|
+
.map((s) => s.trim())
|
|
94
|
+
.filter(Boolean)
|
|
95
|
+
// copy all EXCEPT these
|
|
96
|
+
else if (k === '-h' || k === '--help') a.help = true
|
|
97
|
+
else throw new Error(`unknown argument: ${k}`)
|
|
98
|
+
}
|
|
99
|
+
if (!a.targetNamespace) a.targetNamespace = a.sourceNamespace
|
|
100
|
+
if (!a.ifTargetNonempty) a.ifTargetNonempty = 'refuse'
|
|
101
|
+
return a
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const HELP = `seed-test-account — copy one Dooer org's data into another (test) org.
|
|
105
|
+
|
|
106
|
+
Required:
|
|
107
|
+
--source <uuid> org to copy FROM (the real company)
|
|
108
|
+
--target <uuid> org to copy INTO (existing test entity)
|
|
109
|
+
...or, to create a new target org:
|
|
110
|
+
--name "<string>" name for a new target company row (source's row is cloned, name replaced)
|
|
111
|
+
--owner-user <uuid> user to set as the new org's owner (companies2users) — REQUIRED when creating
|
|
112
|
+
|
|
113
|
+
Optional:
|
|
114
|
+
--source-namespace <ns> default dooer-staging
|
|
115
|
+
--target-namespace <ns> default = source-namespace (set differently to copy across clusters,
|
|
116
|
+
e.g. --source-namespace dooer-production --target-namespace dooer-staging)
|
|
117
|
+
--email <addr> address to scrub every email column to (default testcustomer@dooer.com)
|
|
118
|
+
--skip-users do NOT copy referenced users missing from the target (default: copy them,
|
|
119
|
+
anonymizing customer-role emails). Same-namespace copies are a no-op anyway.
|
|
120
|
+
--if-target-nonempty refuse|insert what to do if a copy table already has target-org rows (default refuse)
|
|
121
|
+
--only <a.b,c.d> copy ONLY these tables (pass-1 still scans all, so cross-refs remap). For
|
|
122
|
+
resuming/chunking a big copy over a flaky link — re-run with the SAME
|
|
123
|
+
--target and --if-target-nonempty insert. Keep transactions + its twins together.
|
|
124
|
+
--skip-tables <a.b,c.d> copy all tables EXCEPT these (same chunking use)
|
|
125
|
+
--salt <string> extra entropy folded into the remap salt (normally unneeded; the salt is
|
|
126
|
+
already derived from source+target so re-runs are deterministic/resumable)
|
|
127
|
+
--execute actually write (default is DRY-RUN / "dummy": plan only, no writes)
|
|
128
|
+
--confirm-production REQUIRED to write when --target-namespace is dooer-production
|
|
129
|
+
--map <path> schema map (default ./schema-map.json)
|
|
130
|
+
`
|
|
131
|
+
|
|
132
|
+
// ── kubectl-sourced connection ───────────────────────────────────────────────
|
|
133
|
+
function sh(file, args) {
|
|
134
|
+
return execFileSync(file, args, { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }).trim()
|
|
135
|
+
}
|
|
136
|
+
function connInfo(namespace) {
|
|
137
|
+
const clusterIP = sh('kubectl', ['get', 'svc', 'dooer-database', '-n', namespace, '-o', 'jsonpath={.spec.clusterIP}'])
|
|
138
|
+
const b64 = sh('kubectl', [
|
|
139
|
+
'get',
|
|
140
|
+
'secret',
|
|
141
|
+
'postgres.dooer-database.credentials.postgresql.acid.zalan.do',
|
|
142
|
+
'-n',
|
|
143
|
+
namespace,
|
|
144
|
+
'-o',
|
|
145
|
+
'jsonpath={.data.password}',
|
|
146
|
+
])
|
|
147
|
+
const password = Buffer.from(b64, 'base64').toString('utf8')
|
|
148
|
+
if (!clusterIP || !password) throw new Error(`could not read dooer-database conn info for namespace ${namespace}`)
|
|
149
|
+
return { host: clusterIP, port: 5432, user: 'postgres', password, database: 'dooer' }
|
|
150
|
+
}
|
|
151
|
+
async function rawConnect(namespace) {
|
|
152
|
+
const info = connInfo(namespace)
|
|
153
|
+
const client = new Client({
|
|
154
|
+
...info,
|
|
155
|
+
ssl: { rejectUnauthorized: false },
|
|
156
|
+
statement_timeout: 0,
|
|
157
|
+
keepAlive: true, // TCP keepalive so an idle connection is not dropped during the long file phase
|
|
158
|
+
keepAliveInitialDelayMillis: 30000,
|
|
159
|
+
})
|
|
160
|
+
// An unexpected disconnect emits 'error'; unhandled it crashes the process. Swallow+log so the
|
|
161
|
+
// caller's next query surfaces the failure normally and we can reconnect. Marked so callers detect it.
|
|
162
|
+
client.on('error', (e) => {
|
|
163
|
+
client._seedDead = e
|
|
164
|
+
console.error(`\n[pg ${namespace}] connection error: ${e.message}`)
|
|
165
|
+
})
|
|
166
|
+
await client.connect()
|
|
167
|
+
return client
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// A self-healing connection: `.query()` retries transient drops, reconnecting a dead client first, so a
|
|
171
|
+
// flaky VPN (read ETIMEDOUT / "Connection terminated unexpectedly") doesn't kill a read. `.client`
|
|
172
|
+
// exposes the live pg Client for the streaming paths (cursor / COPY), which are retried a level up (whole
|
|
173
|
+
// table) since a mid-stream drop can't resume the same statement.
|
|
174
|
+
async function connect(namespace) {
|
|
175
|
+
let client = await rawConnect(namespace)
|
|
176
|
+
return {
|
|
177
|
+
namespace,
|
|
178
|
+
get client() {
|
|
179
|
+
return client
|
|
180
|
+
},
|
|
181
|
+
async reconnect() {
|
|
182
|
+
try {
|
|
183
|
+
await client.end()
|
|
184
|
+
} catch (_) {}
|
|
185
|
+
client = await rawConnect(namespace)
|
|
186
|
+
return client
|
|
187
|
+
},
|
|
188
|
+
async query(sql, params) {
|
|
189
|
+
return withRetry(
|
|
190
|
+
async () => {
|
|
191
|
+
if (client._seedDead) await this.reconnect()
|
|
192
|
+
try {
|
|
193
|
+
return await client.query(sql, params)
|
|
194
|
+
} catch (e) {
|
|
195
|
+
if (isTransient(e)) await this.reconnect()
|
|
196
|
+
throw e
|
|
197
|
+
}
|
|
198
|
+
},
|
|
199
|
+
{ label: `query ${namespace}` }
|
|
200
|
+
)
|
|
201
|
+
},
|
|
202
|
+
async end() {
|
|
203
|
+
try {
|
|
204
|
+
await client.end()
|
|
205
|
+
} catch (_) {}
|
|
206
|
+
},
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// transient = a network/server hiccup worth retrying (vs a real SQL/logic error, which must surface)
|
|
211
|
+
function isTransient(e) {
|
|
212
|
+
return /terminat|ECONNRESET|ETIMEDOUT|EPIPE|socket hang up|Connection terminated|timeout|503|500|InternalError|RequestTimeout|SlowDown|Client has encountered a connection error/i.test(
|
|
213
|
+
`${e.name} ${e.message} ${e.code || ''} ${e.$metadata?.httpStatusCode || ''}`
|
|
214
|
+
)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// run an async op with a few retries on transient network/S3 errors (exponential-ish backoff)
|
|
218
|
+
async function withRetry(fn, { tries = 6, label = 'op' } = {}) {
|
|
219
|
+
let last
|
|
220
|
+
for (let i = 0; i < tries; i++) {
|
|
221
|
+
try {
|
|
222
|
+
return await fn()
|
|
223
|
+
} catch (e) {
|
|
224
|
+
last = e
|
|
225
|
+
if (i === tries - 1 || !isTransient(e)) throw e
|
|
226
|
+
const wait = Math.min(8000, 500 * 2 ** i)
|
|
227
|
+
process.stderr.write(`\n[retry ${label}] ${e.message} — attempt ${i + 2}/${tries} in ${wait}ms\n`)
|
|
228
|
+
await new Promise((r) => setTimeout(r, wait))
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
throw last
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// ── S3 (Ceph RGW) file copy ──────────────────────────────────────────────────
|
|
235
|
+
// The file bytes for a copied fileData row live in S3 keyed by the fileData id. We copy each object
|
|
236
|
+
// from the SOURCE env bucket to the TARGET env bucket at its NEW key (= the copied fileData id), so the
|
|
237
|
+
// copy owns its own files. Endpoint/bucket/credentials are read from the file-storage pod per namespace.
|
|
238
|
+
function s3Config(namespace) {
|
|
239
|
+
const pods = sh('kubectl', [
|
|
240
|
+
'get',
|
|
241
|
+
'pods',
|
|
242
|
+
'-n',
|
|
243
|
+
namespace,
|
|
244
|
+
'-o',
|
|
245
|
+
'jsonpath={range .items[*]}{.metadata.name}{"\\n"}{end}',
|
|
246
|
+
])
|
|
247
|
+
.split('\n')
|
|
248
|
+
.filter((p) => /^service-file-storage-[a-z0-9]+-[a-z0-9]+$/.test(p))
|
|
249
|
+
const pod = pods[0]
|
|
250
|
+
if (!pod) throw new Error(`no service-file-storage pod found in ${namespace} (needed for the S3 file copy)`)
|
|
251
|
+
const allEnv = sh('kubectl', ['exec', '-n', namespace, pod, '--', 'env'])
|
|
252
|
+
const env = (v) => {
|
|
253
|
+
const m = allEnv.match(new RegExp('^' + v + '=(.*)$', 'm'))
|
|
254
|
+
return m ? m[1] : ''
|
|
255
|
+
}
|
|
256
|
+
const svcName = env('AWS_ENDPOINT_URL_S3')
|
|
257
|
+
.replace(/^https?:\/\//, '')
|
|
258
|
+
.split('.')[0]
|
|
259
|
+
const clusterIP = sh('kubectl', ['get', 'svc', svcName, '-n', 'rook-ceph', '-o', 'jsonpath={.spec.clusterIP}'])
|
|
260
|
+
const endpoint = `http://${clusterIP}:80`
|
|
261
|
+
const mk = (ak, sk) =>
|
|
262
|
+
new S3Client({
|
|
263
|
+
endpoint,
|
|
264
|
+
region: 'us-east-1',
|
|
265
|
+
forcePathStyle: true,
|
|
266
|
+
credentials: { accessKeyId: ak, secretAccessKey: sk },
|
|
267
|
+
})
|
|
268
|
+
const cfg = {
|
|
269
|
+
primary: {
|
|
270
|
+
bucket: env('DOOER_STORAGE_PROVIDER_AMAZON_S3_BUCKET'),
|
|
271
|
+
client: mk(env('FILE_STORAGE_BUCKET_ACCESS_KEY_ID'), env('FILE_STORAGE_BUCKET_SECRET_ACCESS_KEY')),
|
|
272
|
+
},
|
|
273
|
+
fallback: null,
|
|
274
|
+
}
|
|
275
|
+
const fb = env('DOOER_STORAGE_PROVIDER_AMAZON_S3_BUCKET_FALLBACK')
|
|
276
|
+
const fak = env('FILE_STORAGE_FALLBACK_BUCKET_ACCESS_KEY_ID')
|
|
277
|
+
if (fb && fak) cfg.fallback = { bucket: fb, client: mk(fak, env('FILE_STORAGE_FALLBACK_BUCKET_SECRET_ACCESS_KEY')) }
|
|
278
|
+
return cfg
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// get an object from a source config, trying the primary bucket then the fallback bucket
|
|
282
|
+
async function s3Get(cfg, key) {
|
|
283
|
+
try {
|
|
284
|
+
return await cfg.primary.client.send(new GetObjectCommand({ Bucket: cfg.primary.bucket, Key: key }))
|
|
285
|
+
} catch (e) {
|
|
286
|
+
const notFound = e.name === 'NoSuchKey' || e.name === 'NotFound' || e.$metadata?.httpStatusCode === 404
|
|
287
|
+
if (notFound && cfg.fallback)
|
|
288
|
+
return await cfg.fallback.client.send(new GetObjectCommand({ Bucket: cfg.fallback.bucket, Key: key }))
|
|
289
|
+
throw e
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// copy the S3 object for every copied s3Blob row (old key → its remapped new key)
|
|
294
|
+
async function copyFiles(src, args, map, remapUuid) {
|
|
295
|
+
const s3Tables = (map.s3BlobTables || [])
|
|
296
|
+
.map((k) => map.tables.find((t) => t.key === k))
|
|
297
|
+
.filter((t) => t && t.action === 'copy')
|
|
298
|
+
if (!s3Tables.length) return
|
|
299
|
+
const srcS3 = s3Config(args.sourceNamespace)
|
|
300
|
+
const tgtS3 = s3Config(args.targetNamespace)
|
|
301
|
+
for (const t of s3Tables) {
|
|
302
|
+
const f = filterClause(t, args.source)
|
|
303
|
+
// src is a self-healing connection — `.query` reconnects a client dropped during the long copy/txn
|
|
304
|
+
// and retries the read on its own.
|
|
305
|
+
const rows = await src.query(
|
|
306
|
+
`SELECT ${qIdent(t.pk.column)} AS id FROM ${qTable(t.schema, t.table)} WHERE ${f.sql}`,
|
|
307
|
+
f.params
|
|
308
|
+
)
|
|
309
|
+
const keys = rows.rows.map((r) => r.id).filter(Boolean)
|
|
310
|
+
console.log(
|
|
311
|
+
`\nS3: copying ${keys.length} ${t.key} objects (${srcS3.primary.bucket}[+fallback] → ${tgtS3.primary.bucket})…`
|
|
312
|
+
)
|
|
313
|
+
let done = 0,
|
|
314
|
+
missing = 0,
|
|
315
|
+
failed = 0,
|
|
316
|
+
idx = 0
|
|
317
|
+
const worker = async () => {
|
|
318
|
+
while (idx < keys.length) {
|
|
319
|
+
const oldKey = keys[idx++]
|
|
320
|
+
const newKey = remapUuid(oldKey)
|
|
321
|
+
try {
|
|
322
|
+
await withRetry(
|
|
323
|
+
async () => {
|
|
324
|
+
const obj = await s3Get(srcS3, oldKey) // primary then fallback
|
|
325
|
+
// buffer the body (streaming GetObject→PutObject fails against Ceph RGW)
|
|
326
|
+
const bytes = await obj.Body.transformToByteArray()
|
|
327
|
+
await tgtS3.primary.client.send(
|
|
328
|
+
new PutObjectCommand({
|
|
329
|
+
Bucket: tgtS3.primary.bucket,
|
|
330
|
+
Key: newKey,
|
|
331
|
+
Body: bytes,
|
|
332
|
+
ContentType: obj.ContentType,
|
|
333
|
+
})
|
|
334
|
+
)
|
|
335
|
+
},
|
|
336
|
+
{ tries: 4, label: `S3 ${oldKey}` }
|
|
337
|
+
)
|
|
338
|
+
done++
|
|
339
|
+
} catch (e) {
|
|
340
|
+
if (e.name === 'NoSuchKey' || e.name === 'NotFound' || e.$metadata?.httpStatusCode === 404) missing++
|
|
341
|
+
else if (++failed <= 5) console.error(` S3 copy failed for ${oldKey}: ${e.name}`)
|
|
342
|
+
}
|
|
343
|
+
const n = done + missing + failed
|
|
344
|
+
if (n % 200 === 0)
|
|
345
|
+
process.stderr.write(`\r S3 ${n}/${keys.length} (ok ${done}, missing ${missing}, failed ${failed})`)
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
await Promise.all(Array.from({ length: 12 }, worker))
|
|
349
|
+
process.stderr.write('\n')
|
|
350
|
+
console.log(`S3: ${t.key} — copied ${done}, missing-in-source ${missing}, failed ${failed}.`)
|
|
351
|
+
if (failed) throw new Error(`S3 copy had ${failed} failures for ${t.key} (see log)`)
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// ── introspection helpers ────────────────────────────────────────────────────
|
|
356
|
+
const qIdent = (s) => '"' + String(s).replace(/"/g, '""') + '"'
|
|
357
|
+
const qTable = (schema, table) => `${qIdent(schema)}.${qIdent(table)}`
|
|
358
|
+
|
|
359
|
+
async function tableExists(client, schema, table) {
|
|
360
|
+
const r = await client.query(
|
|
361
|
+
`SELECT 1 FROM information_schema.tables WHERE table_schema=$1 AND table_name=$2 AND table_type='BASE TABLE'`,
|
|
362
|
+
[schema, table]
|
|
363
|
+
)
|
|
364
|
+
return r.rowCount > 0
|
|
365
|
+
}
|
|
366
|
+
async function columnsOf(client, schema, table) {
|
|
367
|
+
const r = await client.query(
|
|
368
|
+
`SELECT column_name, data_type, udt_name, is_generated,
|
|
369
|
+
(column_default LIKE 'nextval(%') AS is_serial
|
|
370
|
+
FROM information_schema.columns WHERE table_schema=$1 AND table_name=$2 ORDER BY ordinal_position`,
|
|
371
|
+
[schema, table]
|
|
372
|
+
)
|
|
373
|
+
return r.rows.map((c) => ({
|
|
374
|
+
name: c.column_name,
|
|
375
|
+
type: c.data_type, // 'uuid' | 'ARRAY' | 'text' | 'integer' | ...
|
|
376
|
+
udt: c.udt_name, // '_uuid' for uuid[], 'uuid', 'text', ...
|
|
377
|
+
generated: c.is_generated !== 'NEVER',
|
|
378
|
+
serial: c.is_serial,
|
|
379
|
+
}))
|
|
380
|
+
}
|
|
381
|
+
const isUuid = (col) => col.type === 'uuid'
|
|
382
|
+
const isUuidArray = (col) => col.type === 'ARRAY' && col.udt === '_uuid'
|
|
383
|
+
const newUuid = () => crypto.randomUUID()
|
|
384
|
+
|
|
385
|
+
// Deterministic derivation of a fresh uuid from a source uuid (v5-style). This lets us remap references
|
|
386
|
+
// WITHOUT storing a giant old→new map: we keep only the SET of copied source uuids and recompute the
|
|
387
|
+
// target uuid on demand. The salt is derived from (source, target, --salt) so RE-RUNNING the same copy
|
|
388
|
+
// reproduces IDENTICAL target ids — which makes a copy idempotent/resumable over a flaky link. (Was
|
|
389
|
+
// random per run.) A different target org gets different ids (its target uuid differs).
|
|
390
|
+
let REMAP_SALT = crypto.randomBytes(16)
|
|
391
|
+
function setRemapSalt(source, target, extra) {
|
|
392
|
+
REMAP_SALT = crypto
|
|
393
|
+
.createHash('sha256')
|
|
394
|
+
.update(`${source}|${target}|${extra || ''}`)
|
|
395
|
+
.digest()
|
|
396
|
+
.subarray(0, 16)
|
|
397
|
+
}
|
|
398
|
+
function deriveUuid(old) {
|
|
399
|
+
const h = crypto.createHash('sha1')
|
|
400
|
+
h.update(REMAP_SALT)
|
|
401
|
+
h.update(Buffer.from(old.replace(/-/g, ''), 'hex'))
|
|
402
|
+
const b = h.digest().subarray(0, 16)
|
|
403
|
+
b[6] = (b[6] & 0x0f) | 0x50 // version 5
|
|
404
|
+
b[8] = (b[8] & 0x3f) | 0x80 // variant
|
|
405
|
+
const x = b.toString('hex')
|
|
406
|
+
return `${x.slice(0, 8)}-${x.slice(8, 12)}-${x.slice(12, 16)}-${x.slice(16, 20)}-${x.slice(20, 32)}`
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
// ── uniqueness handling is CONFIG-DRIVEN (schema-map.json → uniquenessStrategies). NO column names or
|
|
410
|
+
// strategies are hardcoded here; the two views below are derived from the config object.
|
|
411
|
+
// uuidRemapCols(map) → Map("schema.table" -> [uuid col names]) (strategy: remap-uuid)
|
|
412
|
+
// nonUuidStrategies(map) → the config entries needing a per-column remap map (sequence/max-offset/text-suffix)
|
|
413
|
+
function uuidRemapCols(map) {
|
|
414
|
+
const m = new Map()
|
|
415
|
+
for (const c of map.uniquenessStrategies.columns) {
|
|
416
|
+
if (c.type !== 'uuid' || c.strategy !== 'remap-uuid') continue
|
|
417
|
+
const k = `${c.schema}.${c.table}`
|
|
418
|
+
if (!m.has(k)) m.set(k, [])
|
|
419
|
+
m.get(k).push(c.column)
|
|
420
|
+
}
|
|
421
|
+
return m
|
|
422
|
+
}
|
|
423
|
+
const nonUuidStrategies = (map) =>
|
|
424
|
+
map.uniquenessStrategies.columns.filter((c) => c.type !== 'uuid' && c.strategy !== 'preserve')
|
|
425
|
+
|
|
426
|
+
async function main(passedArgs) {
|
|
427
|
+
const args = passedArgs || parseArgs(process.argv.slice(2))
|
|
428
|
+
if (args.help) return void console.log(HELP)
|
|
429
|
+
if (!args.source) throw new Error('missing --source')
|
|
430
|
+
if (!args.target && !args.name) throw new Error('provide --target <uuid>, or --name to create a new target org')
|
|
431
|
+
if (!args.target && !args.ownerUser) throw new Error('creating a new target org requires --owner-user')
|
|
432
|
+
if (args.targetNamespace === 'dooer-production' && args.execute && !args.confirmProduction)
|
|
433
|
+
throw new Error('refusing to WRITE to dooer-production without --confirm-production')
|
|
434
|
+
|
|
435
|
+
const map = require(args.mapPath)
|
|
436
|
+
const copyTables = map.tables.filter((t) => t.action === 'copy')
|
|
437
|
+
const mode = args.execute ? 'EXECUTE' : 'DRY-RUN (dummy — no writes)'
|
|
438
|
+
|
|
439
|
+
console.log(`\n=== seed-test-account · ${mode} ===`)
|
|
440
|
+
console.log(`source: ${args.source} @ ${args.sourceNamespace}`)
|
|
441
|
+
console.log(`target: ${args.target || `(create "${args.name}", owner ${args.ownerUser})`} @ ${args.targetNamespace}`)
|
|
442
|
+
console.log(
|
|
443
|
+
`email scrub: ${args.emailScrub} · copy tables: ${copyTables.length} · if-target-nonempty: ${args.ifTargetNonempty}\n`
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
// Always separate connections: pass 2 streams each table with a server-side cursor on `src` while
|
|
447
|
+
// COPYing into `tgt`, which cannot share one connection (even for a same-namespace copy).
|
|
448
|
+
const src = await connect(args.sourceNamespace)
|
|
449
|
+
const tgt = await connect(args.targetNamespace)
|
|
450
|
+
|
|
451
|
+
try {
|
|
452
|
+
// resolve / create the target org id
|
|
453
|
+
let targetOrg = args.target
|
|
454
|
+
let createdOrg = false
|
|
455
|
+
if (!targetOrg) {
|
|
456
|
+
targetOrg = newUuid()
|
|
457
|
+
createdOrg = true
|
|
458
|
+
// the owner user must already exist in the TARGET db (we never create users)
|
|
459
|
+
const u = await tgt.query(`SELECT 1 FROM service_accounts.users WHERE users_pk=$1`, [args.ownerUser])
|
|
460
|
+
if (!u.rowCount)
|
|
461
|
+
throw new Error(`--owner-user ${args.ownerUser} not found in service_accounts.users on ${args.targetNamespace}`)
|
|
462
|
+
console.log(
|
|
463
|
+
`${args.execute ? 'will create' : 'would create'} target org ${targetOrg} (name "${args.name}", owner ${
|
|
464
|
+
args.ownerUser
|
|
465
|
+
})`
|
|
466
|
+
)
|
|
467
|
+
} else {
|
|
468
|
+
const exists = await tgt.query(`SELECT 1 FROM service_accounts.companies WHERE companies_pk=$1`, [targetOrg])
|
|
469
|
+
if (!exists.rowCount)
|
|
470
|
+
throw new Error(
|
|
471
|
+
`target org ${targetOrg} has no service_accounts.companies row (create it first, or use --name)`
|
|
472
|
+
)
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
// Fix the remap salt from (source, target, --salt): re-running the same copy now derives IDENTICAL
|
|
476
|
+
// target ids, so a run interrupted by a dropped VPN can be resumed (re-run with the SAME --target)
|
|
477
|
+
// and `--only`/`--skip-tables` can copy a subset while cross-refs still remap consistently.
|
|
478
|
+
setRemapSalt(args.source, targetOrg, args.salt)
|
|
479
|
+
|
|
480
|
+
// The uuid remap keeps only the SET of copied source uuids (not a huge old→new map); the new value
|
|
481
|
+
// is derived on demand. The source org id maps to the (existing/new) target org id. `explicitRemap`
|
|
482
|
+
// pins specific old→new ids that are NOT derived — used for referenced users that already exist in
|
|
483
|
+
// the target under a different pk (matched by natural key), so refs point at the existing user.
|
|
484
|
+
const sourceUuids = new Set()
|
|
485
|
+
const explicitRemap = new Map()
|
|
486
|
+
const remapUuid = (v) =>
|
|
487
|
+
v === args.source
|
|
488
|
+
? targetOrg
|
|
489
|
+
: explicitRemap.has(v)
|
|
490
|
+
? explicitRemap.get(v)
|
|
491
|
+
: sourceUuids.has(v)
|
|
492
|
+
? deriveUuid(v)
|
|
493
|
+
: v
|
|
494
|
+
const columnRemap = new Map() // "schema.table.column" -> Map(oldValueString -> newValue)
|
|
495
|
+
const uuidColsByTable = uuidRemapCols(map) // "schema.table" -> [uuid cols], from config
|
|
496
|
+
|
|
497
|
+
// PASS 1 — collect source PKs, allocate new ids (one query per table; progress to stderr)
|
|
498
|
+
const plan = []
|
|
499
|
+
let idx = 0
|
|
500
|
+
for (const t of copyTables) {
|
|
501
|
+
idx++
|
|
502
|
+
process.stderr.write(`\r[pass1 ${idx}/${copyTables.length}] ${t.key.slice(0, 52).padEnd(52)}`)
|
|
503
|
+
const okSrc = await tableExists(src, t.schema, t.table)
|
|
504
|
+
const okTgt = await tableExists(tgt, t.schema, t.table)
|
|
505
|
+
if (!okSrc || !okTgt) {
|
|
506
|
+
plan.push({ t, skipped: `missing in ${!okSrc ? 'source' : 'target'}`, count: 0 })
|
|
507
|
+
continue
|
|
508
|
+
}
|
|
509
|
+
const filters = filterClause(t, args.source)
|
|
510
|
+
if (!filters) {
|
|
511
|
+
plan.push({ t, skipped: 'no filter column', count: 0 })
|
|
512
|
+
continue
|
|
513
|
+
}
|
|
514
|
+
// cache the source columns (reused in pass 2); validate the config's uuid-remap columns exist here
|
|
515
|
+
const srcCols = await columnsOf(src, t.schema, t.table)
|
|
516
|
+
const srcColNames = new Set(srcCols.map((c) => c.name))
|
|
517
|
+
const freshen = (uuidColsByTable.get(t.key) || []).filter((c) => srcColNames.has(c))
|
|
518
|
+
let count
|
|
519
|
+
if (freshen.length) {
|
|
520
|
+
// one query fetches count + every unique-uuid value to add to the source-uuid set
|
|
521
|
+
const sel = freshen.map((c) => qIdent(c)).join(',')
|
|
522
|
+
const rows = await src.query(
|
|
523
|
+
`SELECT ${sel} FROM ${qTable(t.schema, t.table)} WHERE ${filters.sql}`,
|
|
524
|
+
filters.params
|
|
525
|
+
)
|
|
526
|
+
count = rows.rowCount
|
|
527
|
+
for (const row of rows.rows) for (const c of freshen) if (row[c]) sourceUuids.add(row[c])
|
|
528
|
+
} else {
|
|
529
|
+
const { rows } = await src.query(
|
|
530
|
+
`SELECT count(*)::int AS c FROM ${qTable(t.schema, t.table)} WHERE ${filters.sql}`,
|
|
531
|
+
filters.params
|
|
532
|
+
)
|
|
533
|
+
count = rows[0].c
|
|
534
|
+
}
|
|
535
|
+
plan.push({ t, count, srcCols })
|
|
536
|
+
}
|
|
537
|
+
process.stderr.write('\n')
|
|
538
|
+
|
|
539
|
+
const totalRows = plan.reduce((n, p) => n + (p.skipped ? 0 : p.count), 0)
|
|
540
|
+
console.log('--- plan (tables with rows) ---')
|
|
541
|
+
for (const p of plan.filter((p) => p.count > 0 || p.skipped)) {
|
|
542
|
+
if (p.skipped) console.log(` SKIP ${p.t.key.padEnd(48)} (${p.skipped})`)
|
|
543
|
+
else
|
|
544
|
+
console.log(
|
|
545
|
+
` copy ${p.t.key.padEnd(48)} ${String(p.count).padStart(8)} rows${
|
|
546
|
+
p.t.emailColumns.length ? ` [scrub ${p.t.emailColumns.join(',')}]` : ''
|
|
547
|
+
}${p.t.special ? ' [special]' : ''}`
|
|
548
|
+
)
|
|
549
|
+
}
|
|
550
|
+
console.log(`\ntotal rows to copy: ${totalRows} · distinct uuid values remapped: ${sourceUuids.size}`)
|
|
551
|
+
|
|
552
|
+
// PASS 1b — referenced users. Copied rows point at users (createdByUserId, fk_users_pk, …) that may
|
|
553
|
+
// not exist in the target namespace (cross-namespace copy) → resolvers like `createdByUser` dangle.
|
|
554
|
+
// Collect the ids in the configured user-ref columns, look each up in source (role) and target
|
|
555
|
+
// (existence), and copy ONLY the ones missing from the target — with a new id (added to the remap
|
|
556
|
+
// set so refs follow) and emails anonymized for `customer`-role users. Users already in the target
|
|
557
|
+
// are left untouched and their refs keep the original id.
|
|
558
|
+
const usersToCopy = [] // [{ oldId, role, anonymize }]
|
|
559
|
+
if (map.userTable && !args.skipUsers) {
|
|
560
|
+
const ut = map.userTable
|
|
561
|
+
const candidate = new Set()
|
|
562
|
+
for (const rc of map.userRefColumns || []) {
|
|
563
|
+
const p = plan.find((pp) => pp.t.key === `${rc.schema}.${rc.table}` && !pp.skipped && pp.count > 0)
|
|
564
|
+
if (!p || !(p.srcCols || []).some((c) => c.name === rc.column)) continue
|
|
565
|
+
const f = filterClause(p.t, args.source)
|
|
566
|
+
const { rows } = await withRetry(
|
|
567
|
+
() =>
|
|
568
|
+
src.query(
|
|
569
|
+
`SELECT DISTINCT ${qIdent(rc.column)} AS v FROM ${qTable(rc.schema, rc.table)} WHERE ${
|
|
570
|
+
f.sql
|
|
571
|
+
} AND ${qIdent(rc.column)} IS NOT NULL`,
|
|
572
|
+
f.params
|
|
573
|
+
),
|
|
574
|
+
{ label: `userref ${rc.table}.${rc.column}` }
|
|
575
|
+
)
|
|
576
|
+
for (const r of rows) if (r.v) candidate.add(r.v)
|
|
577
|
+
}
|
|
578
|
+
const ids = [...candidate]
|
|
579
|
+
let matched = 0
|
|
580
|
+
if (ids.length) {
|
|
581
|
+
const srcU = (
|
|
582
|
+
await src.query(
|
|
583
|
+
`SELECT ${qIdent(ut.pk)} AS id, ${qIdent(ut.roleColumn)} AS role, email, ${qIdent(
|
|
584
|
+
'id_number'
|
|
585
|
+
)} AS id_number FROM ${qTable(ut.schema, ut.table)} WHERE ${qIdent(ut.pk)} = ANY($1)`,
|
|
586
|
+
[ids]
|
|
587
|
+
)
|
|
588
|
+
).rows
|
|
589
|
+
// (1) exact pk already in target → ref stays valid, do nothing
|
|
590
|
+
const tgtPk = new Set(
|
|
591
|
+
(
|
|
592
|
+
await tgt.query(
|
|
593
|
+
`SELECT ${qIdent(ut.pk)} AS id FROM ${qTable(ut.schema, ut.table)} WHERE ${qIdent(ut.pk)} = ANY($1)`,
|
|
594
|
+
[ids]
|
|
595
|
+
)
|
|
596
|
+
).rows.map((r) => r.id)
|
|
597
|
+
)
|
|
598
|
+
const unresolved = srcU.filter((u) => !tgtPk.has(u.id))
|
|
599
|
+
// (2) non-customer users often exist in the target under a DIFFERENT pk (same person across
|
|
600
|
+
// namespaces). Match by the natural unique key (email|id_number, role) and pin the ref to the
|
|
601
|
+
// EXISTING target pk (never a duplicate → never a (email,role)/(id_number,role) unique clash).
|
|
602
|
+
const nonCust = unresolved.filter((u) => !(ut.anonymizeRoles || []).includes(u.role))
|
|
603
|
+
const emails = [...new Set(nonCust.map((u) => u.email).filter(Boolean))]
|
|
604
|
+
const idnums = [...new Set(nonCust.map((u) => u.id_number).filter(Boolean))]
|
|
605
|
+
const byEmail = new Map()
|
|
606
|
+
const byId = new Map()
|
|
607
|
+
if (emails.length || idnums.length) {
|
|
608
|
+
const tm = (
|
|
609
|
+
await tgt.query(
|
|
610
|
+
`SELECT ${qIdent(ut.pk)} AS id, ${qIdent(ut.roleColumn)} AS role, email, ${qIdent(
|
|
611
|
+
'id_number'
|
|
612
|
+
)} AS id_number FROM ${qTable(ut.schema, ut.table)} WHERE email = ANY($1) OR ${qIdent(
|
|
613
|
+
'id_number'
|
|
614
|
+
)} = ANY($2)`,
|
|
615
|
+
[emails, idnums]
|
|
616
|
+
)
|
|
617
|
+
).rows
|
|
618
|
+
for (const m of tm) {
|
|
619
|
+
if (m.email) byEmail.set(`${m.role}|${m.email}`, m.id)
|
|
620
|
+
if (m.id_number) byId.set(`${m.role}|${m.id_number}`, m.id)
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
for (const u of unresolved) {
|
|
624
|
+
const anonymize = (ut.anonymizeRoles || []).includes(u.role)
|
|
625
|
+
const hit =
|
|
626
|
+
!anonymize &&
|
|
627
|
+
((u.email && byEmail.get(`${u.role}|${u.email}`)) || (u.id_number && byId.get(`${u.role}|${u.id_number}`)))
|
|
628
|
+
if (hit) {
|
|
629
|
+
explicitRemap.set(u.id, hit) // ref → existing target user (no copy)
|
|
630
|
+
matched++
|
|
631
|
+
continue
|
|
632
|
+
}
|
|
633
|
+
sourceUuids.add(u.id) // will be copied with a new derived id → refs to it now remap
|
|
634
|
+
usersToCopy.push({ oldId: u.id, role: u.role, anonymize })
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
const anon = usersToCopy.filter((u) => u.anonymize).length
|
|
638
|
+
console.log(
|
|
639
|
+
`referenced users: ${candidate.size} distinct · matched to existing target users: ${matched} · copied new: ${
|
|
640
|
+
usersToCopy.length
|
|
641
|
+
} (customer-anonymized: ${anon}, as-is: ${usersToCopy.length - anon})`
|
|
642
|
+
)
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
// --only / --skip-tables filter the WRITE phase only (pass-1 already scanned EVERY table, so the uuid
|
|
646
|
+
// remap set is complete and cross-references into skipped tables still resolve). Lets a big org be
|
|
647
|
+
// copied in survivable chunks over a flaky link: re-run with the same --target (→ same salt) and a
|
|
648
|
+
// different --only, using --if-target-nonempty insert. Keep FK-linked int-key tables together —
|
|
649
|
+
// transactions + its 5 twins share a per-run sequence remap (see GUIDE.md).
|
|
650
|
+
const willCopy = (key) => (!args.only || args.only.includes(key)) && !(args.skipTables || []).includes(key)
|
|
651
|
+
|
|
652
|
+
// target-nonempty guard (only over the tables this run will actually write)
|
|
653
|
+
if (args.ifTargetNonempty === 'refuse') {
|
|
654
|
+
const nonEmpty = []
|
|
655
|
+
for (const p of plan) {
|
|
656
|
+
if (p.skipped || p.count === 0 || !willCopy(p.t.key)) continue
|
|
657
|
+
const f = filterClause(p.t, targetOrg)
|
|
658
|
+
const r = await tgt.query(
|
|
659
|
+
`SELECT count(*)::int AS c FROM ${qTable(p.t.schema, p.t.table)} WHERE ${f.sql}`,
|
|
660
|
+
f.params
|
|
661
|
+
)
|
|
662
|
+
if (r.rows[0].c > 0) nonEmpty.push(`${p.t.key} (${r.rows[0].c})`)
|
|
663
|
+
}
|
|
664
|
+
if (nonEmpty.length) {
|
|
665
|
+
console.log(
|
|
666
|
+
`\nTARGET NOT EMPTY in ${nonEmpty.length} copy table(s): ${nonEmpty.slice(0, 10).join(', ')}${
|
|
667
|
+
nonEmpty.length > 10 ? ' …' : ''
|
|
668
|
+
}`
|
|
669
|
+
)
|
|
670
|
+
if (args.execute)
|
|
671
|
+
throw new Error(
|
|
672
|
+
'refusing: target already has rows in copy tables (pass --if-target-nonempty insert to override)'
|
|
673
|
+
)
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
if (!args.execute) {
|
|
678
|
+
console.log('\nDRY-RUN complete — nothing was written. Re-run with --execute to perform the copy.\n')
|
|
679
|
+
return
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// PASS 2 — write, inside one transaction, FK/triggers disabled for the load. Transactional statements
|
|
683
|
+
// use the RAW client (tgt.client): a mid-transaction drop must fail this run (→ rollback), never
|
|
684
|
+
// silently reconnect onto a fresh connection with no open transaction. (Resumable mode, which commits
|
|
685
|
+
// per table and can survive drops, is a separate path — see --resumable.)
|
|
686
|
+
await tgt.client.query('BEGIN')
|
|
687
|
+
await tgt.client.query("SET session_replication_role = 'replica'") // disable FK + user triggers for the bulk load
|
|
688
|
+
let written = 0
|
|
689
|
+
|
|
690
|
+
// --name: create the target org (clone the source companies row + link the owner) before copying
|
|
691
|
+
if (createdOrg) {
|
|
692
|
+
await createTargetOrg(src, tgt, args, targetOrg)
|
|
693
|
+
written += 2
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
// build a per-column old->new map for every non-uuid unique column, per its CONFIG strategy, and
|
|
697
|
+
// register it under the owning column AND every FK column that references it (so refs stay consistent).
|
|
698
|
+
for (const s of nonUuidStrategies(map)) {
|
|
699
|
+
const pe = plan.find((p) => p.t.key === `${s.schema}.${s.table}` && !p.skipped && p.count > 0)
|
|
700
|
+
if (!pe) continue
|
|
701
|
+
const f = filterClause(pe.t, args.source)
|
|
702
|
+
const srcRows = await src.query(
|
|
703
|
+
`SELECT ${qIdent(s.column)} AS v FROM ${qTable(s.schema, s.table)} WHERE ${f.sql} ORDER BY 1`,
|
|
704
|
+
f.params
|
|
705
|
+
)
|
|
706
|
+
const m = new Map()
|
|
707
|
+
if (s.strategy === 'sequence') {
|
|
708
|
+
// reserve all the fresh ids in ONE round-trip (was one nextval() per row)
|
|
709
|
+
const nonNull = srcRows.rows.filter((r) => r.v != null)
|
|
710
|
+
if (nonNull.length) {
|
|
711
|
+
const nv = await tgt.client.query(`SELECT nextval($1)::bigint AS v FROM generate_series(1, $2)`, [
|
|
712
|
+
s.sequenceName,
|
|
713
|
+
nonNull.length,
|
|
714
|
+
])
|
|
715
|
+
nonNull.forEach((r, i) => m.set(String(r.v), Number(nv.rows[i].v)))
|
|
716
|
+
}
|
|
717
|
+
} else if (s.strategy === 'max-offset') {
|
|
718
|
+
const mx = await tgt.client.query(
|
|
719
|
+
`SELECT COALESCE(max(${qIdent(s.column)}), 0)::bigint AS m FROM ${qTable(s.schema, s.table)}`
|
|
720
|
+
)
|
|
721
|
+
let base = Number(mx.rows[0].m)
|
|
722
|
+
for (const r of srcRows.rows) if (r.v != null) m.set(String(r.v), ++base)
|
|
723
|
+
} else if (s.strategy === 'text-suffix') {
|
|
724
|
+
// deterministic suffix (hash of salt+value) so a re-run/chunk reproduces the same value
|
|
725
|
+
for (const r of srcRows.rows)
|
|
726
|
+
if (r.v != null)
|
|
727
|
+
m.set(
|
|
728
|
+
String(r.v),
|
|
729
|
+
`${r.v}-seedcopy-${crypto
|
|
730
|
+
.createHash('sha1')
|
|
731
|
+
.update(REMAP_SALT)
|
|
732
|
+
.update(String(r.v))
|
|
733
|
+
.digest('hex')
|
|
734
|
+
.slice(0, 8)}`
|
|
735
|
+
)
|
|
736
|
+
}
|
|
737
|
+
columnRemap.set(`${s.schema}.${s.table}.${s.column}`, m)
|
|
738
|
+
for (const ref of s.references || []) columnRemap.set(`${ref.schema}.${ref.table}.${ref.column}`, m)
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
const remapTextStoredUuids = map.options ? map.options.remapTextStoredUuids !== false : true
|
|
742
|
+
|
|
743
|
+
// referenced users missing from the target (discovered in pass 1b) — copy them before the data rows
|
|
744
|
+
// that reference them so every user ref resolves. New derived id + anonymized customer emails.
|
|
745
|
+
if (usersToCopy.length) {
|
|
746
|
+
const nu = await copyUsers(src, tgt, usersToCopy, map.userTable, {
|
|
747
|
+
remapUuid,
|
|
748
|
+
columnRemap,
|
|
749
|
+
targetOrg,
|
|
750
|
+
remapTextStoredUuids,
|
|
751
|
+
emailScrub: args.emailScrub,
|
|
752
|
+
})
|
|
753
|
+
written += nu
|
|
754
|
+
console.log(`copied ${nu} referenced user(s) into ${args.targetNamespace} (missing there before).`)
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
const toCopy = plan.filter((p) => !p.skipped && p.count > 0 && willCopy(p.t.key))
|
|
758
|
+
let ci = 0
|
|
759
|
+
for (const p of toCopy) {
|
|
760
|
+
ci++
|
|
761
|
+
process.stderr.write(
|
|
762
|
+
`\r[copy ${ci}/${toCopy.length}] ${p.t.key.slice(0, 46).padEnd(46)} ${String(p.count).padStart(8)} rows`
|
|
763
|
+
)
|
|
764
|
+
written += await copyTable(src, tgt, p, args, { remapUuid, columnRemap, targetOrg, remapTextStoredUuids })
|
|
765
|
+
}
|
|
766
|
+
process.stderr.write('\n')
|
|
767
|
+
|
|
768
|
+
await tgt.client.query("SET session_replication_role = 'origin'")
|
|
769
|
+
await tgt.client.query('COMMIT')
|
|
770
|
+
console.log(`\nEXECUTE complete — ${written} rows written to ${targetOrg} @ ${args.targetNamespace}.`)
|
|
771
|
+
if (createdOrg) console.log(`Created org ${targetOrg} ("${args.name}") with owner ${args.ownerUser} (role Owner).`)
|
|
772
|
+
// copy the actual files (S3) for every copied blob row — on by default (skip with --skip-files)
|
|
773
|
+
if (!args.skipFiles) await copyFiles(src, args, map, remapUuid)
|
|
774
|
+
else
|
|
775
|
+
console.log('(--skip-files: S3 objects NOT copied; copied documents will have broken file links until copied.)')
|
|
776
|
+
} catch (e) {
|
|
777
|
+
if (args.execute) {
|
|
778
|
+
try {
|
|
779
|
+
await tgt.client.query('ROLLBACK')
|
|
780
|
+
} catch (_) {}
|
|
781
|
+
}
|
|
782
|
+
throw e
|
|
783
|
+
} finally {
|
|
784
|
+
await src.end()
|
|
785
|
+
await tgt.end()
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
// build the `<col> IN (SELECT <parentCol> FROM <parent> WHERE …)` clause for an orphan's fkFilter.
|
|
790
|
+
// Terminal hop: the parent has an org column (`parentOrgColumn=$1`). Multi-level: the parent is itself
|
|
791
|
+
// an orphan (`parentFilter`) → recurse so the whole chain roots at an org-scoped ancestor.
|
|
792
|
+
function fkFilterSql(f) {
|
|
793
|
+
const inner = f.parentOrgColumn ? `${qIdent(f.parentOrgColumn)}=$1` : fkFilterSql(f.parentFilter)
|
|
794
|
+
return `${qIdent(f.column)} IN (SELECT ${qIdent(f.parentColumn)} FROM ${qTable(
|
|
795
|
+
f.parentSchema,
|
|
796
|
+
f.parentTable
|
|
797
|
+
)} WHERE ${inner})`
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
// build a WHERE clause selecting the org's own rows (+ an optional per-table extraFilter, which may
|
|
801
|
+
// reference $1 = the org id — e.g. restrict a big table to rows tied to an active task)
|
|
802
|
+
function filterClause(t, orgId, { includeExtra = true } = {}) {
|
|
803
|
+
let sql
|
|
804
|
+
if (t.special && t.special.some((s) => s.startsWith('crossOrgRelation'))) {
|
|
805
|
+
sql = `(parent_company_id=$1 OR child_company_id=$1)`
|
|
806
|
+
} else if (t.fkFilter) {
|
|
807
|
+
// orphan child table (no org column): select rows whose FK points into a copied parent's org rows.
|
|
808
|
+
// A multi-level orphan (child of an orphan, e.g. document_files_converted → document_files →
|
|
809
|
+
// documents) carries a NESTED parentFilter instead of a terminal parentOrgColumn — recurse.
|
|
810
|
+
sql = fkFilterSql(t.fkFilter)
|
|
811
|
+
} else if (!t.filterColumn) {
|
|
812
|
+
return null
|
|
813
|
+
} else {
|
|
814
|
+
sql = `${qIdent(t.filterColumn)}=$1`
|
|
815
|
+
}
|
|
816
|
+
// extraFilter (e.g. the active-task restriction) narrows a COPY; a PURGE must remove ALL the org's rows,
|
|
817
|
+
// so callers deleting data pass includeExtra:false.
|
|
818
|
+
if (t.extraFilter && includeExtra) sql = `${sql} AND (${t.extraFilter})`
|
|
819
|
+
return { sql, params: [orgId] }
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
// --name path: create the target org by cloning the source companies row (fresh id / name / short_name
|
|
823
|
+
// / api_uuid) and linking the given owner user with role 'Owner'. Never creates the user itself.
|
|
824
|
+
async function createTargetOrg(src, tgt, args, targetOrg) {
|
|
825
|
+
const cols = (await columnsOf(src, 'service_accounts', 'companies')).filter((c) => !c.generated)
|
|
826
|
+
const row = (
|
|
827
|
+
await src.query(
|
|
828
|
+
`SELECT ${cols.map((c) => qIdent(c.name)).join(',')} FROM service_accounts.companies WHERE companies_pk=$1`,
|
|
829
|
+
[args.source]
|
|
830
|
+
)
|
|
831
|
+
).rows[0]
|
|
832
|
+
if (!row) throw new Error(`source companies row ${args.source} not found`)
|
|
833
|
+
const slug =
|
|
834
|
+
args.name
|
|
835
|
+
.toLowerCase()
|
|
836
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
837
|
+
.replace(/^-|-$/g, '')
|
|
838
|
+
.slice(0, 40) || 'org'
|
|
839
|
+
const overrides = {
|
|
840
|
+
companies_pk: targetOrg,
|
|
841
|
+
company_name: args.name,
|
|
842
|
+
short_name: `${slug}-${newUuid().slice(0, 8)}`, // short_name is UNIQUE
|
|
843
|
+
api_uuid: newUuid(),
|
|
844
|
+
}
|
|
845
|
+
const vals = cols.map((c) => {
|
|
846
|
+
if (Object.prototype.hasOwnProperty.call(overrides, c.name)) return overrides[c.name]
|
|
847
|
+
let v = row[c.name]
|
|
848
|
+
if ((c.type === 'jsonb' || c.type === 'json') && v != null) v = JSON.stringify(v)
|
|
849
|
+
return v
|
|
850
|
+
})
|
|
851
|
+
await tgt.client.query(
|
|
852
|
+
`INSERT INTO service_accounts.companies (${cols.map((c) => qIdent(c.name)).join(',')}) VALUES (${cols
|
|
853
|
+
.map((_, i) => `$${i + 1}`)
|
|
854
|
+
.join(',')})`,
|
|
855
|
+
vals
|
|
856
|
+
)
|
|
857
|
+
await tgt.client.query(
|
|
858
|
+
`INSERT INTO service_accounts.companies2users (fk_companies_pk, fk_users_pk, fk_user_roles_at_companies_pk, api_uuid) VALUES ($1,$2,'Owner',$3)`,
|
|
859
|
+
[targetOrg, args.ownerUser, newUuid()]
|
|
860
|
+
)
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// ── COPY (text format) encoding — much faster than row-by-row INSERT ─────────
|
|
864
|
+
// escape a field for COPY text format: backslash, tab, newline, CR
|
|
865
|
+
const copyEscape = (s) => s.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/\r/g, '\\r').replace(/\t/g, '\\t')
|
|
866
|
+
// Postgres array literal for a JS array (quote every element; NULL for null)
|
|
867
|
+
function arrayLiteral(arr) {
|
|
868
|
+
return (
|
|
869
|
+
'{' +
|
|
870
|
+
arr
|
|
871
|
+
.map((el) =>
|
|
872
|
+
el === null || el === undefined ? 'NULL' : '"' + String(el).replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '"'
|
|
873
|
+
)
|
|
874
|
+
.join(',') +
|
|
875
|
+
'}'
|
|
876
|
+
)
|
|
877
|
+
}
|
|
878
|
+
// encode one already-remapped value for its column's type into a COPY field
|
|
879
|
+
function encodeCopyField(col, v) {
|
|
880
|
+
if (v === null || v === undefined) return '\\N'
|
|
881
|
+
if (col.type === 'ARRAY') return copyEscape(arrayLiteral(Array.isArray(v) ? v : [v]))
|
|
882
|
+
if (col.type === 'boolean') return v ? 't' : 'f'
|
|
883
|
+
if (col.type === 'jsonb' || col.type === 'json') return copyEscape(typeof v === 'string' ? v : JSON.stringify(v))
|
|
884
|
+
return copyEscape(String(v))
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
// copy one table's source rows into the target via COPY … FROM STDIN. Reads the source with a
|
|
888
|
+
// server-side CURSOR in batches (constant memory regardless of table size) and streams each batch
|
|
889
|
+
// into the COPY with backpressure. Requires src and tgt to be separate connections.
|
|
890
|
+
const READ_BATCH = 2000
|
|
891
|
+
async function copyTable(src, tgt, p, args, ctx) {
|
|
892
|
+
const { remapUuid, columnRemap, targetOrg, remapTextStoredUuids } = ctx
|
|
893
|
+
const t = p.t
|
|
894
|
+
const srcCols = p.srcCols || (await columnsOf(src, t.schema, t.table))
|
|
895
|
+
const tgtCols = await columnsOf(tgt, t.schema, t.table)
|
|
896
|
+
const tgtNames = new Set(tgtCols.map((c) => c.name))
|
|
897
|
+
// insertable columns = present in both, not generated
|
|
898
|
+
const cols = srcCols.filter((c) => tgtNames.has(c.name) && !c.generated)
|
|
899
|
+
const f = filterClause(t, args.source)
|
|
900
|
+
const colList = cols.map((c) => qIdent(c.name)).join(',')
|
|
901
|
+
const rctx = {
|
|
902
|
+
remapUuid,
|
|
903
|
+
columnRemap,
|
|
904
|
+
targetOrg,
|
|
905
|
+
emailSet: new Set(t.emailColumns || []),
|
|
906
|
+
emailScrub: args.emailScrub,
|
|
907
|
+
table: t,
|
|
908
|
+
remapTextStoredUuids,
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
const cursor = src.client.query(
|
|
912
|
+
new Cursor(`SELECT ${colList} FROM ${qTable(t.schema, t.table)} WHERE ${f.sql}`, f.params)
|
|
913
|
+
)
|
|
914
|
+
const readBatch = () =>
|
|
915
|
+
new Promise((resolve, reject) => cursor.read(READ_BATCH, (err, rows) => (err ? reject(err) : resolve(rows))))
|
|
916
|
+
const closeCursor = () => new Promise((resolve) => cursor.close(() => resolve()))
|
|
917
|
+
|
|
918
|
+
let n = 0
|
|
919
|
+
// async generator pulls a batch at a time from the cursor (constant memory) and yields COPY lines;
|
|
920
|
+
// pipeline() wires it to the COPY stream with backpressure AND one clean error path (any COPY/read
|
|
921
|
+
// error rejects here and is caught by the caller → transaction rollback).
|
|
922
|
+
async function* lines() {
|
|
923
|
+
for (let batch = await readBatch(); batch.length; batch = await readBatch()) {
|
|
924
|
+
for (const row of batch) {
|
|
925
|
+
yield cols.map((c) => encodeCopyField(c, remapValue(c, row[c.name], rctx))).join('\t') + '\n'
|
|
926
|
+
}
|
|
927
|
+
n += batch.length
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
try {
|
|
931
|
+
await pipeline(
|
|
932
|
+
Readable.from(lines()),
|
|
933
|
+
tgt.client.query(copyFrom(`COPY ${qTable(t.schema, t.table)} (${colList}) FROM STDIN`))
|
|
934
|
+
)
|
|
935
|
+
} finally {
|
|
936
|
+
await closeCursor()
|
|
937
|
+
}
|
|
938
|
+
return n
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
// copy specific referenced-user rows into the target (the ones missing there). Each gets a new derived
|
|
942
|
+
// id (already added to sourceUuids, so refs follow); `customer`-role rows get a UNIQUE anonymized email
|
|
943
|
+
// (per the `(email, role)` unique key) and their PII/credential columns (id_number, password_hash)
|
|
944
|
+
// nulled. Every other column goes through the normal value remap. Small set → one plain SELECT, no cursor.
|
|
945
|
+
async function copyUsers(src, tgt, users, ut, ctx) {
|
|
946
|
+
const { remapUuid } = ctx
|
|
947
|
+
const oldIds = users.map((u) => u.oldId)
|
|
948
|
+
const meta = new Map(users.map((u) => [u.oldId, u]))
|
|
949
|
+
const srcCols = (await columnsOf(src, ut.schema, ut.table)).filter((c) => !c.generated)
|
|
950
|
+
const tgtNames = new Set((await columnsOf(tgt, ut.schema, ut.table)).map((c) => c.name))
|
|
951
|
+
const cols = srcCols.filter((c) => tgtNames.has(c.name))
|
|
952
|
+
const colList = cols.map((c) => qIdent(c.name)).join(',')
|
|
953
|
+
const emailSet = new Set(ut.emailColumns || [])
|
|
954
|
+
const nullSet = new Set(ut.nullColumns || [])
|
|
955
|
+
const pattern = ut.anonymizedEmailPattern || 'testcustomer+u<id>@dooer.com'
|
|
956
|
+
// neutralise the generic email scrub — user emails are handled explicitly (unique per row) below
|
|
957
|
+
const rctx = { ...ctx, emailSet: new Set(), table: { key: `${ut.schema}.${ut.table}`, emailColumns: [] } }
|
|
958
|
+
const { rows } = await src.query(
|
|
959
|
+
`SELECT ${colList} FROM ${qTable(ut.schema, ut.table)} WHERE ${qIdent(ut.pk)} = ANY($1)`,
|
|
960
|
+
[oldIds]
|
|
961
|
+
)
|
|
962
|
+
function* lines() {
|
|
963
|
+
for (const row of rows) {
|
|
964
|
+
const m = meta.get(row[ut.pk])
|
|
965
|
+
const newId = remapUuid(row[ut.pk])
|
|
966
|
+
yield cols
|
|
967
|
+
.map((c) => {
|
|
968
|
+
if (m.anonymize && emailSet.has(c.name))
|
|
969
|
+
return encodeCopyField(c, row[c.name] == null ? null : pattern.replace('<id>', newId.replace(/-/g, '')))
|
|
970
|
+
if (m.anonymize && nullSet.has(c.name)) return encodeCopyField(c, null)
|
|
971
|
+
return encodeCopyField(c, remapValue(c, row[c.name], rctx))
|
|
972
|
+
})
|
|
973
|
+
.join('\t') + '\n'
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
await pipeline(
|
|
977
|
+
Readable.from(lines()),
|
|
978
|
+
tgt.client.query(copyFrom(`COPY ${qTable(ut.schema, ut.table)} (${colList}) FROM STDIN`))
|
|
979
|
+
)
|
|
980
|
+
return rows.length
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
|
|
984
|
+
const looksLikeUuid = (s) => s.length === 36 && UUID_RE.test(s)
|
|
985
|
+
// global: every uuid-shaped substring (for remapping uuids inside raw jsonb text)
|
|
986
|
+
const UUID_SUBSTR_RE_G = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
|
|
987
|
+
|
|
988
|
+
// remap a single column value: json, email scrub, config-driven per-column remap, uuid, uuid[]
|
|
989
|
+
function remapValue(col, value, ctx) {
|
|
990
|
+
const { remapUuid, columnRemap, emailSet, emailScrub, table, remapTextStoredUuids } = ctx
|
|
991
|
+
if (value === null || value === undefined) return value
|
|
992
|
+
// json/jsonb arrives as RAW TEXT (see type parsers). Remap uuids directly in the text: replace every
|
|
993
|
+
// uuid-shaped substring that belongs to the copy with its derived uuid. A uuid → uuid replacement is
|
|
994
|
+
// same-length + valid chars, so the JSON stays valid. This avoids JSON.parse+walk+stringify — the
|
|
995
|
+
// per-row cost that made big taskAction/ocrData/documentField payloads slow and OOM the process.
|
|
996
|
+
if (col.type === 'jsonb' || col.type === 'json') {
|
|
997
|
+
const s = typeof value === 'string' ? value : JSON.stringify(value)
|
|
998
|
+
return s.replace(UUID_SUBSTR_RE_G, (m) => remapUuid(m))
|
|
999
|
+
}
|
|
1000
|
+
// email scrub (type-aware): text -> scalar, uuid never, boolean/other skipped
|
|
1001
|
+
if (emailSet.has(col.name)) {
|
|
1002
|
+
if (col.type === 'ARRAY' && col.udt === '_text') return [emailScrub]
|
|
1003
|
+
if (col.type === 'text' || col.type === 'character varying') return emailScrub
|
|
1004
|
+
// non-string email-named column (e.g. boolean flag) — leave as-is
|
|
1005
|
+
}
|
|
1006
|
+
// config-driven per-column remap (non-uuid unique columns: sequence / max-offset / text-suffix, plus
|
|
1007
|
+
// every FK column that references them — all registered in columnRemap by "schema.table.column")
|
|
1008
|
+
const cm = columnRemap.get(`${table.key}.${col.name}`)
|
|
1009
|
+
if (cm) {
|
|
1010
|
+
const mapped = cm.get(String(value))
|
|
1011
|
+
if (mapped !== undefined) return mapped
|
|
1012
|
+
}
|
|
1013
|
+
// uuid columns: rewrite if part of the copy (PKs, org id, internal FKs); leave external refs untouched
|
|
1014
|
+
if (isUuid(col)) return remapUuid(value)
|
|
1015
|
+
if (isUuidArray(col) && Array.isArray(value)) return value.map((v) => (v ? remapUuid(v) : v))
|
|
1016
|
+
// a text/varchar value equal to a copied uuid is a reference (polymorphic id stored as text) → remap
|
|
1017
|
+
// (only for uuid-shaped values, so we don't hash every ordinary text field)
|
|
1018
|
+
if (remapTextStoredUuids && (col.type === 'text' || col.type === 'character varying') && looksLikeUuid(value))
|
|
1019
|
+
return remapUuid(value)
|
|
1020
|
+
return value
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
// helpers reused by sibling commands (purge, shred, db build)
|
|
1024
|
+
module.exports = {
|
|
1025
|
+
main,
|
|
1026
|
+
parseArgs,
|
|
1027
|
+
HELP,
|
|
1028
|
+
connect,
|
|
1029
|
+
withRetry,
|
|
1030
|
+
filterClause,
|
|
1031
|
+
qIdent,
|
|
1032
|
+
qTable,
|
|
1033
|
+
columnsOf,
|
|
1034
|
+
s3Config,
|
|
1035
|
+
s3Get,
|
|
1036
|
+
DeleteObjectCommand,
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
// still runnable standalone (node lib/engine/seed.js …) for now
|
|
1040
|
+
if (require.main === module) {
|
|
1041
|
+
main().catch((e) => {
|
|
1042
|
+
console.error(`\nERROR: ${e.message}\n`)
|
|
1043
|
+
process.exit(1)
|
|
1044
|
+
})
|
|
1045
|
+
}
|