@dooer/dooer-test-env 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/bin/index.js +7 -0
  2. package/discovery-router/Dockerfile +18 -0
  3. package/discovery-router/README.md +99 -0
  4. package/discovery-router/package.json +13 -0
  5. package/discovery-router/registry.example.json +5 -0
  6. package/discovery-router/server.js +272 -0
  7. package/lib/account.js +120 -0
  8. package/lib/auth-dev-keys.js +12 -0
  9. package/lib/bankid.js +130 -0
  10. package/lib/cli.js +27 -0
  11. package/lib/command/bankid.js +45 -0
  12. package/lib/command/customer.js +108 -0
  13. package/lib/command/db.js +156 -0
  14. package/lib/command/env.js +114 -0
  15. package/lib/command/logs.js +143 -0
  16. package/lib/command/measure.js +81 -0
  17. package/lib/command/service.js +166 -0
  18. package/lib/command/setup.js +92 -0
  19. package/lib/command/shred.js +60 -0
  20. package/lib/compose/README.md +98 -0
  21. package/lib/compose/generate.js +375 -0
  22. package/lib/compose/manifests.js +108 -0
  23. package/lib/db/roles.js +118 -0
  24. package/lib/discovery/client.js +40 -0
  25. package/lib/engine/GUIDE.md +176 -0
  26. package/lib/engine/PROCESS.md +571 -0
  27. package/lib/engine/dbbuild.js +325 -0
  28. package/lib/engine/gen-schema-map.js +479 -0
  29. package/lib/engine/purge.js +137 -0
  30. package/lib/engine/schema-map.json +11016 -0
  31. package/lib/engine/seed.js +1045 -0
  32. package/lib/obc.js +72 -0
  33. package/lib/registry.js +123 -0
  34. package/lib/runtime.js +101 -0
  35. package/lib/service-token.js +40 -0
  36. package/lib/shred/README.md +118 -0
  37. package/lib/shred/audit.js +128 -0
  38. package/lib/shred/faker.js +545 -0
  39. package/lib/shred/index.js +126 -0
  40. package/lib/shred/scripts/base-partner-emails.sql +9 -0
  41. package/lib/shred/scripts/dev-accounts.sql +195 -0
  42. package/lib/shred/scripts/emails.sql +48 -0
  43. package/lib/shred/scripts/institution-browser.sql +3 -0
  44. package/lib/shred/scripts/notification-targets.sql +5 -0
  45. package/lib/shred/scripts/partners.sql +2 -0
  46. package/lib/shred/scripts/passwords.sql +8 -0
  47. package/lib/shred/scripts/personal-numbers.sql +177 -0
  48. package/lib/shred/scripts/phone-numbers.sql +22 -0
  49. package/lib/shred/scripts/salary-spec-reports.sql +5 -0
  50. package/lib/shred/scripts/service-activity-tracker-data.sql +4 -0
  51. package/lib/shred/scripts/service-core-objects.sql +19 -0
  52. package/lib/shred/scripts/service-event-stream.sql +2 -0
  53. package/lib/shred/scripts/service-integrations.sql +4 -0
  54. package/lib/shred/scripts/template.sql +4 -0
  55. package/lib/shred/scripts/x-service-billing.sql +34 -0
  56. package/lib/shred/scripts/xxx-history-tables.sql +25 -0
  57. package/lib/stub.js +8 -0
  58. package/local-postgres/Dockerfile +11 -0
  59. package/package.json +46 -0
  60. package/readme.md +92 -0
@@ -0,0 +1,1045 @@
1
+ #!/usr/bin/env node
2
+ /*
3
+ * seed-test-account.js — copy ALL customer-specific data for one Dooer organization into another.
4
+ *
5
+ * WHAT IT DOES
6
+ * Reads every org-scoped table (schema-map.json, action:"copy") for the SOURCE org and inserts the
7
+ * rows into the TARGET org, in the SAME database or ACROSS namespaces (e.g. read dooer-production,
8
+ * write dooer-staging). The org identity is one uuid (organizationId == companies.companies_pk); the
9
+ * source org uuid is remapped to the target org uuid, and every other copied uuid PK is remapped to a
10
+ * fresh uuid so intra-org references rewire automatically. References to non-copied rows (users,
11
+ * global templates) are left untouched. Emails are scrubbed so the source customer gets no mail.
12
+ *
13
+ * SAFETY
14
+ * - DRY-RUN BY DEFAULT ("dummy" mode): prints the full plan and writes NOTHING. Pass --execute to write.
15
+ * - Writing to the dooer-production namespace additionally requires --confirm-production.
16
+ * - All writes run inside ONE transaction on the target; any error rolls the whole copy back.
17
+ * - Referenced users missing from the target ARE copied (customer emails anonymized); existing users and
18
+ * the target org's own account/owners are left as-is. Never creates brand-new login users.
19
+ *
20
+ * RESILIENCE (flaky VPN)
21
+ * - Reads reconnect + retry on a dropped connection (pass-1, discovery, strategy/guard reads, file-id read).
22
+ * - The remap salt is derived from (source, target), so RE-RUNNING the same copy yields identical ids.
23
+ * - --only / --skip-tables copy a subset (pass-1 still scans all → cross-refs remap), so a big org can be
24
+ * copied in survivable chunks; re-run with the same --target and --if-target-nonempty insert. See GUIDE.md.
25
+ *
26
+ * CONNECTION
27
+ * Per namespace it reads the dooer-database ClusterIP and the postgres password from kubectl
28
+ * (requires VPN access to the ClusterIP). Nothing is written to disk.
29
+ *
30
+ * USAGE (see GUIDE.md for full examples)
31
+ * node seed-test-account.js --source <uuid> --target <uuid> \
32
+ * [--source-namespace dooer-staging] [--target-namespace dooer-staging] \
33
+ * [--execute] [--confirm-production] [--email testcustomer@dooer.com]
34
+ * node seed-test-account.js --source <uuid> --name "Copy of X" --owner-user <uuid> --execute
35
+ */
36
+
37
+ const { execFileSync } = require('child_process')
38
+ const crypto = require('crypto')
39
+ const path = require('path')
40
+ const { Client, types } = require('pg')
41
+ const copyFrom = require('pg-copy-streams').from
42
+ const Cursor = require('pg-cursor')
43
+ const { Readable } = require('stream')
44
+ const { pipeline } = require('stream/promises')
45
+ const { S3Client, GetObjectCommand, PutObjectCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3')
46
+
47
+ // Read date/timestamp columns as their raw Postgres text (no JS Date round-trip), so COPY writes them
48
+ // back verbatim with no timezone drift. numeric/int8/uuid already come back as strings.
49
+ types.setTypeParser(1082, (v) => v) // date
50
+ types.setTypeParser(1114, (v) => v) // timestamp
51
+ types.setTypeParser(1184, (v) => v) // timestamptz
52
+ // Read json/jsonb as RAW TEXT too: node-pg parses a jsonb value of `null` into JS null, which is
53
+ // indistinguishable from a SQL NULL — writing it back as SQL NULL breaks NOT-NULL jsonb columns.
54
+ // Raw text keeps JSON null as the string "null" (distinct from a real SQL NULL, which stays JS null).
55
+ types.setTypeParser(114, (v) => v) // json
56
+ types.setTypeParser(3802, (v) => v) // jsonb
57
+
58
+ // ── args ─────────────────────────────────────────────────────────────────────
59
+ function parseArgs(argv) {
60
+ const a = {
61
+ emailScrub: 'testcustomer@dooer.com',
62
+ sourceNamespace: 'dooer-staging',
63
+ mapPath: path.join(__dirname, 'schema-map.json'),
64
+ }
65
+ for (let i = 0; i < argv.length; i++) {
66
+ const k = argv[i]
67
+ const val = () => argv[++i]
68
+ if (k === '--source') a.source = val()
69
+ else if (k === '--target') a.target = val()
70
+ else if (k === '--name') a.name = val()
71
+ else if (k === '--owner-user') a.ownerUser = val()
72
+ else if (k === '--source-namespace') a.sourceNamespace = val()
73
+ else if (k === '--target-namespace') a.targetNamespace = val()
74
+ else if (k === '--email') a.emailScrub = val()
75
+ else if (k === '--map') a.mapPath = val()
76
+ else if (k === '--execute') a.execute = true
77
+ else if (k === '--skip-files') a.skipFiles = true
78
+ else if (k === '--skip-users') a.skipUsers = true
79
+ else if (k === '--confirm-production') a.confirmProduction = true
80
+ else if (k === '--if-target-nonempty') a.ifTargetNonempty = val()
81
+ // refuse (default) | insert
82
+ else if (k === '--salt') a.salt = val()
83
+ // extra entropy folded into the deterministic remap salt
84
+ else if (k === '--only')
85
+ a.only = val()
86
+ .split(',')
87
+ .map((s) => s.trim())
88
+ .filter(Boolean)
89
+ // copy ONLY these tables (pass-1 still scans all)
90
+ else if (k === '--skip-tables')
91
+ a.skipTables = val()
92
+ .split(',')
93
+ .map((s) => s.trim())
94
+ .filter(Boolean)
95
+ // copy all EXCEPT these
96
+ else if (k === '-h' || k === '--help') a.help = true
97
+ else throw new Error(`unknown argument: ${k}`)
98
+ }
99
+ if (!a.targetNamespace) a.targetNamespace = a.sourceNamespace
100
+ if (!a.ifTargetNonempty) a.ifTargetNonempty = 'refuse'
101
+ return a
102
+ }
103
+
104
+ const HELP = `seed-test-account — copy one Dooer org's data into another (test) org.
105
+
106
+ Required:
107
+ --source <uuid> org to copy FROM (the real company)
108
+ --target <uuid> org to copy INTO (existing test entity)
109
+ ...or, to create a new target org:
110
+ --name "<string>" name for a new target company row (source's row is cloned, name replaced)
111
+ --owner-user <uuid> user to set as the new org's owner (companies2users) — REQUIRED when creating
112
+
113
+ Optional:
114
+ --source-namespace <ns> default dooer-staging
115
+ --target-namespace <ns> default = source-namespace (set differently to copy across clusters,
116
+ e.g. --source-namespace dooer-production --target-namespace dooer-staging)
117
+ --email <addr> address to scrub every email column to (default testcustomer@dooer.com)
118
+ --skip-users do NOT copy referenced users missing from the target (default: copy them,
119
+ anonymizing customer-role emails). Same-namespace copies are a no-op anyway.
120
+ --if-target-nonempty refuse|insert what to do if a copy table already has target-org rows (default refuse)
121
+ --only <a.b,c.d> copy ONLY these tables (pass-1 still scans all, so cross-refs remap). For
122
+ resuming/chunking a big copy over a flaky link — re-run with the SAME
123
+ --target and --if-target-nonempty insert. Keep transactions + its twins together.
124
+ --skip-tables <a.b,c.d> copy all tables EXCEPT these (same chunking use)
125
+ --salt <string> extra entropy folded into the remap salt (normally unneeded; the salt is
126
+ already derived from source+target so re-runs are deterministic/resumable)
127
+ --execute actually write (default is DRY-RUN / "dummy": plan only, no writes)
128
+ --confirm-production REQUIRED to write when --target-namespace is dooer-production
129
+ --map <path> schema map (default ./schema-map.json)
130
+ `
131
+
132
+ // ── kubectl-sourced connection ───────────────────────────────────────────────
133
+ function sh(file, args) {
134
+ return execFileSync(file, args, { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }).trim()
135
+ }
136
+ function connInfo(namespace) {
137
+ const clusterIP = sh('kubectl', ['get', 'svc', 'dooer-database', '-n', namespace, '-o', 'jsonpath={.spec.clusterIP}'])
138
+ const b64 = sh('kubectl', [
139
+ 'get',
140
+ 'secret',
141
+ 'postgres.dooer-database.credentials.postgresql.acid.zalan.do',
142
+ '-n',
143
+ namespace,
144
+ '-o',
145
+ 'jsonpath={.data.password}',
146
+ ])
147
+ const password = Buffer.from(b64, 'base64').toString('utf8')
148
+ if (!clusterIP || !password) throw new Error(`could not read dooer-database conn info for namespace ${namespace}`)
149
+ return { host: clusterIP, port: 5432, user: 'postgres', password, database: 'dooer' }
150
+ }
151
+ async function rawConnect(namespace) {
152
+ const info = connInfo(namespace)
153
+ const client = new Client({
154
+ ...info,
155
+ ssl: { rejectUnauthorized: false },
156
+ statement_timeout: 0,
157
+ keepAlive: true, // TCP keepalive so an idle connection is not dropped during the long file phase
158
+ keepAliveInitialDelayMillis: 30000,
159
+ })
160
+ // An unexpected disconnect emits 'error'; unhandled it crashes the process. Swallow+log so the
161
+ // caller's next query surfaces the failure normally and we can reconnect. Marked so callers detect it.
162
+ client.on('error', (e) => {
163
+ client._seedDead = e
164
+ console.error(`\n[pg ${namespace}] connection error: ${e.message}`)
165
+ })
166
+ await client.connect()
167
+ return client
168
+ }
169
+
170
+ // A self-healing connection: `.query()` retries transient drops, reconnecting a dead client first, so a
171
+ // flaky VPN (read ETIMEDOUT / "Connection terminated unexpectedly") doesn't kill a read. `.client`
172
+ // exposes the live pg Client for the streaming paths (cursor / COPY), which are retried a level up (whole
173
+ // table) since a mid-stream drop can't resume the same statement.
174
+ async function connect(namespace) {
175
+ let client = await rawConnect(namespace)
176
+ return {
177
+ namespace,
178
+ get client() {
179
+ return client
180
+ },
181
+ async reconnect() {
182
+ try {
183
+ await client.end()
184
+ } catch (_) {}
185
+ client = await rawConnect(namespace)
186
+ return client
187
+ },
188
+ async query(sql, params) {
189
+ return withRetry(
190
+ async () => {
191
+ if (client._seedDead) await this.reconnect()
192
+ try {
193
+ return await client.query(sql, params)
194
+ } catch (e) {
195
+ if (isTransient(e)) await this.reconnect()
196
+ throw e
197
+ }
198
+ },
199
+ { label: `query ${namespace}` }
200
+ )
201
+ },
202
+ async end() {
203
+ try {
204
+ await client.end()
205
+ } catch (_) {}
206
+ },
207
+ }
208
+ }
209
+
210
+ // transient = a network/server hiccup worth retrying (vs a real SQL/logic error, which must surface)
211
+ function isTransient(e) {
212
+ return /terminat|ECONNRESET|ETIMEDOUT|EPIPE|socket hang up|Connection terminated|timeout|503|500|InternalError|RequestTimeout|SlowDown|Client has encountered a connection error/i.test(
213
+ `${e.name} ${e.message} ${e.code || ''} ${e.$metadata?.httpStatusCode || ''}`
214
+ )
215
+ }
216
+
217
+ // run an async op with a few retries on transient network/S3 errors (exponential-ish backoff)
218
+ async function withRetry(fn, { tries = 6, label = 'op' } = {}) {
219
+ let last
220
+ for (let i = 0; i < tries; i++) {
221
+ try {
222
+ return await fn()
223
+ } catch (e) {
224
+ last = e
225
+ if (i === tries - 1 || !isTransient(e)) throw e
226
+ const wait = Math.min(8000, 500 * 2 ** i)
227
+ process.stderr.write(`\n[retry ${label}] ${e.message} — attempt ${i + 2}/${tries} in ${wait}ms\n`)
228
+ await new Promise((r) => setTimeout(r, wait))
229
+ }
230
+ }
231
+ throw last
232
+ }
233
+
234
+ // ── S3 (Ceph RGW) file copy ──────────────────────────────────────────────────
235
+ // The file bytes for a copied fileData row live in S3 keyed by the fileData id. We copy each object
236
+ // from the SOURCE env bucket to the TARGET env bucket at its NEW key (= the copied fileData id), so the
237
+ // copy owns its own files. Endpoint/bucket/credentials are read from the file-storage pod per namespace.
238
+ function s3Config(namespace) {
239
+ const pods = sh('kubectl', [
240
+ 'get',
241
+ 'pods',
242
+ '-n',
243
+ namespace,
244
+ '-o',
245
+ 'jsonpath={range .items[*]}{.metadata.name}{"\\n"}{end}',
246
+ ])
247
+ .split('\n')
248
+ .filter((p) => /^service-file-storage-[a-z0-9]+-[a-z0-9]+$/.test(p))
249
+ const pod = pods[0]
250
+ if (!pod) throw new Error(`no service-file-storage pod found in ${namespace} (needed for the S3 file copy)`)
251
+ const allEnv = sh('kubectl', ['exec', '-n', namespace, pod, '--', 'env'])
252
+ const env = (v) => {
253
+ const m = allEnv.match(new RegExp('^' + v + '=(.*)$', 'm'))
254
+ return m ? m[1] : ''
255
+ }
256
+ const svcName = env('AWS_ENDPOINT_URL_S3')
257
+ .replace(/^https?:\/\//, '')
258
+ .split('.')[0]
259
+ const clusterIP = sh('kubectl', ['get', 'svc', svcName, '-n', 'rook-ceph', '-o', 'jsonpath={.spec.clusterIP}'])
260
+ const endpoint = `http://${clusterIP}:80`
261
+ const mk = (ak, sk) =>
262
+ new S3Client({
263
+ endpoint,
264
+ region: 'us-east-1',
265
+ forcePathStyle: true,
266
+ credentials: { accessKeyId: ak, secretAccessKey: sk },
267
+ })
268
+ const cfg = {
269
+ primary: {
270
+ bucket: env('DOOER_STORAGE_PROVIDER_AMAZON_S3_BUCKET'),
271
+ client: mk(env('FILE_STORAGE_BUCKET_ACCESS_KEY_ID'), env('FILE_STORAGE_BUCKET_SECRET_ACCESS_KEY')),
272
+ },
273
+ fallback: null,
274
+ }
275
+ const fb = env('DOOER_STORAGE_PROVIDER_AMAZON_S3_BUCKET_FALLBACK')
276
+ const fak = env('FILE_STORAGE_FALLBACK_BUCKET_ACCESS_KEY_ID')
277
+ if (fb && fak) cfg.fallback = { bucket: fb, client: mk(fak, env('FILE_STORAGE_FALLBACK_BUCKET_SECRET_ACCESS_KEY')) }
278
+ return cfg
279
+ }
280
+
281
+ // get an object from a source config, trying the primary bucket then the fallback bucket
282
+ async function s3Get(cfg, key) {
283
+ try {
284
+ return await cfg.primary.client.send(new GetObjectCommand({ Bucket: cfg.primary.bucket, Key: key }))
285
+ } catch (e) {
286
+ const notFound = e.name === 'NoSuchKey' || e.name === 'NotFound' || e.$metadata?.httpStatusCode === 404
287
+ if (notFound && cfg.fallback)
288
+ return await cfg.fallback.client.send(new GetObjectCommand({ Bucket: cfg.fallback.bucket, Key: key }))
289
+ throw e
290
+ }
291
+ }
292
+
293
+ // copy the S3 object for every copied s3Blob row (old key → its remapped new key)
294
+ async function copyFiles(src, args, map, remapUuid) {
295
+ const s3Tables = (map.s3BlobTables || [])
296
+ .map((k) => map.tables.find((t) => t.key === k))
297
+ .filter((t) => t && t.action === 'copy')
298
+ if (!s3Tables.length) return
299
+ const srcS3 = s3Config(args.sourceNamespace)
300
+ const tgtS3 = s3Config(args.targetNamespace)
301
+ for (const t of s3Tables) {
302
+ const f = filterClause(t, args.source)
303
+ // src is a self-healing connection — `.query` reconnects a client dropped during the long copy/txn
304
+ // and retries the read on its own.
305
+ const rows = await src.query(
306
+ `SELECT ${qIdent(t.pk.column)} AS id FROM ${qTable(t.schema, t.table)} WHERE ${f.sql}`,
307
+ f.params
308
+ )
309
+ const keys = rows.rows.map((r) => r.id).filter(Boolean)
310
+ console.log(
311
+ `\nS3: copying ${keys.length} ${t.key} objects (${srcS3.primary.bucket}[+fallback] → ${tgtS3.primary.bucket})…`
312
+ )
313
+ let done = 0,
314
+ missing = 0,
315
+ failed = 0,
316
+ idx = 0
317
+ const worker = async () => {
318
+ while (idx < keys.length) {
319
+ const oldKey = keys[idx++]
320
+ const newKey = remapUuid(oldKey)
321
+ try {
322
+ await withRetry(
323
+ async () => {
324
+ const obj = await s3Get(srcS3, oldKey) // primary then fallback
325
+ // buffer the body (streaming GetObject→PutObject fails against Ceph RGW)
326
+ const bytes = await obj.Body.transformToByteArray()
327
+ await tgtS3.primary.client.send(
328
+ new PutObjectCommand({
329
+ Bucket: tgtS3.primary.bucket,
330
+ Key: newKey,
331
+ Body: bytes,
332
+ ContentType: obj.ContentType,
333
+ })
334
+ )
335
+ },
336
+ { tries: 4, label: `S3 ${oldKey}` }
337
+ )
338
+ done++
339
+ } catch (e) {
340
+ if (e.name === 'NoSuchKey' || e.name === 'NotFound' || e.$metadata?.httpStatusCode === 404) missing++
341
+ else if (++failed <= 5) console.error(` S3 copy failed for ${oldKey}: ${e.name}`)
342
+ }
343
+ const n = done + missing + failed
344
+ if (n % 200 === 0)
345
+ process.stderr.write(`\r S3 ${n}/${keys.length} (ok ${done}, missing ${missing}, failed ${failed})`)
346
+ }
347
+ }
348
+ await Promise.all(Array.from({ length: 12 }, worker))
349
+ process.stderr.write('\n')
350
+ console.log(`S3: ${t.key} — copied ${done}, missing-in-source ${missing}, failed ${failed}.`)
351
+ if (failed) throw new Error(`S3 copy had ${failed} failures for ${t.key} (see log)`)
352
+ }
353
+ }
354
+
355
+ // ── introspection helpers ────────────────────────────────────────────────────
356
+ const qIdent = (s) => '"' + String(s).replace(/"/g, '""') + '"'
357
+ const qTable = (schema, table) => `${qIdent(schema)}.${qIdent(table)}`
358
+
359
+ async function tableExists(client, schema, table) {
360
+ const r = await client.query(
361
+ `SELECT 1 FROM information_schema.tables WHERE table_schema=$1 AND table_name=$2 AND table_type='BASE TABLE'`,
362
+ [schema, table]
363
+ )
364
+ return r.rowCount > 0
365
+ }
366
+ async function columnsOf(client, schema, table) {
367
+ const r = await client.query(
368
+ `SELECT column_name, data_type, udt_name, is_generated,
369
+ (column_default LIKE 'nextval(%') AS is_serial
370
+ FROM information_schema.columns WHERE table_schema=$1 AND table_name=$2 ORDER BY ordinal_position`,
371
+ [schema, table]
372
+ )
373
+ return r.rows.map((c) => ({
374
+ name: c.column_name,
375
+ type: c.data_type, // 'uuid' | 'ARRAY' | 'text' | 'integer' | ...
376
+ udt: c.udt_name, // '_uuid' for uuid[], 'uuid', 'text', ...
377
+ generated: c.is_generated !== 'NEVER',
378
+ serial: c.is_serial,
379
+ }))
380
+ }
381
+ const isUuid = (col) => col.type === 'uuid'
382
+ const isUuidArray = (col) => col.type === 'ARRAY' && col.udt === '_uuid'
383
+ const newUuid = () => crypto.randomUUID()
384
+
385
+ // Deterministic derivation of a fresh uuid from a source uuid (v5-style). This lets us remap references
386
+ // WITHOUT storing a giant old→new map: we keep only the SET of copied source uuids and recompute the
387
+ // target uuid on demand. The salt is derived from (source, target, --salt) so RE-RUNNING the same copy
388
+ // reproduces IDENTICAL target ids — which makes a copy idempotent/resumable over a flaky link. (Was
389
+ // random per run.) A different target org gets different ids (its target uuid differs).
390
+ let REMAP_SALT = crypto.randomBytes(16)
391
+ function setRemapSalt(source, target, extra) {
392
+ REMAP_SALT = crypto
393
+ .createHash('sha256')
394
+ .update(`${source}|${target}|${extra || ''}`)
395
+ .digest()
396
+ .subarray(0, 16)
397
+ }
398
+ function deriveUuid(old) {
399
+ const h = crypto.createHash('sha1')
400
+ h.update(REMAP_SALT)
401
+ h.update(Buffer.from(old.replace(/-/g, ''), 'hex'))
402
+ const b = h.digest().subarray(0, 16)
403
+ b[6] = (b[6] & 0x0f) | 0x50 // version 5
404
+ b[8] = (b[8] & 0x3f) | 0x80 // variant
405
+ const x = b.toString('hex')
406
+ return `${x.slice(0, 8)}-${x.slice(8, 12)}-${x.slice(12, 16)}-${x.slice(16, 20)}-${x.slice(20, 32)}`
407
+ }
408
+
409
+ // ── uniqueness handling is CONFIG-DRIVEN (schema-map.json → uniquenessStrategies). NO column names or
410
+ // strategies are hardcoded here; the two views below are derived from the config object.
411
+ // uuidRemapCols(map) → Map("schema.table" -> [uuid col names]) (strategy: remap-uuid)
412
+ // nonUuidStrategies(map) → the config entries needing a per-column remap map (sequence/max-offset/text-suffix)
413
+ function uuidRemapCols(map) {
414
+ const m = new Map()
415
+ for (const c of map.uniquenessStrategies.columns) {
416
+ if (c.type !== 'uuid' || c.strategy !== 'remap-uuid') continue
417
+ const k = `${c.schema}.${c.table}`
418
+ if (!m.has(k)) m.set(k, [])
419
+ m.get(k).push(c.column)
420
+ }
421
+ return m
422
+ }
423
+ const nonUuidStrategies = (map) =>
424
+ map.uniquenessStrategies.columns.filter((c) => c.type !== 'uuid' && c.strategy !== 'preserve')
425
+
426
+ async function main(passedArgs) {
427
+ const args = passedArgs || parseArgs(process.argv.slice(2))
428
+ if (args.help) return void console.log(HELP)
429
+ if (!args.source) throw new Error('missing --source')
430
+ if (!args.target && !args.name) throw new Error('provide --target <uuid>, or --name to create a new target org')
431
+ if (!args.target && !args.ownerUser) throw new Error('creating a new target org requires --owner-user')
432
+ if (args.targetNamespace === 'dooer-production' && args.execute && !args.confirmProduction)
433
+ throw new Error('refusing to WRITE to dooer-production without --confirm-production')
434
+
435
+ const map = require(args.mapPath)
436
+ const copyTables = map.tables.filter((t) => t.action === 'copy')
437
+ const mode = args.execute ? 'EXECUTE' : 'DRY-RUN (dummy — no writes)'
438
+
439
+ console.log(`\n=== seed-test-account · ${mode} ===`)
440
+ console.log(`source: ${args.source} @ ${args.sourceNamespace}`)
441
+ console.log(`target: ${args.target || `(create "${args.name}", owner ${args.ownerUser})`} @ ${args.targetNamespace}`)
442
+ console.log(
443
+ `email scrub: ${args.emailScrub} · copy tables: ${copyTables.length} · if-target-nonempty: ${args.ifTargetNonempty}\n`
444
+ )
445
+
446
+ // Always separate connections: pass 2 streams each table with a server-side cursor on `src` while
447
+ // COPYing into `tgt`, which cannot share one connection (even for a same-namespace copy).
448
+ const src = await connect(args.sourceNamespace)
449
+ const tgt = await connect(args.targetNamespace)
450
+
451
+ try {
452
+ // resolve / create the target org id
453
+ let targetOrg = args.target
454
+ let createdOrg = false
455
+ if (!targetOrg) {
456
+ targetOrg = newUuid()
457
+ createdOrg = true
458
+ // the owner user must already exist in the TARGET db (we never create users)
459
+ const u = await tgt.query(`SELECT 1 FROM service_accounts.users WHERE users_pk=$1`, [args.ownerUser])
460
+ if (!u.rowCount)
461
+ throw new Error(`--owner-user ${args.ownerUser} not found in service_accounts.users on ${args.targetNamespace}`)
462
+ console.log(
463
+ `${args.execute ? 'will create' : 'would create'} target org ${targetOrg} (name "${args.name}", owner ${
464
+ args.ownerUser
465
+ })`
466
+ )
467
+ } else {
468
+ const exists = await tgt.query(`SELECT 1 FROM service_accounts.companies WHERE companies_pk=$1`, [targetOrg])
469
+ if (!exists.rowCount)
470
+ throw new Error(
471
+ `target org ${targetOrg} has no service_accounts.companies row (create it first, or use --name)`
472
+ )
473
+ }
474
+
475
+ // Fix the remap salt from (source, target, --salt): re-running the same copy now derives IDENTICAL
476
+ // target ids, so a run interrupted by a dropped VPN can be resumed (re-run with the SAME --target)
477
+ // and `--only`/`--skip-tables` can copy a subset while cross-refs still remap consistently.
478
+ setRemapSalt(args.source, targetOrg, args.salt)
479
+
480
+ // The uuid remap keeps only the SET of copied source uuids (not a huge old→new map); the new value
481
+ // is derived on demand. The source org id maps to the (existing/new) target org id. `explicitRemap`
482
+ // pins specific old→new ids that are NOT derived — used for referenced users that already exist in
483
+ // the target under a different pk (matched by natural key), so refs point at the existing user.
484
+ const sourceUuids = new Set()
485
+ const explicitRemap = new Map()
486
+ const remapUuid = (v) =>
487
+ v === args.source
488
+ ? targetOrg
489
+ : explicitRemap.has(v)
490
+ ? explicitRemap.get(v)
491
+ : sourceUuids.has(v)
492
+ ? deriveUuid(v)
493
+ : v
494
+ const columnRemap = new Map() // "schema.table.column" -> Map(oldValueString -> newValue)
495
+ const uuidColsByTable = uuidRemapCols(map) // "schema.table" -> [uuid cols], from config
496
+
497
+ // PASS 1 — collect source PKs, allocate new ids (one query per table; progress to stderr)
498
+ const plan = []
499
+ let idx = 0
500
+ for (const t of copyTables) {
501
+ idx++
502
+ process.stderr.write(`\r[pass1 ${idx}/${copyTables.length}] ${t.key.slice(0, 52).padEnd(52)}`)
503
+ const okSrc = await tableExists(src, t.schema, t.table)
504
+ const okTgt = await tableExists(tgt, t.schema, t.table)
505
+ if (!okSrc || !okTgt) {
506
+ plan.push({ t, skipped: `missing in ${!okSrc ? 'source' : 'target'}`, count: 0 })
507
+ continue
508
+ }
509
+ const filters = filterClause(t, args.source)
510
+ if (!filters) {
511
+ plan.push({ t, skipped: 'no filter column', count: 0 })
512
+ continue
513
+ }
514
+ // cache the source columns (reused in pass 2); validate the config's uuid-remap columns exist here
515
+ const srcCols = await columnsOf(src, t.schema, t.table)
516
+ const srcColNames = new Set(srcCols.map((c) => c.name))
517
+ const freshen = (uuidColsByTable.get(t.key) || []).filter((c) => srcColNames.has(c))
518
+ let count
519
+ if (freshen.length) {
520
+ // one query fetches count + every unique-uuid value to add to the source-uuid set
521
+ const sel = freshen.map((c) => qIdent(c)).join(',')
522
+ const rows = await src.query(
523
+ `SELECT ${sel} FROM ${qTable(t.schema, t.table)} WHERE ${filters.sql}`,
524
+ filters.params
525
+ )
526
+ count = rows.rowCount
527
+ for (const row of rows.rows) for (const c of freshen) if (row[c]) sourceUuids.add(row[c])
528
+ } else {
529
+ const { rows } = await src.query(
530
+ `SELECT count(*)::int AS c FROM ${qTable(t.schema, t.table)} WHERE ${filters.sql}`,
531
+ filters.params
532
+ )
533
+ count = rows[0].c
534
+ }
535
+ plan.push({ t, count, srcCols })
536
+ }
537
+ process.stderr.write('\n')
538
+
539
+ const totalRows = plan.reduce((n, p) => n + (p.skipped ? 0 : p.count), 0)
540
+ console.log('--- plan (tables with rows) ---')
541
+ for (const p of plan.filter((p) => p.count > 0 || p.skipped)) {
542
+ if (p.skipped) console.log(` SKIP ${p.t.key.padEnd(48)} (${p.skipped})`)
543
+ else
544
+ console.log(
545
+ ` copy ${p.t.key.padEnd(48)} ${String(p.count).padStart(8)} rows${
546
+ p.t.emailColumns.length ? ` [scrub ${p.t.emailColumns.join(',')}]` : ''
547
+ }${p.t.special ? ' [special]' : ''}`
548
+ )
549
+ }
550
+ console.log(`\ntotal rows to copy: ${totalRows} · distinct uuid values remapped: ${sourceUuids.size}`)
551
+
552
+ // PASS 1b — referenced users. Copied rows point at users (createdByUserId, fk_users_pk, …) that may
553
+ // not exist in the target namespace (cross-namespace copy) → resolvers like `createdByUser` dangle.
554
+ // Collect the ids in the configured user-ref columns, look each up in source (role) and target
555
+ // (existence), and copy ONLY the ones missing from the target — with a new id (added to the remap
556
+ // set so refs follow) and emails anonymized for `customer`-role users. Users already in the target
557
+ // are left untouched and their refs keep the original id.
558
+ const usersToCopy = [] // [{ oldId, role, anonymize }]
559
+ if (map.userTable && !args.skipUsers) {
560
+ const ut = map.userTable
561
+ const candidate = new Set()
562
+ for (const rc of map.userRefColumns || []) {
563
+ const p = plan.find((pp) => pp.t.key === `${rc.schema}.${rc.table}` && !pp.skipped && pp.count > 0)
564
+ if (!p || !(p.srcCols || []).some((c) => c.name === rc.column)) continue
565
+ const f = filterClause(p.t, args.source)
566
+ const { rows } = await withRetry(
567
+ () =>
568
+ src.query(
569
+ `SELECT DISTINCT ${qIdent(rc.column)} AS v FROM ${qTable(rc.schema, rc.table)} WHERE ${
570
+ f.sql
571
+ } AND ${qIdent(rc.column)} IS NOT NULL`,
572
+ f.params
573
+ ),
574
+ { label: `userref ${rc.table}.${rc.column}` }
575
+ )
576
+ for (const r of rows) if (r.v) candidate.add(r.v)
577
+ }
578
+ const ids = [...candidate]
579
+ let matched = 0
580
+ if (ids.length) {
581
+ const srcU = (
582
+ await src.query(
583
+ `SELECT ${qIdent(ut.pk)} AS id, ${qIdent(ut.roleColumn)} AS role, email, ${qIdent(
584
+ 'id_number'
585
+ )} AS id_number FROM ${qTable(ut.schema, ut.table)} WHERE ${qIdent(ut.pk)} = ANY($1)`,
586
+ [ids]
587
+ )
588
+ ).rows
589
+ // (1) exact pk already in target → ref stays valid, do nothing
590
+ const tgtPk = new Set(
591
+ (
592
+ await tgt.query(
593
+ `SELECT ${qIdent(ut.pk)} AS id FROM ${qTable(ut.schema, ut.table)} WHERE ${qIdent(ut.pk)} = ANY($1)`,
594
+ [ids]
595
+ )
596
+ ).rows.map((r) => r.id)
597
+ )
598
+ const unresolved = srcU.filter((u) => !tgtPk.has(u.id))
599
+ // (2) non-customer users often exist in the target under a DIFFERENT pk (same person across
600
+ // namespaces). Match by the natural unique key (email|id_number, role) and pin the ref to the
601
+ // EXISTING target pk (never a duplicate → never a (email,role)/(id_number,role) unique clash).
602
+ const nonCust = unresolved.filter((u) => !(ut.anonymizeRoles || []).includes(u.role))
603
+ const emails = [...new Set(nonCust.map((u) => u.email).filter(Boolean))]
604
+ const idnums = [...new Set(nonCust.map((u) => u.id_number).filter(Boolean))]
605
+ const byEmail = new Map()
606
+ const byId = new Map()
607
+ if (emails.length || idnums.length) {
608
+ const tm = (
609
+ await tgt.query(
610
+ `SELECT ${qIdent(ut.pk)} AS id, ${qIdent(ut.roleColumn)} AS role, email, ${qIdent(
611
+ 'id_number'
612
+ )} AS id_number FROM ${qTable(ut.schema, ut.table)} WHERE email = ANY($1) OR ${qIdent(
613
+ 'id_number'
614
+ )} = ANY($2)`,
615
+ [emails, idnums]
616
+ )
617
+ ).rows
618
+ for (const m of tm) {
619
+ if (m.email) byEmail.set(`${m.role}|${m.email}`, m.id)
620
+ if (m.id_number) byId.set(`${m.role}|${m.id_number}`, m.id)
621
+ }
622
+ }
623
+ for (const u of unresolved) {
624
+ const anonymize = (ut.anonymizeRoles || []).includes(u.role)
625
+ const hit =
626
+ !anonymize &&
627
+ ((u.email && byEmail.get(`${u.role}|${u.email}`)) || (u.id_number && byId.get(`${u.role}|${u.id_number}`)))
628
+ if (hit) {
629
+ explicitRemap.set(u.id, hit) // ref → existing target user (no copy)
630
+ matched++
631
+ continue
632
+ }
633
+ sourceUuids.add(u.id) // will be copied with a new derived id → refs to it now remap
634
+ usersToCopy.push({ oldId: u.id, role: u.role, anonymize })
635
+ }
636
+ }
637
+ const anon = usersToCopy.filter((u) => u.anonymize).length
638
+ console.log(
639
+ `referenced users: ${candidate.size} distinct · matched to existing target users: ${matched} · copied new: ${
640
+ usersToCopy.length
641
+ } (customer-anonymized: ${anon}, as-is: ${usersToCopy.length - anon})`
642
+ )
643
+ }
644
+
645
+ // --only / --skip-tables filter the WRITE phase only (pass-1 already scanned EVERY table, so the uuid
646
+ // remap set is complete and cross-references into skipped tables still resolve). Lets a big org be
647
+ // copied in survivable chunks over a flaky link: re-run with the same --target (→ same salt) and a
648
+ // different --only, using --if-target-nonempty insert. Keep FK-linked int-key tables together —
649
+ // transactions + its 5 twins share a per-run sequence remap (see GUIDE.md).
650
+ const willCopy = (key) => (!args.only || args.only.includes(key)) && !(args.skipTables || []).includes(key)
651
+
652
+ // target-nonempty guard (only over the tables this run will actually write)
653
+ if (args.ifTargetNonempty === 'refuse') {
654
+ const nonEmpty = []
655
+ for (const p of plan) {
656
+ if (p.skipped || p.count === 0 || !willCopy(p.t.key)) continue
657
+ const f = filterClause(p.t, targetOrg)
658
+ const r = await tgt.query(
659
+ `SELECT count(*)::int AS c FROM ${qTable(p.t.schema, p.t.table)} WHERE ${f.sql}`,
660
+ f.params
661
+ )
662
+ if (r.rows[0].c > 0) nonEmpty.push(`${p.t.key} (${r.rows[0].c})`)
663
+ }
664
+ if (nonEmpty.length) {
665
+ console.log(
666
+ `\nTARGET NOT EMPTY in ${nonEmpty.length} copy table(s): ${nonEmpty.slice(0, 10).join(', ')}${
667
+ nonEmpty.length > 10 ? ' …' : ''
668
+ }`
669
+ )
670
+ if (args.execute)
671
+ throw new Error(
672
+ 'refusing: target already has rows in copy tables (pass --if-target-nonempty insert to override)'
673
+ )
674
+ }
675
+ }
676
+
677
+ if (!args.execute) {
678
+ console.log('\nDRY-RUN complete — nothing was written. Re-run with --execute to perform the copy.\n')
679
+ return
680
+ }
681
+
682
+ // PASS 2 — write, inside one transaction, FK/triggers disabled for the load. Transactional statements
683
+ // use the RAW client (tgt.client): a mid-transaction drop must fail this run (→ rollback), never
684
+ // silently reconnect onto a fresh connection with no open transaction. (Resumable mode, which commits
685
+ // per table and can survive drops, is a separate path — see --resumable.)
686
+ await tgt.client.query('BEGIN')
687
+ await tgt.client.query("SET session_replication_role = 'replica'") // disable FK + user triggers for the bulk load
688
+ let written = 0
689
+
690
+ // --name: create the target org (clone the source companies row + link the owner) before copying
691
+ if (createdOrg) {
692
+ await createTargetOrg(src, tgt, args, targetOrg)
693
+ written += 2
694
+ }
695
+
696
+ // build a per-column old->new map for every non-uuid unique column, per its CONFIG strategy, and
697
+ // register it under the owning column AND every FK column that references it (so refs stay consistent).
698
+ for (const s of nonUuidStrategies(map)) {
699
+ const pe = plan.find((p) => p.t.key === `${s.schema}.${s.table}` && !p.skipped && p.count > 0)
700
+ if (!pe) continue
701
+ const f = filterClause(pe.t, args.source)
702
+ const srcRows = await src.query(
703
+ `SELECT ${qIdent(s.column)} AS v FROM ${qTable(s.schema, s.table)} WHERE ${f.sql} ORDER BY 1`,
704
+ f.params
705
+ )
706
+ const m = new Map()
707
+ if (s.strategy === 'sequence') {
708
+ // reserve all the fresh ids in ONE round-trip (was one nextval() per row)
709
+ const nonNull = srcRows.rows.filter((r) => r.v != null)
710
+ if (nonNull.length) {
711
+ const nv = await tgt.client.query(`SELECT nextval($1)::bigint AS v FROM generate_series(1, $2)`, [
712
+ s.sequenceName,
713
+ nonNull.length,
714
+ ])
715
+ nonNull.forEach((r, i) => m.set(String(r.v), Number(nv.rows[i].v)))
716
+ }
717
+ } else if (s.strategy === 'max-offset') {
718
+ const mx = await tgt.client.query(
719
+ `SELECT COALESCE(max(${qIdent(s.column)}), 0)::bigint AS m FROM ${qTable(s.schema, s.table)}`
720
+ )
721
+ let base = Number(mx.rows[0].m)
722
+ for (const r of srcRows.rows) if (r.v != null) m.set(String(r.v), ++base)
723
+ } else if (s.strategy === 'text-suffix') {
724
+ // deterministic suffix (hash of salt+value) so a re-run/chunk reproduces the same value
725
+ for (const r of srcRows.rows)
726
+ if (r.v != null)
727
+ m.set(
728
+ String(r.v),
729
+ `${r.v}-seedcopy-${crypto
730
+ .createHash('sha1')
731
+ .update(REMAP_SALT)
732
+ .update(String(r.v))
733
+ .digest('hex')
734
+ .slice(0, 8)}`
735
+ )
736
+ }
737
+ columnRemap.set(`${s.schema}.${s.table}.${s.column}`, m)
738
+ for (const ref of s.references || []) columnRemap.set(`${ref.schema}.${ref.table}.${ref.column}`, m)
739
+ }
740
+
741
+ const remapTextStoredUuids = map.options ? map.options.remapTextStoredUuids !== false : true
742
+
743
+ // referenced users missing from the target (discovered in pass 1b) — copy them before the data rows
744
+ // that reference them so every user ref resolves. New derived id + anonymized customer emails.
745
+ if (usersToCopy.length) {
746
+ const nu = await copyUsers(src, tgt, usersToCopy, map.userTable, {
747
+ remapUuid,
748
+ columnRemap,
749
+ targetOrg,
750
+ remapTextStoredUuids,
751
+ emailScrub: args.emailScrub,
752
+ })
753
+ written += nu
754
+ console.log(`copied ${nu} referenced user(s) into ${args.targetNamespace} (missing there before).`)
755
+ }
756
+
757
+ const toCopy = plan.filter((p) => !p.skipped && p.count > 0 && willCopy(p.t.key))
758
+ let ci = 0
759
+ for (const p of toCopy) {
760
+ ci++
761
+ process.stderr.write(
762
+ `\r[copy ${ci}/${toCopy.length}] ${p.t.key.slice(0, 46).padEnd(46)} ${String(p.count).padStart(8)} rows`
763
+ )
764
+ written += await copyTable(src, tgt, p, args, { remapUuid, columnRemap, targetOrg, remapTextStoredUuids })
765
+ }
766
+ process.stderr.write('\n')
767
+
768
+ await tgt.client.query("SET session_replication_role = 'origin'")
769
+ await tgt.client.query('COMMIT')
770
+ console.log(`\nEXECUTE complete — ${written} rows written to ${targetOrg} @ ${args.targetNamespace}.`)
771
+ if (createdOrg) console.log(`Created org ${targetOrg} ("${args.name}") with owner ${args.ownerUser} (role Owner).`)
772
+ // copy the actual files (S3) for every copied blob row — on by default (skip with --skip-files)
773
+ if (!args.skipFiles) await copyFiles(src, args, map, remapUuid)
774
+ else
775
+ console.log('(--skip-files: S3 objects NOT copied; copied documents will have broken file links until copied.)')
776
+ } catch (e) {
777
+ if (args.execute) {
778
+ try {
779
+ await tgt.client.query('ROLLBACK')
780
+ } catch (_) {}
781
+ }
782
+ throw e
783
+ } finally {
784
+ await src.end()
785
+ await tgt.end()
786
+ }
787
+ }
788
+
789
+ // build the `<col> IN (SELECT <parentCol> FROM <parent> WHERE …)` clause for an orphan's fkFilter.
790
+ // Terminal hop: the parent has an org column (`parentOrgColumn=$1`). Multi-level: the parent is itself
791
+ // an orphan (`parentFilter`) → recurse so the whole chain roots at an org-scoped ancestor.
792
+ function fkFilterSql(f) {
793
+ const inner = f.parentOrgColumn ? `${qIdent(f.parentOrgColumn)}=$1` : fkFilterSql(f.parentFilter)
794
+ return `${qIdent(f.column)} IN (SELECT ${qIdent(f.parentColumn)} FROM ${qTable(
795
+ f.parentSchema,
796
+ f.parentTable
797
+ )} WHERE ${inner})`
798
+ }
799
+
800
+ // build a WHERE clause selecting the org's own rows (+ an optional per-table extraFilter, which may
801
+ // reference $1 = the org id — e.g. restrict a big table to rows tied to an active task)
802
+ function filterClause(t, orgId, { includeExtra = true } = {}) {
803
+ let sql
804
+ if (t.special && t.special.some((s) => s.startsWith('crossOrgRelation'))) {
805
+ sql = `(parent_company_id=$1 OR child_company_id=$1)`
806
+ } else if (t.fkFilter) {
807
+ // orphan child table (no org column): select rows whose FK points into a copied parent's org rows.
808
+ // A multi-level orphan (child of an orphan, e.g. document_files_converted → document_files →
809
+ // documents) carries a NESTED parentFilter instead of a terminal parentOrgColumn — recurse.
810
+ sql = fkFilterSql(t.fkFilter)
811
+ } else if (!t.filterColumn) {
812
+ return null
813
+ } else {
814
+ sql = `${qIdent(t.filterColumn)}=$1`
815
+ }
816
+ // extraFilter (e.g. the active-task restriction) narrows a COPY; a PURGE must remove ALL the org's rows,
817
+ // so callers deleting data pass includeExtra:false.
818
+ if (t.extraFilter && includeExtra) sql = `${sql} AND (${t.extraFilter})`
819
+ return { sql, params: [orgId] }
820
+ }
821
+
822
+ // --name path: create the target org by cloning the source companies row (fresh id / name / short_name
823
+ // / api_uuid) and linking the given owner user with role 'Owner'. Never creates the user itself.
824
+ async function createTargetOrg(src, tgt, args, targetOrg) {
825
+ const cols = (await columnsOf(src, 'service_accounts', 'companies')).filter((c) => !c.generated)
826
+ const row = (
827
+ await src.query(
828
+ `SELECT ${cols.map((c) => qIdent(c.name)).join(',')} FROM service_accounts.companies WHERE companies_pk=$1`,
829
+ [args.source]
830
+ )
831
+ ).rows[0]
832
+ if (!row) throw new Error(`source companies row ${args.source} not found`)
833
+ const slug =
834
+ args.name
835
+ .toLowerCase()
836
+ .replace(/[^a-z0-9]+/g, '-')
837
+ .replace(/^-|-$/g, '')
838
+ .slice(0, 40) || 'org'
839
+ const overrides = {
840
+ companies_pk: targetOrg,
841
+ company_name: args.name,
842
+ short_name: `${slug}-${newUuid().slice(0, 8)}`, // short_name is UNIQUE
843
+ api_uuid: newUuid(),
844
+ }
845
+ const vals = cols.map((c) => {
846
+ if (Object.prototype.hasOwnProperty.call(overrides, c.name)) return overrides[c.name]
847
+ let v = row[c.name]
848
+ if ((c.type === 'jsonb' || c.type === 'json') && v != null) v = JSON.stringify(v)
849
+ return v
850
+ })
851
+ await tgt.client.query(
852
+ `INSERT INTO service_accounts.companies (${cols.map((c) => qIdent(c.name)).join(',')}) VALUES (${cols
853
+ .map((_, i) => `$${i + 1}`)
854
+ .join(',')})`,
855
+ vals
856
+ )
857
+ await tgt.client.query(
858
+ `INSERT INTO service_accounts.companies2users (fk_companies_pk, fk_users_pk, fk_user_roles_at_companies_pk, api_uuid) VALUES ($1,$2,'Owner',$3)`,
859
+ [targetOrg, args.ownerUser, newUuid()]
860
+ )
861
+ }
862
+
863
+ // ── COPY (text format) encoding — much faster than row-by-row INSERT ─────────
864
+ // escape a field for COPY text format: backslash, tab, newline, CR
865
+ const copyEscape = (s) => s.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/\r/g, '\\r').replace(/\t/g, '\\t')
866
+ // Postgres array literal for a JS array (quote every element; NULL for null)
867
+ function arrayLiteral(arr) {
868
+ return (
869
+ '{' +
870
+ arr
871
+ .map((el) =>
872
+ el === null || el === undefined ? 'NULL' : '"' + String(el).replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '"'
873
+ )
874
+ .join(',') +
875
+ '}'
876
+ )
877
+ }
878
+ // encode one already-remapped value for its column's type into a COPY field
879
+ function encodeCopyField(col, v) {
880
+ if (v === null || v === undefined) return '\\N'
881
+ if (col.type === 'ARRAY') return copyEscape(arrayLiteral(Array.isArray(v) ? v : [v]))
882
+ if (col.type === 'boolean') return v ? 't' : 'f'
883
+ if (col.type === 'jsonb' || col.type === 'json') return copyEscape(typeof v === 'string' ? v : JSON.stringify(v))
884
+ return copyEscape(String(v))
885
+ }
886
+
887
+ // copy one table's source rows into the target via COPY … FROM STDIN. Reads the source with a
888
+ // server-side CURSOR in batches (constant memory regardless of table size) and streams each batch
889
+ // into the COPY with backpressure. Requires src and tgt to be separate connections.
890
+ const READ_BATCH = 2000
891
+ async function copyTable(src, tgt, p, args, ctx) {
892
+ const { remapUuid, columnRemap, targetOrg, remapTextStoredUuids } = ctx
893
+ const t = p.t
894
+ const srcCols = p.srcCols || (await columnsOf(src, t.schema, t.table))
895
+ const tgtCols = await columnsOf(tgt, t.schema, t.table)
896
+ const tgtNames = new Set(tgtCols.map((c) => c.name))
897
+ // insertable columns = present in both, not generated
898
+ const cols = srcCols.filter((c) => tgtNames.has(c.name) && !c.generated)
899
+ const f = filterClause(t, args.source)
900
+ const colList = cols.map((c) => qIdent(c.name)).join(',')
901
+ const rctx = {
902
+ remapUuid,
903
+ columnRemap,
904
+ targetOrg,
905
+ emailSet: new Set(t.emailColumns || []),
906
+ emailScrub: args.emailScrub,
907
+ table: t,
908
+ remapTextStoredUuids,
909
+ }
910
+
911
+ const cursor = src.client.query(
912
+ new Cursor(`SELECT ${colList} FROM ${qTable(t.schema, t.table)} WHERE ${f.sql}`, f.params)
913
+ )
914
+ const readBatch = () =>
915
+ new Promise((resolve, reject) => cursor.read(READ_BATCH, (err, rows) => (err ? reject(err) : resolve(rows))))
916
+ const closeCursor = () => new Promise((resolve) => cursor.close(() => resolve()))
917
+
918
+ let n = 0
919
+ // async generator pulls a batch at a time from the cursor (constant memory) and yields COPY lines;
920
+ // pipeline() wires it to the COPY stream with backpressure AND one clean error path (any COPY/read
921
+ // error rejects here and is caught by the caller → transaction rollback).
922
+ async function* lines() {
923
+ for (let batch = await readBatch(); batch.length; batch = await readBatch()) {
924
+ for (const row of batch) {
925
+ yield cols.map((c) => encodeCopyField(c, remapValue(c, row[c.name], rctx))).join('\t') + '\n'
926
+ }
927
+ n += batch.length
928
+ }
929
+ }
930
+ try {
931
+ await pipeline(
932
+ Readable.from(lines()),
933
+ tgt.client.query(copyFrom(`COPY ${qTable(t.schema, t.table)} (${colList}) FROM STDIN`))
934
+ )
935
+ } finally {
936
+ await closeCursor()
937
+ }
938
+ return n
939
+ }
940
+
941
+ // copy specific referenced-user rows into the target (the ones missing there). Each gets a new derived
942
+ // id (already added to sourceUuids, so refs follow); `customer`-role rows get a UNIQUE anonymized email
943
+ // (per the `(email, role)` unique key) and their PII/credential columns (id_number, password_hash)
944
+ // nulled. Every other column goes through the normal value remap. Small set → one plain SELECT, no cursor.
945
+ async function copyUsers(src, tgt, users, ut, ctx) {
946
+ const { remapUuid } = ctx
947
+ const oldIds = users.map((u) => u.oldId)
948
+ const meta = new Map(users.map((u) => [u.oldId, u]))
949
+ const srcCols = (await columnsOf(src, ut.schema, ut.table)).filter((c) => !c.generated)
950
+ const tgtNames = new Set((await columnsOf(tgt, ut.schema, ut.table)).map((c) => c.name))
951
+ const cols = srcCols.filter((c) => tgtNames.has(c.name))
952
+ const colList = cols.map((c) => qIdent(c.name)).join(',')
953
+ const emailSet = new Set(ut.emailColumns || [])
954
+ const nullSet = new Set(ut.nullColumns || [])
955
+ const pattern = ut.anonymizedEmailPattern || 'testcustomer+u<id>@dooer.com'
956
+ // neutralise the generic email scrub — user emails are handled explicitly (unique per row) below
957
+ const rctx = { ...ctx, emailSet: new Set(), table: { key: `${ut.schema}.${ut.table}`, emailColumns: [] } }
958
+ const { rows } = await src.query(
959
+ `SELECT ${colList} FROM ${qTable(ut.schema, ut.table)} WHERE ${qIdent(ut.pk)} = ANY($1)`,
960
+ [oldIds]
961
+ )
962
+ function* lines() {
963
+ for (const row of rows) {
964
+ const m = meta.get(row[ut.pk])
965
+ const newId = remapUuid(row[ut.pk])
966
+ yield cols
967
+ .map((c) => {
968
+ if (m.anonymize && emailSet.has(c.name))
969
+ return encodeCopyField(c, row[c.name] == null ? null : pattern.replace('<id>', newId.replace(/-/g, '')))
970
+ if (m.anonymize && nullSet.has(c.name)) return encodeCopyField(c, null)
971
+ return encodeCopyField(c, remapValue(c, row[c.name], rctx))
972
+ })
973
+ .join('\t') + '\n'
974
+ }
975
+ }
976
+ await pipeline(
977
+ Readable.from(lines()),
978
+ tgt.client.query(copyFrom(`COPY ${qTable(ut.schema, ut.table)} (${colList}) FROM STDIN`))
979
+ )
980
+ return rows.length
981
+ }
982
+
983
+ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
984
+ const looksLikeUuid = (s) => s.length === 36 && UUID_RE.test(s)
985
+ // global: every uuid-shaped substring (for remapping uuids inside raw jsonb text)
986
+ const UUID_SUBSTR_RE_G = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
987
+
988
+ // remap a single column value: json, email scrub, config-driven per-column remap, uuid, uuid[]
989
+ function remapValue(col, value, ctx) {
990
+ const { remapUuid, columnRemap, emailSet, emailScrub, table, remapTextStoredUuids } = ctx
991
+ if (value === null || value === undefined) return value
992
+ // json/jsonb arrives as RAW TEXT (see type parsers). Remap uuids directly in the text: replace every
993
+ // uuid-shaped substring that belongs to the copy with its derived uuid. A uuid → uuid replacement is
994
+ // same-length + valid chars, so the JSON stays valid. This avoids JSON.parse+walk+stringify — the
995
+ // per-row cost that made big taskAction/ocrData/documentField payloads slow and OOM the process.
996
+ if (col.type === 'jsonb' || col.type === 'json') {
997
+ const s = typeof value === 'string' ? value : JSON.stringify(value)
998
+ return s.replace(UUID_SUBSTR_RE_G, (m) => remapUuid(m))
999
+ }
1000
+ // email scrub (type-aware): text -> scalar, uuid never, boolean/other skipped
1001
+ if (emailSet.has(col.name)) {
1002
+ if (col.type === 'ARRAY' && col.udt === '_text') return [emailScrub]
1003
+ if (col.type === 'text' || col.type === 'character varying') return emailScrub
1004
+ // non-string email-named column (e.g. boolean flag) — leave as-is
1005
+ }
1006
+ // config-driven per-column remap (non-uuid unique columns: sequence / max-offset / text-suffix, plus
1007
+ // every FK column that references them — all registered in columnRemap by "schema.table.column")
1008
+ const cm = columnRemap.get(`${table.key}.${col.name}`)
1009
+ if (cm) {
1010
+ const mapped = cm.get(String(value))
1011
+ if (mapped !== undefined) return mapped
1012
+ }
1013
+ // uuid columns: rewrite if part of the copy (PKs, org id, internal FKs); leave external refs untouched
1014
+ if (isUuid(col)) return remapUuid(value)
1015
+ if (isUuidArray(col) && Array.isArray(value)) return value.map((v) => (v ? remapUuid(v) : v))
1016
+ // a text/varchar value equal to a copied uuid is a reference (polymorphic id stored as text) → remap
1017
+ // (only for uuid-shaped values, so we don't hash every ordinary text field)
1018
+ if (remapTextStoredUuids && (col.type === 'text' || col.type === 'character varying') && looksLikeUuid(value))
1019
+ return remapUuid(value)
1020
+ return value
1021
+ }
1022
+
1023
+ // helpers reused by sibling commands (purge, shred, db build)
1024
+ module.exports = {
1025
+ main,
1026
+ parseArgs,
1027
+ HELP,
1028
+ connect,
1029
+ withRetry,
1030
+ filterClause,
1031
+ qIdent,
1032
+ qTable,
1033
+ columnsOf,
1034
+ s3Config,
1035
+ s3Get,
1036
+ DeleteObjectCommand,
1037
+ }
1038
+
1039
+ // still runnable standalone (node lib/engine/seed.js …) for now
1040
+ if (require.main === module) {
1041
+ main().catch((e) => {
1042
+ console.error(`\nERROR: ${e.message}\n`)
1043
+ process.exit(1)
1044
+ })
1045
+ }