@batadata/cli 0.1.17 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1316 @@
1
+ /**
2
+ * bata import — Neon (or any Postgres) → BataDB migration.
3
+ *
4
+ * bata import --source <postgres-uri> [--project <id> | --name <name>] [--pg <major>] [--yes] [--json]
5
+ *
6
+ * Agent-native: shells out to the standard `pg_dump` / `psql` client tools
7
+ * (zero runtime deps) and streams a dump straight into a fresh BataDB project.
8
+ *
9
+ * BataDB runs PostgreSQL 16 today; Neon defaults to 17. A PG17 → PG16
10
+ * dump/restore is clean for standard schemas EXCEPT that pg_dump 17 emits
11
+ * `SET transaction_timeout = 0;` — a GUC PG16 doesn't know — into the dump
12
+ * preamble. We strip that one line (and any future target-unknown GUC SETs)
13
+ * out of the stream. Genuinely PG17-only schema features still fail loudly at
14
+ * restore time, and we surface those errors honestly rather than pretending
15
+ * the migration succeeded.
16
+ */
17
+ import { spawn, spawnSync, execSync } from "node:child_process";
18
+ import { URL } from "node:url";
19
+ import { api, apiError, resolveTeamId, asList } from "../api.js";
20
+ import { requireToken, loadConfig, isJsonMode, isYes } from "../config.js";
21
+ import { colors, log, json, success, warn, heading, kvList, table } from "../utils/logger.js";
22
+ import { emitError } from "../utils/errors.js";
23
+ // ── Pure, unit-tested helpers ────────────────────────────────────────────────
24
+ /**
25
+ * GUC SET statements pg_dump emits that a BataDB (PG16) target rejects. Kept as
26
+ * an extensible list so a future target-unknown GUC is a one-line addition.
27
+ * `transaction_timeout` is new in PG17 and is the only one seen in practice.
28
+ */
29
+ export const TARGET_UNKNOWN_GUCS = ["transaction_timeout"];
30
+ /**
31
+ * A streaming line filter that drops target-unknown GUC `SET` statements from a
32
+ * pg_dump stream. It buffers a partial trailing line across chunk boundaries so
33
+ * a `SET transaction_timeout = 0;` split across two reads is still recognised
34
+ * and removed. `push()` returns the text to forward to psql for that chunk;
35
+ * `flush()` returns any buffered final line (a dump without a trailing newline).
36
+ *
37
+ * PREAMBLE-ONLY: pg_dump emits the target-unknown GUCs ONLY in the initial
38
+ * connection preamble (the block of `SET …;` / `SELECT pg_catalog.set_config(…)`
39
+ * statements at the very top). Filtering the whole stream is unsafe: a dumped
40
+ * function/procedure body can contain a standalone line `SET transaction_timeout
41
+ * = 0;` inside its dollar-quoted text, and a COPY data row can be literally
42
+ * anything — mutating those would silently corrupt the restore. So we only strip
43
+ * inside the preamble: the first line that is NOT one of {empty, a `--` comment,
44
+ * `SET …;`, `SELECT pg_catalog.set_config(…);`} permanently ends the preamble,
45
+ * after which every line passes through verbatim. A GUC-looking SET after the
46
+ * preamble is therefore NOT stripped — which is correct: transaction_timeout only
47
+ * ever appears in the preamble; were a future pg_dump to emit it mid-dump, the
48
+ * restore would fail loudly rather than be silently altered.
49
+ */
50
+ export function makeGucLineFilter(gucs = TARGET_UNKNOWN_GUCS) {
51
+ // `^SET <guc> …;` — a standalone GUC SET statement. Anchored to the start of
52
+ // the line and requiring the trailing semicolon so it can't match a data row
53
+ // that merely begins with the same text.
54
+ const patterns = gucs.map((g) => new RegExp(`^SET\\s+${g}\\b[^\\n]*;\\s*$`, "i"));
55
+ const isStripped = (line) => patterns.some((re) => re.test(line));
56
+ // Lines that legitimately appear in the preamble; the first line that matches
57
+ // NONE of these ends the preamble for good. `\restrict <token>` is a psql
58
+ // meta-command pg_dump 17.x emits at the top (paired with `\unrestrict` at the
59
+ // very end) — it sits BEFORE the GUC SETs, so it must be recognised or the
60
+ // preamble would end early and the GUC we strip would sail through untouched.
61
+ const PREAMBLE_LINE = [
62
+ /^\s*$/, // blank
63
+ /^--/, // SQL comment
64
+ /^\\(un)?restrict\b/i, // psql \restrict / \unrestrict (PG17.x)
65
+ /^SET\s.+;\s*$/i, // any GUC SET (incl. the ones we strip)
66
+ /^SELECT\s+pg_catalog\.set_config\(.*\);\s*$/i, // set_config form
67
+ ];
68
+ const isPreambleLine = (line) => PREAMBLE_LINE.some((re) => re.test(line));
69
+ let preambleDone = false;
70
+ let pending = "";
71
+ // Whether a preamble line survives (updating the preambleDone latch). Only
72
+ // called while still in the preamble.
73
+ const keepPreambleLine = (raw) => {
74
+ const line = raw.replace(/\r$/, "");
75
+ if (!isPreambleLine(line)) {
76
+ preambleDone = true; // first non-preamble line ends filtering permanently
77
+ return true;
78
+ }
79
+ return !isStripped(line);
80
+ };
81
+ return {
82
+ push(chunk) {
83
+ // Once out of the preamble, bypass line-splitting entirely and stream the
84
+ // chunk through verbatim. Buffering-until-newline here would hold a single
85
+ // huge COPY value (a multi-MB TOAST row on one line) entirely in memory and
86
+ // defeat backpressure — the body is never filtered, so there's no reason to
87
+ // reassemble its lines.
88
+ if (preambleDone)
89
+ return chunk;
90
+ pending += chunk;
91
+ const lines = pending.split("\n");
92
+ // The final element has no terminating newline yet — hold it for next push.
93
+ pending = lines.pop() ?? "";
94
+ let out = "";
95
+ for (let i = 0; i < lines.length; i++) {
96
+ if (keepPreambleLine(lines[i]))
97
+ out += lines[i] + "\n";
98
+ // The moment the preamble ends, everything remaining in this chunk is
99
+ // body — emit it verbatim (with its newlines) and stop line-processing.
100
+ if (preambleDone) {
101
+ const rest = lines.slice(i + 1);
102
+ if (rest.length > 0)
103
+ out += rest.join("\n") + "\n";
104
+ if (pending) {
105
+ out += pending;
106
+ pending = "";
107
+ }
108
+ break;
109
+ }
110
+ }
111
+ return out;
112
+ },
113
+ flush() {
114
+ // After the preamble, push() already emitted everything (pending is empty).
115
+ const last = pending;
116
+ pending = "";
117
+ if (!last)
118
+ return "";
119
+ // A dump that is ALL preamble (no trailing newline on the last line).
120
+ return keepPreambleLine(last) ? last : "";
121
+ },
122
+ };
123
+ }
124
+ /**
125
+ * Extract a PostgreSQL major version from any of the shapes we encounter:
126
+ * - `server_version_num` (e.g. "170002", "160003", "90603")
127
+ * - dotted `server_version` (e.g. "17.2", "16.3 (Ubuntu 16.3-1.pgdg…)")
128
+ * - `pg_dump --version` output ("pg_dump (PostgreSQL) 16.3")
129
+ * Returns the integer major, or null if none can be found.
130
+ */
131
+ export function parsePgMajor(v) {
132
+ const t = (v ?? "").trim();
133
+ // server_version_num form: all digits, wide enough to be an encoded version.
134
+ if (/^\d+$/.test(t) && t.length >= 5) {
135
+ return Math.floor(parseInt(t, 10) / 10000);
136
+ }
137
+ const m = t.match(/(\d+)/);
138
+ return m ? parseInt(m[1], 10) : null;
139
+ }
140
+ /** A client tool must be at least the source major (pg_dump can't dump a newer
141
+ * server than itself; psql can't parse a newer pg_dump's meta-commands). */
142
+ export function toolMajorSatisfies(toolMajor, sourceMajor) {
143
+ return toolMajor >= sourceMajor;
144
+ }
145
+ /** Clear, actionable message when a local client tool is too old for the source. */
146
+ export function toolVersionErrorMessage(tool, toolMajor, sourceMajor) {
147
+ return (`Your ${tool} is version ${toolMajor}, but the source server is PostgreSQL ${sourceMajor}. ` +
148
+ `A client older than the source can't reliably migrate it (e.g. psql ${sourceMajor - 1} can't parse ` +
149
+ `pg_dump ${sourceMajor}'s \\restrict meta-commands). Install PostgreSQL ${sourceMajor} (or newer) ` +
150
+ `client tools and re-run.`);
151
+ }
152
+ /** Message when psql can't restore the stream THIS pg_dump produces. */
153
+ export function psqlVsDumpErrorMessage(psqlMajor, dumpMajor) {
154
+ return (`Your psql is version ${psqlMajor}, but pg_dump is version ${dumpMajor} — psql can't reliably ` +
155
+ `restore a stream produced by a newer pg_dump (pg_dump ${dumpMajor} emits \\restrict meta-commands ` +
156
+ `psql ${psqlMajor} doesn't understand). Install matching PostgreSQL ${dumpMajor} client tools and re-run.`);
157
+ }
158
+ /**
159
+ * Verdict on whether the local client tools can migrate a source of `sourceMajor`
160
+ * (null when the source version couldn't be parsed). Rules:
161
+ * - pg_dump must be ≥ source (can't dump a newer server than itself)
162
+ * - psql must be ≥ source AND ≥ pg_dump — the restore stream is produced by the
163
+ * LOCAL pg_dump, and pg_dump 17 emits \restrict/\unrestrict regardless of the
164
+ * source version, so psql 16 + pg_dump 17 fails even against a PG16 source.
165
+ * Effectively psql must satisfy max(sourceMajor, dumpMajor). Returns the install
166
+ * target so the caller can point the user at the right client package.
167
+ */
168
+ export function checkClientVersions(dumpMajor, psqlMajor, sourceMajor) {
169
+ if (sourceMajor !== null && !toolMajorSatisfies(dumpMajor, sourceMajor)) {
170
+ return { ok: false, message: toolVersionErrorMessage("pg_dump", dumpMajor, sourceMajor), needMajor: sourceMajor };
171
+ }
172
+ if (sourceMajor !== null && !toolMajorSatisfies(psqlMajor, sourceMajor)) {
173
+ return { ok: false, message: toolVersionErrorMessage("psql", psqlMajor, sourceMajor), needMajor: sourceMajor };
174
+ }
175
+ if (!toolMajorSatisfies(psqlMajor, dumpMajor)) {
176
+ return { ok: false, message: psqlVsDumpErrorMessage(psqlMajor, dumpMajor), needMajor: dumpMajor };
177
+ }
178
+ return { ok: true };
179
+ }
180
+ /** Pull the first meaningful line out of a psql connection error for the user. */
181
+ export function sourcePreflightErrorMessage(stderr) {
182
+ const line = (stderr ?? "")
183
+ .split("\n")
184
+ .map((s) => s.trim())
185
+ .find((s) => s.length > 0);
186
+ return line
187
+ ? line.replace(/^psql:\s*(error:\s*)?/i, "")
188
+ : "Could not connect to the source database.";
189
+ }
190
+ /** Default target project name = the source database name, else the URI path. */
191
+ export function defaultProjectName(preflightDb, sourceUri) {
192
+ if (preflightDb && preflightDb.trim())
193
+ return preflightDb.trim();
194
+ try {
195
+ const p = new URL(sourceUri).pathname.replace(/^\//, "");
196
+ if (p)
197
+ return p;
198
+ }
199
+ catch {
200
+ /* not a parseable URL — fall through */
201
+ }
202
+ return "imported-db";
203
+ }
204
+ /**
205
+ * Compare two name→value maps (table row counts, or sequence last_values) and
206
+ * return only the mismatches. A name present on one side but not the other is a
207
+ * mismatch too (rendered as ∅). Deterministically sorted by name.
208
+ */
209
+ export function diffCounts(source, target) {
210
+ const names = new Set([...source.keys(), ...target.keys()]);
211
+ const rows = [];
212
+ for (const name of [...names].sort()) {
213
+ const s = source.get(name) ?? "∅";
214
+ const t = target.get(name) ?? "∅";
215
+ if (s !== t)
216
+ rows.push({ name, source: s, target: t });
217
+ }
218
+ return rows;
219
+ }
220
+ /** Build the `table()` rows for a verify mismatch report (tables + sequences). */
221
+ export function mismatchRows(tableRows, seqRows) {
222
+ return [
223
+ ...tableRows.map((r) => [`table ${r.name}`, r.source, r.target]),
224
+ ...seqRows.map((r) => [`seq ${r.name}`, r.source, r.target]),
225
+ ];
226
+ }
227
+ /** Double-quote a SQL identifier (escaping embedded quotes). */
228
+ export function quoteIdent(id) {
229
+ return `"${id.replace(/"/g, '""')}"`;
230
+ }
231
+ /**
232
+ * Map a child process 'close' `(code, signal)` to the numeric exit used for the
233
+ * migration verdict. A signal death (`code === null`, e.g. OOM-killer or an
234
+ * operator `kill`) is a FAILURE — it must never read as success just because its
235
+ * peer exited cleanly. The one exception: when WE deliberately SIGTERM'd the dump
236
+ * after a restore failure (`intentionalKill`), that path already has a nonzero
237
+ * restoreCode so the overall verdict is false regardless; we normalize to 0 to
238
+ * avoid a redundant, confusing error line.
239
+ */
240
+ export function resolveChildExit(code, signal, opts = {}) {
241
+ if (code !== null)
242
+ return { code };
243
+ if (opts.intentionalKill)
244
+ return { code: 0 };
245
+ return { code: -1, error: `terminated by signal ${signal ?? "(unknown)"}` };
246
+ }
247
+ /**
248
+ * Source extensions the target can't satisfy = source extensions not in the
249
+ * target's available set. `plpgsql` is built into every PostgreSQL and always
250
+ * present, so it's excluded — it can never be a real gap even though it shows up
251
+ * in pg_extension. Used both for the pre-restore availability gate (against
252
+ * `pg_available_extensions`) and the residual post-restore warning (against the
253
+ * target's installed `pg_extension`).
254
+ */
255
+ export function missingExtensions(sourceExts, targetHasExts) {
256
+ const has = new Set(targetHasExts);
257
+ return sourceExts.filter((e) => e !== "plpgsql" && !has.has(e));
258
+ }
259
+ /**
260
+ * Objects BataDB's provisioning seeds into EVERY new project — platform furniture,
261
+ * not user data. Because a freshly created target ALWAYS contains these, they must
262
+ * be excluded from all TARGET-side "unexpected object" accounting: the
263
+ * created-project hard-fail, the existing-target leftover warnings/JSON, and the
264
+ * `--yes` non-empty guard (a target holding ONLY these is effectively empty). They
265
+ * are NEVER excluded on the SOURCE side — a source `public.health_check` (or a
266
+ * source `neon_migration` schema, e.g. a migration FROM another BataDB) is treated
267
+ * as real data and must verify normally.
268
+ *
269
+ * Two flavours:
270
+ * - Named objects: `public.health_check` (+ its sequence) — the data-path
271
+ * health-check row/table seeded on provision.
272
+ * - Whole schemas: `neon_migration` — our Neon-derived compute engine's
273
+ * compute_ctl records applied internal SQL migrations here (table
274
+ * `neon_migration.migration_id`) on EVERY compute startup. Real Neon hides this
275
+ * schema from customer `pg_tables`; our compute exposes it, so ANY object under
276
+ * `neon_migration.*` is platform furniture, never user data.
277
+ */
278
+ export const PLATFORM_SCAFFOLDING_TABLES = ["public.health_check"];
279
+ export const PLATFORM_SCAFFOLDING_SEQUENCES = ["public.health_check_id_seq"];
280
+ /** Schemas whose ENTIRE contents are platform furniture (any `<schema>.*` object). */
281
+ export const PLATFORM_SCAFFOLDING_SCHEMAS = ["neon_migration"];
282
+ let PLATFORM_SCAFFOLDING_NAMES = new Set([
283
+ ...PLATFORM_SCAFFOLDING_TABLES,
284
+ ...PLATFORM_SCAFFOLDING_SEQUENCES,
285
+ ]);
286
+ let PLATFORM_SCAFFOLDING_SCHEMA_PREFIXES = PLATFORM_SCAFFOLDING_SCHEMAS.map((s) => `${s}.`);
287
+ /**
288
+ * Merge the server-published scaffolding list (GET /v1/platform/import-scaffolding)
289
+ * into the baked-in defaults above. The server is the source of truth going
290
+ * forward — if provisioning grows new furniture, an already-installed CLI keeps
291
+ * verifying correctly instead of false-failing on it. Union (never replace): the
292
+ * baked defaults still apply against an older control plane, and a partial or
293
+ * malformed server response can only ever ADD exclusions it explicitly names.
294
+ * The exported const arrays are extended in place so every existing reference
295
+ * (the --yes guard, the created-target check, verify) sees the merged lists.
296
+ */
297
+ export function applyServerScaffolding(server) {
298
+ const merge = (into, from) => {
299
+ if (!Array.isArray(from))
300
+ return;
301
+ for (const v of from) {
302
+ if (typeof v === "string" && v.length > 0 && !into.includes(v))
303
+ into.push(v);
304
+ }
305
+ };
306
+ merge(PLATFORM_SCAFFOLDING_TABLES, server.tables);
307
+ merge(PLATFORM_SCAFFOLDING_SEQUENCES, server.sequences);
308
+ merge(PLATFORM_SCAFFOLDING_SCHEMAS, server.schemas);
309
+ PLATFORM_SCAFFOLDING_NAMES = new Set([
310
+ ...PLATFORM_SCAFFOLDING_TABLES,
311
+ ...PLATFORM_SCAFFOLDING_SEQUENCES,
312
+ ]);
313
+ PLATFORM_SCAFFOLDING_SCHEMA_PREFIXES = PLATFORM_SCAFFOLDING_SCHEMAS.map((s) => `${s}.`);
314
+ }
315
+ /** Is this qualified name one of BataDB's seeded platform-scaffolding objects —
316
+ * either an exact named object, or anything inside a scaffolding schema? */
317
+ export function isPlatformScaffolding(qualifiedName) {
318
+ if (PLATFORM_SCAFFOLDING_NAMES.has(qualifiedName))
319
+ return true;
320
+ return PLATFORM_SCAFFOLDING_SCHEMA_PREFIXES.some((p) => qualifiedName.startsWith(p));
321
+ }
322
+ /** Drop platform-scaffolding names — applied ONLY to TARGET-side object lists,
323
+ * never to source accounting. */
324
+ export function excludeScaffolding(names) {
325
+ return names.filter((n) => !isPlatformScaffolding(n));
326
+ }
327
+ /**
328
+ * Scaffolding names that appear in a SOURCE object list — i.e. a user table that
329
+ * shares a name with BataDB's seeded platform table. On restore that `CREATE
330
+ * TABLE` clashes with the scaffolding already present on the target: with
331
+ * ON_ERROR_STOP the restore fails ("relation already exists" → RESTORE_FAILED),
332
+ * or, if the schemas happen to be compatible, rows append and the row-count verify
333
+ * catches the drift. Either way verify stays honest — the source name is NOT
334
+ * excluded from source accounting; we only warn the user up front.
335
+ */
336
+ export function scaffoldingCollisions(sourceQualifiedNames) {
337
+ return sourceQualifiedNames.filter((n) => isPlatformScaffolding(n));
338
+ }
339
+ /**
340
+ * Qualified names present on the TARGET but absent from the source — used for both
341
+ * tables and sequences. Verification only enumerates SOURCE objects, so on a
342
+ * `--project --yes` import into a non-empty target these are invisible to the
343
+ * count/value checks: they were left untouched and aren't covered by verify.
344
+ * Surfaced honestly (a warning for an existing target; unexpected for a freshly
345
+ * created one). Sorted, deduped by set.
346
+ */
347
+ export function targetOnlyNames(sourceQualified, targetQualified) {
348
+ const source = new Set(sourceQualified);
349
+ return targetQualified.filter((t) => !source.has(t)).sort();
350
+ }
351
+ /** Single-quote a SQL string literal (escaping embedded quotes). */
352
+ function quoteLiteral(s) {
353
+ return `'${s.replace(/'/g, "''")}'`;
354
+ }
355
+ /** Split an array into fixed-size batches (used to bound verify query size). */
356
+ function chunk(items, size) {
357
+ const out = [];
358
+ for (let i = 0; i < items.length; i += size)
359
+ out.push(items.slice(i, i + size));
360
+ return out;
361
+ }
362
+ /**
363
+ * Run a one-shot capture query with `psql -Atc`. `-X` skips ~/.psqlrc so a user's
364
+ * custom formatting / field separators / pager can't silently break the -Atc
365
+ * parsers; `-w` never prompts for a password.
366
+ */
367
+ function runPsql(uri, sql) {
368
+ const res = spawnSync("psql", [uri, "-X", "-w", "-Atc", sql], {
369
+ encoding: "utf8",
370
+ env: { ...process.env, PGCONNECT_TIMEOUT: process.env.PGCONNECT_TIMEOUT ?? "10" },
371
+ maxBuffer: 128 * 1024 * 1024,
372
+ });
373
+ if (res.error)
374
+ return { code: -1, stdout: "", stderr: res.error.message };
375
+ return { code: res.status ?? -1, stdout: res.stdout ?? "", stderr: res.stderr ?? "" };
376
+ }
377
+ /** Is a client tool on PATH? Mirrors `bata connect`'s `which` probe. */
378
+ function hasTool(name) {
379
+ try {
380
+ execSync(`which ${name}`, { stdio: "ignore" });
381
+ return true;
382
+ }
383
+ catch {
384
+ return false;
385
+ }
386
+ }
387
+ /** The local pg_dump major version, or null if it can't be determined. */
388
+ function pgDumpMajor() {
389
+ const res = spawnSync("pg_dump", ["--version"], { encoding: "utf8" });
390
+ if ((res.status ?? -1) !== 0 || !res.stdout)
391
+ return null;
392
+ return parsePgMajor(res.stdout);
393
+ }
394
+ /** The local psql major version, or null if it can't be determined. */
395
+ function psqlMajor() {
396
+ const res = spawnSync("psql", ["--version"], { encoding: "utf8" });
397
+ if ((res.status ?? -1) !== 0 || !res.stdout)
398
+ return null;
399
+ return parsePgMajor(res.stdout);
400
+ }
401
+ /** Parse `psql -At` `key|value` rows into a Map. */
402
+ function parseKvRows(stdout) {
403
+ const m = new Map();
404
+ for (const line of stdout.split("\n")) {
405
+ if (!line)
406
+ continue;
407
+ const idx = line.indexOf("|");
408
+ if (idx === -1)
409
+ continue;
410
+ m.set(line.slice(0, idx), line.slice(idx + 1));
411
+ }
412
+ return m;
413
+ }
414
+ function sleep(ms) {
415
+ return new Promise((resolve) => setTimeout(resolve, ms));
416
+ }
417
+ /**
418
+ * Stream `pg_dump <source>` → GUC filter → `psql <target> -v ON_ERROR_STOP=1`.
419
+ * Nothing is buffered whole; the dump flows chunk-by-chunk. Deliberately NOT
420
+ * --single-transaction: one giant transaction through a serverless proxy is
421
+ * fragile. Collects restore stderr (the actionable errors, e.g. a PG17-only
422
+ * feature) and resolves ok only when both processes exit 0.
423
+ */
424
+ function runMigration(sourceUri, targetUri) {
425
+ return new Promise((resolve) => {
426
+ const filter = makeGucLineFilter();
427
+ const env = { ...process.env, PGCONNECT_TIMEOUT: process.env.PGCONNECT_TIMEOUT ?? "10" };
428
+ // `--encoding=UTF8` is load-bearing: the dump stream passes through JS strings
429
+ // here (dump.stdout is decoded as utf8 so the line filter can operate on text).
430
+ // Without it, a LATIN1 (or any non-UTF-8) source database's non-ASCII bytes
431
+ // would be replacement-mangled during that decode BEFORE reaching psql — silent
432
+ // data corruption that row-count verify wouldn't catch. Forcing UTF8 makes the
433
+ // server convert, guaranteeing a valid-UTF-8 stream; the dump carries
434
+ // `SET client_encoding = 'UTF8';`, which restores correctly into any target.
435
+ const dump = spawn("pg_dump", [sourceUri, "--no-owner", "--no-privileges", "--encoding=UTF8"], {
436
+ stdio: ["ignore", "pipe", "pipe"],
437
+ env,
438
+ });
439
+ // `-X` skips ~/.psqlrc: a user's custom settings (e.g. AUTOCOMMIT off, a
440
+ // different ON_ERROR_ROLLBACK, output tweaks) must not change restore
441
+ // semantics. `-w` never prompts for a password.
442
+ const restore = spawn("psql", [targetUri, "-X", "-w", "-v", "ON_ERROR_STOP=1"], {
443
+ stdio: ["pipe", "ignore", "pipe"],
444
+ env,
445
+ });
446
+ let restoreAlive = true;
447
+ const restoreErr = [];
448
+ let dumpErr = "";
449
+ let dumpCode = null;
450
+ let restoreCode = null;
451
+ let settled = false;
452
+ // Set true exactly where WE SIGTERM the dump after a restore failure, so the
453
+ // dump 'close' handler can tell our deliberate kill from an external one.
454
+ let intentionalDumpKill = false;
455
+ // Backpressure contract: psql can restore slower than pg_dump produces
456
+ // (index builds, TOAST writes, fsync). If we ignored write()'s return value,
457
+ // an unbounded backlog would buffer the ENTIRE dump in CLI memory and OOM on
458
+ // a multi-GB source. So when psql's stdin buffer is full (write() === false),
459
+ // pause the dump and resume only on 'drain'. This caps memory at ~one dump
460
+ // chunk plus psql's own buffer.
461
+ const safeWrite = (s) => {
462
+ if (!restoreAlive || !s)
463
+ return;
464
+ try {
465
+ if (!restore.stdin.write(s))
466
+ dump.stdout.pause();
467
+ }
468
+ catch {
469
+ /* psql closed its stdin (early exit) — stop feeding it */
470
+ }
471
+ };
472
+ restore.stdin.on("drain", () => {
473
+ if (restoreAlive)
474
+ dump.stdout.resume();
475
+ });
476
+ dump.stdout.setEncoding("utf8");
477
+ dump.stdout.on("data", (chunk) => safeWrite(filter.push(chunk)));
478
+ dump.stdout.on("end", () => {
479
+ safeWrite(filter.flush());
480
+ if (restoreAlive) {
481
+ try {
482
+ restore.stdin.end();
483
+ }
484
+ catch {
485
+ /* already closed */
486
+ }
487
+ }
488
+ });
489
+ dump.stderr.setEncoding("utf8");
490
+ dump.stderr.on("data", (d) => {
491
+ dumpErr += d;
492
+ });
493
+ restore.stderr.setEncoding("utf8");
494
+ restore.stderr.on("data", (d) => {
495
+ for (const line of d.split("\n")) {
496
+ const t = line.trim();
497
+ if (t)
498
+ restoreErr.push(t);
499
+ }
500
+ });
501
+ restore.stdin.on("error", () => {
502
+ /* EPIPE when psql exits before we finish writing — expected on failure */
503
+ });
504
+ const maybeSettle = () => {
505
+ if (settled || dumpCode === null || restoreCode === null)
506
+ return;
507
+ settled = true;
508
+ const ok = dumpCode === 0 && restoreCode === 0;
509
+ const errors = restoreErr.slice(0, 12);
510
+ if (dumpCode !== 0 && dumpErr.trim()) {
511
+ errors.push(...dumpErr
512
+ .trim()
513
+ .split("\n")
514
+ .map((s) => s.trim())
515
+ .filter(Boolean)
516
+ .slice(0, 4));
517
+ }
518
+ resolve({ ok, errors });
519
+ };
520
+ dump.on("error", (e) => {
521
+ dumpErr += `pg_dump failed to start: ${e.message}\n`;
522
+ dumpCode = dumpCode ?? -1;
523
+ restoreAlive = false;
524
+ try {
525
+ restore.stdin.end();
526
+ }
527
+ catch {
528
+ /* noop */
529
+ }
530
+ maybeSettle();
531
+ });
532
+ restore.on("error", (e) => {
533
+ restoreErr.push(`psql failed to start: ${e.message}`);
534
+ restoreCode = restoreCode ?? -1;
535
+ maybeSettle();
536
+ });
537
+ dump.on("close", (code, signal) => {
538
+ // A signal death (code === null) is a FAILURE — an OOM-killed or externally
539
+ // terminated pg_dump must not read as success just because psql exited
540
+ // cleanly. The ONLY exception is our own deliberate SIGTERM after a restore
541
+ // failure: restoreCode is already nonzero there so `ok` stays false anyway,
542
+ // and we normalize to 0 to avoid a redundant error line. On restore SUCCESS
543
+ // we never kill the dump — it closes naturally and its REAL exit code is
544
+ // honored here, so a late pg_dump failure can't be masked into success.
545
+ const r = resolveChildExit(code, signal, { intentionalKill: intentionalDumpKill });
546
+ if (r.error)
547
+ dumpErr += `pg_dump ${r.error}\n`;
548
+ dumpCode = r.code;
549
+ maybeSettle();
550
+ });
551
+ restore.on("close", (code, signal) => {
552
+ restoreAlive = false;
553
+ // A signal death is likewise a failure (never success on an external kill).
554
+ const r = resolveChildExit(code, signal);
555
+ if (r.error)
556
+ restoreErr.push(`psql ${r.error}`);
557
+ restoreCode = r.code;
558
+ // Kill the dump ONLY when the restore FAILED: a failed psql means no 'drain'
559
+ // will ever come, so a backpressure-paused pg_dump would hang forever and
560
+ // maybeSettle() would wait on dumpCode indefinitely — and its remaining
561
+ // output is useless anyway. On restore SUCCESS we deliberately do NOT kill:
562
+ // let pg_dump finish and report its real exit code, so maybeSettle (which
563
+ // waits for both) catches a late dump failure instead of hiding it.
564
+ if (restoreCode !== 0) {
565
+ intentionalDumpKill = true;
566
+ try {
567
+ dump.kill("SIGTERM");
568
+ }
569
+ catch {
570
+ /* already gone */
571
+ }
572
+ }
573
+ maybeSettle();
574
+ });
575
+ });
576
+ }
577
+ /**
578
+ * Delete a project we created for this import (DELETE /v1/projects/:id — the same
579
+ * endpoint `bata projects delete` uses). Returns whether it was actually removed,
580
+ * so the caller can tell the user to clean up manually if it wasn't. Only ever
581
+ * called for a project THIS run created — never a pre-existing --project target.
582
+ */
583
+ async function deleteCreatedProject(projectId, token, teamId) {
584
+ const query = {};
585
+ if (teamId)
586
+ query.team_id = teamId;
587
+ try {
588
+ const res = await api.del(`/v1/projects/${projectId}`, token, query);
589
+ return res.ok;
590
+ }
591
+ catch {
592
+ return false;
593
+ }
594
+ }
595
+ /** Fetch the primary branch's usable (password-revealed) direct URI — the same
596
+ * path `bata connect` / `bata db url` use. Never fabricated locally. */
597
+ async function getTargetUri(projectId, token) {
598
+ const res = await api.get(`/v1/connection-info/${projectId}`, token, { reveal: "true" });
599
+ if (!res.ok)
600
+ return null;
601
+ const conns = res.data?.connections ?? [];
602
+ const c = conns.find((x) => x.is_primary) ?? conns[0];
603
+ return c?.direct ?? null;
604
+ }
605
+ /**
606
+ * Poll GET /v1/branches?project_id= until the primary compute is ready. Cold
607
+ * attach can take 40s+, so the budget is patient (~180s). Falls back to a real
608
+ * `SELECT 1` probe once past the warm-up window, since a working query is the
609
+ * true readiness signal even if the `ready` flag lags.
610
+ */
611
+ async function waitForReady(projectId, targetUri, token, teamId, onTick) {
612
+ const budget = 180_000;
613
+ const interval = 3_000;
614
+ const probeAfter = 30_000;
615
+ const start = Date.now();
616
+ while (Date.now() - start < budget) {
617
+ const query = { project_id: projectId };
618
+ if (teamId)
619
+ query.team_id = teamId;
620
+ const res = await api.get("/v1/branches", token, query);
621
+ if (res.ok) {
622
+ const branches = asList(res.data);
623
+ const primary = branches.find((b) => b.isPrimary ?? b.is_primary) ?? branches[0];
624
+ if (primary?.ready === true)
625
+ return true;
626
+ }
627
+ if (Date.now() - start > probeAfter && runPsql(targetUri, "SELECT 1").code === 0) {
628
+ return true;
629
+ }
630
+ onTick(Math.round((Date.now() - start) / 1000));
631
+ await sleep(interval);
632
+ }
633
+ // Last chance: a successful query means ready regardless of the flag.
634
+ return runPsql(targetUri, "SELECT 1").code === 0;
635
+ }
636
+ // ── Command ──────────────────────────────────────────────────────────────────
637
+ export function parseImportArgs(args) {
638
+ let source;
639
+ let projectId;
640
+ let name;
641
+ let pg;
642
+ let help = false;
643
+ for (let i = 0; i < args.length; i++) {
644
+ const a = args[i];
645
+ const takeVal = () => {
646
+ const next = args[i + 1];
647
+ if (next === undefined || next.startsWith("-"))
648
+ return undefined;
649
+ i++;
650
+ return next;
651
+ };
652
+ if (a === "--help" || a === "-h")
653
+ help = true;
654
+ else if (a === "--source") {
655
+ const v = takeVal();
656
+ if (v === undefined)
657
+ return { error: "--source requires a value (a postgres:// connection URI)." };
658
+ source = v;
659
+ }
660
+ else if (a.startsWith("--source="))
661
+ source = a.slice("--source=".length);
662
+ else if (a === "--project") {
663
+ const v = takeVal();
664
+ if (v === undefined)
665
+ return { error: "--project requires a value (a project id)." };
666
+ projectId = v;
667
+ }
668
+ else if (a.startsWith("--project="))
669
+ projectId = a.slice("--project=".length);
670
+ else if (a === "--name") {
671
+ const v = takeVal();
672
+ if (v === undefined)
673
+ return { error: "--name requires a value." };
674
+ name = v;
675
+ }
676
+ else if (a.startsWith("--name="))
677
+ name = a.slice("--name=".length);
678
+ else if (a === "--pg") {
679
+ const v = takeVal();
680
+ if (v === undefined)
681
+ return { error: "--pg requires a value (a PostgreSQL major, e.g. 17)." };
682
+ pg = v;
683
+ }
684
+ else if (a.startsWith("--pg="))
685
+ pg = a.slice("--pg=".length);
686
+ // Unknown tokens / positionals are ignored — --source is the only input.
687
+ }
688
+ if (pg !== undefined && !/^\d+$/.test(pg)) {
689
+ return { error: `--pg must be a PostgreSQL major version number (got "${pg}").` };
690
+ }
691
+ // --pg picks the version of a NEW project; an existing project's version is fixed.
692
+ if (pg !== undefined && projectId !== undefined && !help) {
693
+ return { error: "--pg only applies when creating a new project — it cannot change an existing --project's version." };
694
+ }
695
+ // --project (import into an existing project) and --name (create a new one) are
696
+ // opposite intents. Silently taking --project was the riskier read (mutating an
697
+ // existing project while ignoring the requested name) — make it an explicit error.
698
+ if (projectId !== undefined && name !== undefined && !help) {
699
+ return {
700
+ error: "--project and --name are mutually exclusive (import into an existing project OR create a new one).",
701
+ };
702
+ }
703
+ return { source, projectId, name, pg, help };
704
+ }
705
+ /** Highest PostgreSQL major BataDB can create today — used only to pick the
706
+ * DEFAULT version for a new import target (min(source major, this), floored at
707
+ * 16). An explicit --pg is passed through as-is and validated by the API's
708
+ * allowlist, so a newer server accepts newer majors without a CLI release. */
709
+ const HIGHEST_SUPPORTED_PG = 17;
710
+ /** Default target major for a new project: match the source so features survive,
711
+ * capped at what BataDB ships (a PG18 source → 17, a PG16 or older source → 16). */
712
+ export function defaultTargetPg(sourceMajor) {
713
+ return String(Math.max(16, Math.min(sourceMajor, HIGHEST_SUPPORTED_PG)));
714
+ }
715
+ function printHelp() {
716
+ log();
717
+ log(` ${colors.bold("bata import")} ${colors.dim("--source <postgres-uri> [--project <id> | --name <name>] [--pg <major>]")}`);
718
+ log();
719
+ log(` Migrate a Neon (or any Postgres) database into BataDB via pg_dump | psql.`);
720
+ log();
721
+ log(` ${colors.bold("Options")}`);
722
+ log(` ${colors.dim("--source <uri>")} Source Postgres connection URI ${colors.dim("(required)")}`);
723
+ log(` ${colors.dim("--project <id>")} Import into an existing BataDB project`);
724
+ log(` ${colors.dim("--name <name>")} Create a new project with this name ${colors.dim("(default: source db name)")}`);
725
+ log(` ${colors.dim("--pg <major>")} PostgreSQL major for a NEW project ${colors.dim("(default: match the source, capped at 17)")}`);
726
+ log(` ${colors.dim("--yes, -y")} Proceed even if the target already has tables`);
727
+ log(` ${colors.dim("--json")} Machine-readable result`);
728
+ log();
729
+ log(` ${colors.bold("Prerequisites")}`);
730
+ log(` Local ${colors.cyan("pg_dump")} + ${colors.cyan("psql")} whose major ≥ the source server's.`);
731
+ log(` ${colors.dim("macOS:")} ${colors.cyan("brew install libpq")} ${colors.dim("Ubuntu:")} ${colors.cyan("sudo apt install postgresql-client-17")}`);
732
+ log();
733
+ log(` ${colors.bold("Examples")}`);
734
+ log(` ${colors.cyan('bata import --source "postgresql://…@ep-x.neon.tech/neondb"')}`);
735
+ log(` ${colors.cyan('bata import --source "$NEON_URL" --name my-app --yes --json')}`);
736
+ log();
737
+ log(` ${colors.dim("BataDB runs PostgreSQL 16 and 17. A new target defaults to the source's")}`);
738
+ log(` ${colors.dim("major (capped at 17); features newer than the target fail loudly during")}`);
739
+ log(` ${colors.dim("restore (nothing is hidden).")}`);
740
+ log();
741
+ }
742
+ export async function importDb(args) {
743
+ const parsed = parseImportArgs(args);
744
+ if ("error" in parsed) {
745
+ emitError("INVALID_FLAG", parsed.error, "Run `bata import --help` for usage.");
746
+ return;
747
+ }
748
+ if (parsed.help) {
749
+ printHelp();
750
+ return;
751
+ }
752
+ if (!parsed.source) {
753
+ emitError("MISSING_ARG", "A source database is required.", "Pass --source <postgres-uri>. Run `bata import --help` for usage.");
754
+ return;
755
+ }
756
+ const jsonMode = isJsonMode();
757
+ const token = requireToken();
758
+ const config = loadConfig();
759
+ const source = parsed.source;
760
+ const phase = (msg) => {
761
+ if (!jsonMode)
762
+ log(` ${colors.cyan("›")} ${msg}`);
763
+ };
764
+ if (!jsonMode)
765
+ heading("Import into BataDB");
766
+ // Ask the control plane for the current platform-scaffolding list (tables/
767
+ // sequences/schemas provisioning seeds into every project) and merge it into
768
+ // the baked-in defaults, so every downstream consumer — the source-collision
769
+ // warning, the --yes non-empty guard, and verification — keeps excluding the
770
+ // platform's own furniture even when it grows after this CLI was installed.
771
+ // Best-effort: any failure — older server (404), network error, or a transport
772
+ // THROW (timeout/DNS) — silently keeps the defaults; it must never abort the run.
773
+ try {
774
+ const scaffRes = await api.get("/v1/platform/import-scaffolding", token);
775
+ if (scaffRes.ok && scaffRes.data && typeof scaffRes.data === "object") {
776
+ applyServerScaffolding(scaffRes.data);
777
+ }
778
+ }
779
+ catch {
780
+ // baked-in defaults stay in effect
781
+ }
782
+ // The client tools are needed to even inspect the source, so verify they're on
783
+ // PATH up front (the pg_dump-major-vs-source-major gate stays in phase 2, once
784
+ // the source major is known).
785
+ if (!hasTool("pg_dump") || !hasTool("psql")) {
786
+ emitError("TOOLS_MISSING", "pg_dump and psql are required but were not found on PATH.", "macOS: brew install libpq · Ubuntu: sudo apt install postgresql-client", 1);
787
+ return;
788
+ }
789
+ // ── Phase 1: preflight the source ──────────────────────────────────────────
790
+ phase("Inspecting source database");
791
+ const summarySql = "SELECT json_build_object(" +
792
+ "'database', current_database()," +
793
+ "'server_version', current_setting('server_version')," +
794
+ "'server_version_num', current_setting('server_version_num')," +
795
+ "'size', pg_size_pretty(pg_database_size(current_database()))," +
796
+ "'tables', (SELECT count(*) FROM pg_tables WHERE schemaname NOT IN ('pg_catalog','information_schema'))" +
797
+ ")";
798
+ const srcRes = runPsql(source, summarySql);
799
+ if (srcRes.code !== 0) {
800
+ emitError("SOURCE_UNREACHABLE", `Could not read the source database: ${sourcePreflightErrorMessage(srcRes.stderr)}`, "Check the --source URI (host, credentials, sslmode) and that the server is reachable.", 1);
801
+ return;
802
+ }
803
+ let sourceSummary;
804
+ try {
805
+ sourceSummary = JSON.parse(srcRes.stdout.trim());
806
+ }
807
+ catch {
808
+ emitError("SOURCE_UNREACHABLE", "Unexpected response from the source database.", "", 1);
809
+ return;
810
+ }
811
+ const extRes = runPsql(source, "SELECT extname FROM pg_extension ORDER BY extname");
812
+ const sourceExtensions = extRes.code === 0
813
+ ? extRes.stdout.split("\n").map((s) => s.trim()).filter(Boolean)
814
+ : [];
815
+ const sourceMajor = parsePgMajor(sourceSummary.server_version_num || sourceSummary.server_version);
816
+ phase(`Source: ${sourceSummary.database} · PostgreSQL ${sourceSummary.server_version} · ` +
817
+ `${sourceSummary.size} · ${sourceSummary.tables} tables`);
818
+ // Warn up front if the SOURCE contains anything sharing a name with BataDB's
819
+ // seeded platform scaffolding — the named public.health_check table, or ANY
820
+ // object in a scaffolding schema (neon_migration), e.g. when migrating FROM
821
+ // another BataDB. Its CREATE will clash with the furniture the target is
822
+ // provisioned with, so the restore will likely fail later with "relation already
823
+ // exists" — flagging it now makes that error unsurprising. We do NOT touch source
824
+ // accounting; a source health_check / neon_migration object is treated as real
825
+ // data and verifies normally if it does restore.
826
+ const scaffoldingBareTables = PLATFORM_SCAFFOLDING_TABLES.filter((n) => n.startsWith("public."))
827
+ .map((n) => quoteLiteral(n.slice("public.".length)))
828
+ .join(",");
829
+ const scaffoldingSchemaLits = PLATFORM_SCAFFOLDING_SCHEMAS.map((s) => quoteLiteral(s)).join(",");
830
+ const srcCollisionRes = runPsql(source, "SELECT schemaname||'.'||tablename FROM pg_tables " +
831
+ `WHERE (schemaname = 'public' AND tablename IN (${scaffoldingBareTables})) ` +
832
+ `OR schemaname IN (${scaffoldingSchemaLits}) ORDER BY 1`);
833
+ const sourceScaffoldingHits = srcCollisionRes.code === 0
834
+ ? scaffoldingCollisions(srcCollisionRes.stdout.split("\n").map((s) => s.trim()).filter(Boolean))
835
+ : [];
836
+ if (sourceScaffoldingHits.length > 0 && !jsonMode) {
837
+ warn(`Source contains ${sourceScaffoldingHits.join(", ")}, which shares a name with BataDB's seeded ` +
838
+ 'platform scaffolding on the target. The restore will likely fail with "relation already ' +
839
+ 'exists" — rename or exclude it on the source to migrate cleanly.');
840
+ }
841
+ // ── Phase 2: check pg_dump AND psql are new enough ──────────────────────────
842
+ // BOTH matter: pg_dump can't dump a newer server than itself, and psql must be
843
+ // able to restore the stream THIS pg_dump produces — pg_dump 17 emits
844
+ // \restrict/\unrestrict regardless of the source version, so psql 16 + pg_dump
845
+ // 17 fails even against a PG16 source. checkClientVersions() encodes the full
846
+ // rule (psql ≥ max(source, pg_dump)); without it the restore would fail late,
847
+ // after a target was already created and warmed.
848
+ phase("Checking pg_dump / psql versions");
849
+ const dumpMajor = pgDumpMajor();
850
+ if (dumpMajor === null) {
851
+ emitError("TOOLS_MISSING", "Could not determine the pg_dump version.", "", 1);
852
+ return;
853
+ }
854
+ const psqlVer = psqlMajor();
855
+ if (psqlVer === null) {
856
+ emitError("TOOLS_MISSING", "Could not determine the psql version.", "", 1);
857
+ return;
858
+ }
859
+ const versionVerdict = checkClientVersions(dumpMajor, psqlVer, sourceMajor);
860
+ if (!versionVerdict.ok) {
861
+ emitError("PG_VERSION_TOO_OLD", versionVerdict.message, "macOS: brew install libpq · Ubuntu: sudo apt install postgresql-client-" + versionVerdict.needMajor, 1);
862
+ return;
863
+ }
864
+ // ── Phase 3: resolve / create the target ───────────────────────────────────
865
+ const teamId = await resolveTeamId(token);
866
+ let targetProjectId;
867
+ let created = false;
868
+ // The target's PostgreSQL major (for messaging + the JSON result). For a new
869
+ // project this is what we asked for; for --project it's read from the record.
870
+ let targetPg;
871
+ if (parsed.projectId) {
872
+ phase(`Resolving target project ${parsed.projectId}`);
873
+ const q = {};
874
+ if (teamId)
875
+ q.team_id = teamId;
876
+ const projRes = await api.get(`/v1/projects/${parsed.projectId}`, token, q);
877
+ if (!projRes.ok) {
878
+ emitError(projRes.status === 404 ? "NOT_FOUND"
879
+ : projRes.status === 401 || projRes.status === 403 ? "INVALID_KEY"
880
+ : projRes.status >= 500 || projRes.status === 0 ? "API_UNAVAILABLE"
881
+ : "CLI_ERROR", apiError(projRes, "Target project not found"), "");
882
+ return;
883
+ }
884
+ targetProjectId = parsed.projectId;
885
+ // Older control planes may omit pgVersion; fall back to the platform's
886
+ // historical default rather than guessing anything newer.
887
+ targetPg = projRes.data.pgVersion ?? "16";
888
+ }
889
+ else {
890
+ const name = parsed.name ?? defaultProjectName(sourceSummary.database, source);
891
+ // An unparseable source version keeps the conservative historical default.
892
+ targetPg = parsed.pg ?? (sourceMajor === null ? "16" : defaultTargetPg(sourceMajor));
893
+ phase(`Creating project ${name} (serverless · PostgreSQL ${targetPg})`);
894
+ if (sourceMajor !== null && sourceMajor > Number(targetPg) && !jsonMode) {
895
+ warn(`source is PG${sourceMajor}, BataDB target is PG${targetPg} — standard schemas migrate cleanly; ` +
896
+ `PG${sourceMajor}-only features will fail loudly during restore`);
897
+ }
898
+ const body = {
899
+ name,
900
+ region: "us-east-1",
901
+ compute: { tier: "serverless", size_cu: 1 },
902
+ pg_version: targetPg,
903
+ };
904
+ if (teamId)
905
+ body.team_id = teamId;
906
+ const res = await api.post("/v1/projects", body, token);
907
+ if (!res.ok) {
908
+ const explicitPg = parsed.pg !== undefined;
909
+ emitError(res.status === 401 || res.status === 403 ? "INVALID_KEY"
910
+ : res.status >= 500 || res.status === 0 ? "API_UNAVAILABLE"
911
+ : "CLI_ERROR", apiError(res, "Failed to create target project"),
912
+ // A 400 on an explicit --pg is almost always an unsupported major.
913
+ explicitPg && res.status === 400 ? "Try --pg 16 (or omit --pg for the default)." : "");
914
+ return;
915
+ }
916
+ targetProjectId = res.data.id;
917
+ created = true;
918
+ }
919
+ // Resolve the usable target URI (the deliverable) via the API — never local.
920
+ const targetUri = await getTargetUri(targetProjectId, token);
921
+ if (!targetUri) {
922
+ emitError("API_UNAVAILABLE", "Could not fetch the target connection string.", "", 6);
923
+ return;
924
+ }
925
+ phase("Waiting for target compute to be ready");
926
+ const ready = await waitForReady(targetProjectId, targetUri, token, teamId, (secs) => {
927
+ if (!jsonMode && secs > 0 && secs % 15 === 0)
928
+ phase(` …still waiting (${secs}s)`);
929
+ });
930
+ if (!ready) {
931
+ emitError("COMPUTE_STARTING", "Target compute did not become ready within 180s.", `Retry once it warms: bata status --project ${targetProjectId}`, 6);
932
+ return;
933
+ }
934
+ // For an existing target, ANY pre-existing user OBJECT risks a collision in a
935
+ // non-transactional restore — not just relations. This ONE fail-closed query
936
+ // counts: relations (table/partitioned/view/matview/sequence/foreign table),
937
+ // user-created schemas (excluding public), user types (enums, domains, and
938
+ // free-standing composites — NOT a table's implicit row type), and user
939
+ // functions (excluding extension-owned ones). If the query itself errors we must
940
+ // NOT treat the target as empty and silently proceed.
941
+ if (parsed.projectId) {
942
+ // BataDB seeds public.health_check (+ its sequence) into every project and our
943
+ // compute exposes an internal neon_migration schema, so a target holding ONLY
944
+ // scaffolding is effectively empty — exclude those from the counts so the --yes
945
+ // guard doesn't trip on our own furniture. Derived from the same constants as
946
+ // the rest of the scaffolding handling.
947
+ const scaffoldingPublicRelnames = [...PLATFORM_SCAFFOLDING_TABLES, ...PLATFORM_SCAFFOLDING_SEQUENCES]
948
+ .filter((n) => n.startsWith("public."))
949
+ .map((n) => quoteLiteral(n.slice("public.".length)))
950
+ .join(",");
951
+ const scaffoldingSchemaLiterals = PLATFORM_SCAFFOLDING_SCHEMAS.map((s) => quoteLiteral(s)).join(",");
952
+ const cnt = runPsql(targetUri, "SELECT" +
953
+ // relations, in any non-system namespace (incl. public), minus scaffolding:
954
+ // named public objects AND everything in a scaffolding schema.
955
+ " (SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace" +
956
+ " WHERE c.relkind IN ('r','p','v','m','S','f')" +
957
+ " AND n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_%'" +
958
+ ` AND n.nspname NOT IN (${scaffoldingSchemaLiterals})` +
959
+ ` AND NOT (n.nspname = 'public' AND c.relname IN (${scaffoldingPublicRelnames})))` +
960
+ // user-created schemas (public is a default, not user-created; scaffolding
961
+ // schemas are platform furniture)
962
+ " + (SELECT count(*) FROM pg_namespace n" +
963
+ " WHERE n.nspname NOT IN ('public','pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_%'" +
964
+ ` AND n.nspname NOT IN (${scaffoldingSchemaLiterals}))` +
965
+ // user types: enums, domains, and standalone composites (relkind 'c'),
966
+ // but not a table's implicit row type
967
+ " + (SELECT count(*) FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace" +
968
+ " WHERE n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_%'" +
969
+ ` AND n.nspname NOT IN (${scaffoldingSchemaLiterals})` +
970
+ " AND (t.typtype IN ('e','d')" +
971
+ " OR (t.typtype = 'c' AND EXISTS (SELECT 1 FROM pg_class cc WHERE cc.oid = t.typrelid AND cc.relkind = 'c'))))" +
972
+ // user functions, excluding extension-owned (pg_depend deptype 'e')
973
+ " + (SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace" +
974
+ " WHERE n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_%'" +
975
+ ` AND n.nspname NOT IN (${scaffoldingSchemaLiterals})` +
976
+ " AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = p.oid AND d.classid = 'pg_proc'::regclass AND d.deptype = 'e'))" +
977
+ " AS n");
978
+ if (cnt.code !== 0) {
979
+ emitError("CLI_ERROR", "Could not verify whether the target project is empty before importing.", "Retry once the target compute is warm, or check the target connection.", 1);
980
+ return;
981
+ }
982
+ const existingObjects = parseInt(cnt.stdout.trim(), 10) || 0;
983
+ if (existingObjects > 0 && !isYes()) {
984
+ emitError("INVALID_FLAG", `Target project already has ${existingObjects} user object(s). Importing may collide with existing objects.`, "Re-run with --yes to import into it anyway.");
985
+ return;
986
+ }
987
+ if (existingObjects > 0 && !jsonMode) {
988
+ warn(`Target already has ${existingObjects} user object(s) — proceeding because --yes was set.`);
989
+ }
990
+ }
991
+ // Extension preflight — BEFORE the dump. With ON_ERROR_STOP=1 the restore
992
+ // would die at the first `CREATE EXTENSION <x>;` the target can't provide, so
993
+ // checking availability up front lets us fail loudly with the full list
994
+ // instead of a cryptic mid-restore error (and a half-imported target). Fail
995
+ // CLOSED: if we can't even query availability, don't start a restore we know
996
+ // might half-apply.
997
+ if (sourceExtensions.length > 0) {
998
+ phase("Checking target extension availability");
999
+ // We reach this AFTER (possibly) creating the project and waiting for its
1000
+ // compute — so a failure here would strand an empty project. If THIS run
1001
+ // created it, delete it before exiting and say so; if the delete fails, tell
1002
+ // the user the id to clean up. A pre-existing --project target is never
1003
+ // touched (created === false).
1004
+ const failWithCleanup = async (message, baseHint) => {
1005
+ let hint = baseHint;
1006
+ if (created) {
1007
+ const removed = await deleteCreatedProject(targetProjectId, token, teamId);
1008
+ hint += removed
1009
+ ? " Removed the empty project that was created for this import."
1010
+ : ` Could not remove the project created for this import — delete it manually: bata projects delete ${targetProjectId}`;
1011
+ }
1012
+ emitError("EXTENSION_UNSUPPORTED", message, hint, 1);
1013
+ throw new Error("unreachable"); // emitError exits; satisfy the `never` return
1014
+ };
1015
+ const availRes = runPsql(targetUri, "SELECT name FROM pg_available_extensions ORDER BY name");
1016
+ if (availRes.code !== 0) {
1017
+ await failWithCleanup("Could not verify which extensions the target supports.", "Retry once the target compute is warm — a restore isn't safe to start until this is known.");
1018
+ }
1019
+ const available = availRes.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
1020
+ const unsupported = missingExtensions(sourceExtensions, available);
1021
+ if (unsupported.length > 0) {
1022
+ await failWithCleanup(`The source uses extension(s) the target can't provide: ${unsupported.join(", ")}.`, "BataDB may not offer these yet. Drop them on the source (DROP EXTENSION …) before importing, or contact BataDB support to request them.");
1023
+ }
1024
+ }
1025
+ // Once we're past target creation + restore start, a failure leaves a
1026
+ // partially-populated target. We DELIBERATELY do NOT auto-delete it here
1027
+ // (unlike the extension preflight, which fails before any data moves): a
1028
+ // partial restore can represent significant transfer time and is useful for
1029
+ // diagnosis and retry, so destroying it silently would be worse. Instead every
1030
+ // post-restore failure path NAMES the project and gives actionable next steps.
1031
+ const strandedTargetJson = () => ({
1032
+ project_id: targetProjectId,
1033
+ created,
1034
+ connection: targetUri,
1035
+ });
1036
+ const printStrandedTargetHints = () => {
1037
+ log(` ${colors.dim("The target project was left in place for diagnosis/retry:")}`);
1038
+ kvList([["Project ID", colors.dim(targetProjectId)]]);
1039
+ log();
1040
+ log(` ${colors.dim("Retry into the SAME target after fixing the issue:")}`);
1041
+ log(` ${colors.cyan(`bata import --source <source> --project ${targetProjectId} --yes`)}`);
1042
+ if (created) {
1043
+ log(` ${colors.dim("…or remove the partial project:")}`);
1044
+ log(` ${colors.cyan(`bata projects delete ${targetProjectId} --yes`)}`);
1045
+ }
1046
+ log();
1047
+ };
1048
+ const strandedHintJson = "The target project was left in place (NOT deleted) for diagnosis/retry — re-run with " +
1049
+ `--project ${targetProjectId} --yes after fixing` +
1050
+ (created ? `, or remove it with: bata projects delete ${targetProjectId} --yes.` : ".");
1051
+ // ── Phase 4: migrate ───────────────────────────────────────────────────────
1052
+ phase("Migrating schema + data (pg_dump | psql)");
1053
+ const migration = await runMigration(source, targetUri);
1054
+ if (!migration.ok) {
1055
+ if (jsonMode) {
1056
+ json({
1057
+ error: "Restore failed — the target rejected part of the dump.",
1058
+ code: "RESTORE_FAILED",
1059
+ hint: `Schema features newer than the target's PostgreSQL ${targetPg} aren't supported there. ` +
1060
+ "Review the errors below. " +
1061
+ strandedHintJson,
1062
+ errors: migration.errors,
1063
+ target: strandedTargetJson(),
1064
+ });
1065
+ }
1066
+ else {
1067
+ log();
1068
+ emitErrorHuman("Restore failed — the target rejected part of the dump.", migration.errors);
1069
+ printStrandedTargetHints();
1070
+ }
1071
+ process.exit(1);
1072
+ return;
1073
+ }
1074
+ // ── Phase 5: verify ────────────────────────────────────────────────────────
1075
+ // Fail CLOSED: a verify query that can't run must never read as "no mismatch"
1076
+ // (that would report a false success). Any failed query, or an unexpectedly
1077
+ // empty result for a non-empty input, is a hard failure that tells the user
1078
+ // verification couldn't run — never silent success.
1079
+ phase("Verifying row counts + sequences");
1080
+ let verifyFailed = null;
1081
+ const tableListRes = runPsql(source, "SELECT COALESCE(json_agg(json_build_object('schema', schemaname, 'table', tablename) " +
1082
+ "ORDER BY schemaname, tablename), '[]') " +
1083
+ "FROM pg_tables WHERE schemaname NOT IN ('pg_catalog','information_schema')");
1084
+ if (tableListRes.code !== 0)
1085
+ verifyFailed = "could not list source tables";
1086
+ const sourceTables = tableListRes.code === 0 ? safeJsonArray(tableListRes.stdout) : [];
1087
+ const srcCounts = new Map();
1088
+ const tgtCounts = new Map();
1089
+ if (!verifyFailed && sourceTables.length > 0) {
1090
+ // Chunk into batches so a huge table list can't blow psql's statement limit
1091
+ // (which would fail the whole UNION ALL and leave both maps empty).
1092
+ for (const batch of chunk(sourceTables, 100)) {
1093
+ const countsSql = batch
1094
+ .map((t) => `SELECT ${quoteLiteral(`${t.schema}.${t.table}`)} AS t, count(*)::text AS c ` +
1095
+ `FROM ${quoteIdent(t.schema)}.${quoteIdent(t.table)}`)
1096
+ .join("\nUNION ALL\n");
1097
+ const sRes = runPsql(source, countsSql);
1098
+ const tRes = runPsql(targetUri, countsSql);
1099
+ if (sRes.code !== 0 || tRes.code !== 0) {
1100
+ verifyFailed = "row-count query failed";
1101
+ break;
1102
+ }
1103
+ const sMap = parseKvRows(sRes.stdout);
1104
+ const tMap = parseKvRows(tRes.stdout);
1105
+ // A query that returned fewer rows than tables didn't run cleanly.
1106
+ if (sMap.size !== batch.length || tMap.size !== batch.length) {
1107
+ verifyFailed = "row-count query returned an incomplete result";
1108
+ break;
1109
+ }
1110
+ for (const [k, v] of sMap)
1111
+ srcCounts.set(k, v);
1112
+ for (const [k, v] of tMap)
1113
+ tgtCounts.set(k, v);
1114
+ }
1115
+ }
1116
+ const tableMismatches = verifyFailed ? [] : diffCounts(srcCounts, tgtCounts);
1117
+ const seqSql = "SELECT schemaname||'.'||sequencename AS s, COALESCE(last_value::text,'') AS v " +
1118
+ "FROM pg_sequences ORDER BY 1";
1119
+ const srcSeq = new Map();
1120
+ const tgtSeq = new Map();
1121
+ if (!verifyFailed) {
1122
+ const sSeqRes = runPsql(source, seqSql);
1123
+ const tSeqRes = runPsql(targetUri, seqSql);
1124
+ if (sSeqRes.code !== 0 || tSeqRes.code !== 0) {
1125
+ verifyFailed = "sequence query failed";
1126
+ }
1127
+ else {
1128
+ for (const [k, v] of parseKvRows(sSeqRes.stdout))
1129
+ srcSeq.set(k, v);
1130
+ for (const [k, v] of parseKvRows(tSeqRes.stdout))
1131
+ tgtSeq.set(k, v);
1132
+ }
1133
+ }
1134
+ // Target-only sequences (present on target, absent from source) get the SAME
1135
+ // treatment as target-only tables: on a --project --yes existing target they're
1136
+ // expected leftovers (e.g. a pre-existing serial/identity table contributes one),
1137
+ // so they're excluded from the mismatch diff and surfaced as untouched — NOT a
1138
+ // failure. Source-only sequences and value drift on shared ones stay mismatches.
1139
+ let targetOnlySequences = [];
1140
+ let seqMismatches = [];
1141
+ if (!verifyFailed) {
1142
+ // The RAW set (incl. BataDB scaffolding) is what we exclude from the mismatch
1143
+ // diff — a target-only sequence must never read as a mismatch. The REPORTED
1144
+ // list + the created-project hard-fail then drop platform scaffolding (BataDB
1145
+ // seeds public.health_check_id_seq into every project, so it's expected on any
1146
+ // target and must not false-positive a freshly created one).
1147
+ const rawTargetOnlySeq = targetOnlyNames([...srcSeq.keys()], [...tgtSeq.keys()]);
1148
+ const rawTargetOnlySet = new Set(rawTargetOnlySeq);
1149
+ seqMismatches = diffCounts(srcSeq, tgtSeq).filter((r) => !rawTargetOnlySet.has(r.name));
1150
+ targetOnlySequences = excludeScaffolding(rawTargetOnlySeq);
1151
+ // A freshly created project must have no NON-scaffolding sequences the source lacks.
1152
+ if (created && targetOnlySequences.length > 0) {
1153
+ verifyFailed =
1154
+ `freshly created target unexpectedly has ${targetOnlySequences.length} sequence(s) absent ` +
1155
+ `from the source: ${targetOnlySequences.join(", ")}`;
1156
+ }
1157
+ }
1158
+ // Enumerate the TARGET's tables too — verify only counts SOURCE tables, so a
1159
+ // --project --yes import into a non-empty target has pre-existing target-only
1160
+ // tables our row-count check never sees. Fail closed like the rest of verify.
1161
+ let targetOnly = [];
1162
+ if (!verifyFailed) {
1163
+ const tgtTablesRes = runPsql(targetUri, "SELECT schemaname||'.'||tablename FROM pg_tables " +
1164
+ "WHERE schemaname NOT IN ('pg_catalog','information_schema') ORDER BY 1");
1165
+ if (tgtTablesRes.code !== 0) {
1166
+ verifyFailed = "could not list target tables";
1167
+ }
1168
+ else {
1169
+ const targetQualified = tgtTablesRes.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
1170
+ const sourceQualified = sourceTables.map((t) => `${t.schema}.${t.table}`);
1171
+ // Drop BataDB scaffolding (public.health_check + the neon_migration schema) —
1172
+ // seeded/created on every project, so never a real "target-only" leftover and
1173
+ // must not false-positive a freshly created target. (Target-only tables never
1174
+ // enter the count mismatch diff, so no raw/reported split is needed here.)
1175
+ targetOnly = excludeScaffolding(targetOnlyNames(sourceQualified, targetQualified));
1176
+ // A FRESHLY CREATED project should have no NON-scaffolding tables the source
1177
+ // lacks. If it does, something is genuinely wrong (wrong target, contaminated
1178
+ // template) — hard-fail rather than wave it through as an expected leftover.
1179
+ if (created && targetOnly.length > 0) {
1180
+ verifyFailed =
1181
+ `freshly created target unexpectedly has ${targetOnly.length} table(s) absent ` +
1182
+ `from the source: ${targetOnly.join(", ")}`;
1183
+ }
1184
+ }
1185
+ }
1186
+ const tgtExtRes = runPsql(targetUri, "SELECT extname FROM pg_extension ORDER BY extname");
1187
+ const targetExtensions = tgtExtRes.code === 0
1188
+ ? tgtExtRes.stdout.split("\n").map((s) => s.trim()).filter(Boolean)
1189
+ : [];
1190
+ // Residual check only: unsupported extensions were already hard-failed in the
1191
+ // pre-restore preflight, so this now catches only the oddity of an extension
1192
+ // that IS available on the target but somehow didn't get installed.
1193
+ const residualMissingExtensions = missingExtensions(sourceExtensions, targetExtensions);
1194
+ // Verification couldn't run — the data was restored but we can't confirm it
1195
+ // matches. Hard-fail rather than imply success.
1196
+ if (verifyFailed) {
1197
+ if (jsonMode) {
1198
+ json({
1199
+ error: `Verification could not run: ${verifyFailed}.`,
1200
+ code: "VERIFY_FAILED",
1201
+ hint: "The dump/restore reported success, but the CLI could not confirm row counts/sequences match. " +
1202
+ "Verify manually before relying on the target. " +
1203
+ strandedHintJson,
1204
+ target: strandedTargetJson(),
1205
+ });
1206
+ }
1207
+ else {
1208
+ log();
1209
+ log(` ${colors.red("Verification could not run")} — ${verifyFailed}.`);
1210
+ log(` ${colors.dim("The data was restored, but counts/sequences were NOT confirmed. Verify manually before relying on it.")}`);
1211
+ log();
1212
+ printStrandedTargetHints();
1213
+ }
1214
+ process.exit(1);
1215
+ return;
1216
+ }
1217
+ if (tableMismatches.length > 0 || seqMismatches.length > 0) {
1218
+ if (jsonMode) {
1219
+ json({
1220
+ error: "Verification failed — source and target differ after import.",
1221
+ code: "VERIFY_MISMATCH",
1222
+ hint: "The dump/restore reported success but row counts or sequences don't match. " + strandedHintJson,
1223
+ mismatches: {
1224
+ tables: tableMismatches,
1225
+ sequences: seqMismatches,
1226
+ },
1227
+ target: strandedTargetJson(),
1228
+ });
1229
+ }
1230
+ else {
1231
+ log();
1232
+ log(` ${colors.red("Verification failed")} — source and target differ:`);
1233
+ log();
1234
+ table(["OBJECT", "SOURCE", "TARGET"], mismatchRows(tableMismatches, seqMismatches));
1235
+ log();
1236
+ printStrandedTargetHints();
1237
+ }
1238
+ process.exit(1);
1239
+ return;
1240
+ }
1241
+ // ── Phase 6: done ──────────────────────────────────────────────────────────
1242
+ if (jsonMode) {
1243
+ json({
1244
+ ok: true,
1245
+ source: {
1246
+ database: sourceSummary.database,
1247
+ server_version: sourceSummary.server_version,
1248
+ server_major: sourceMajor,
1249
+ size: sourceSummary.size,
1250
+ tables: sourceSummary.tables,
1251
+ extensions: sourceExtensions,
1252
+ },
1253
+ target: {
1254
+ project_id: targetProjectId,
1255
+ created,
1256
+ pg_version: targetPg,
1257
+ connection: targetUri,
1258
+ },
1259
+ verify: {
1260
+ tables_checked: sourceTables.length,
1261
+ sequences_checked: srcSeq.size,
1262
+ extensions_missing_on_target: residualMissingExtensions,
1263
+ // Pre-existing target tables/sequences the source doesn't have — left
1264
+ // untouched and NOT covered by verification (only possible on --project
1265
+ // --yes into a non-empty target).
1266
+ target_only_tables: targetOnly,
1267
+ target_only_sequences: targetOnlySequences,
1268
+ },
1269
+ });
1270
+ return;
1271
+ }
1272
+ if (targetOnly.length > 0) {
1273
+ warn(`${targetOnly.length} pre-existing table(s) on the target were left untouched and are ` +
1274
+ `NOT covered by verification: ${targetOnly.join(", ")}`);
1275
+ }
1276
+ if (targetOnlySequences.length > 0) {
1277
+ warn(`${targetOnlySequences.length} pre-existing sequence(s) on the target were left untouched and are ` +
1278
+ `NOT covered by verification: ${targetOnlySequences.join(", ")}`);
1279
+ }
1280
+ if (residualMissingExtensions.length > 0) {
1281
+ warn(`Extensions on the source not installed on the target: ${residualMissingExtensions.join(", ")} ` +
1282
+ "(available but not created during restore — install them manually if needed).");
1283
+ }
1284
+ log();
1285
+ success(`Imported into ${colors.cyan(targetProjectId)} — ${sourceSummary.tables} tables verified.`);
1286
+ log();
1287
+ kvList([
1288
+ ["Project ID", colors.dim(targetProjectId)],
1289
+ ["PostgreSQL", targetPg],
1290
+ ["Connection", colors.dim(targetUri)],
1291
+ ]);
1292
+ log();
1293
+ log(` ${colors.dim("Connect:")} ${colors.cyan(`bata connect ${targetProjectId}`)}`);
1294
+ log();
1295
+ }
1296
+ /** Print a restore failure with its first errors (human mode). */
1297
+ function emitErrorHuman(message, errors) {
1298
+ log(` ${colors.red("Error:")} ${message}`);
1299
+ if (errors.length > 0) {
1300
+ log();
1301
+ for (const e of errors)
1302
+ log(` ${colors.dim(e)}`);
1303
+ }
1304
+ log();
1305
+ log(` ${colors.dim("Schema features newer than the target's PostgreSQL major aren't supported there.")}`);
1306
+ log();
1307
+ }
1308
+ function safeJsonArray(stdout) {
1309
+ try {
1310
+ const v = JSON.parse(stdout.trim());
1311
+ return Array.isArray(v) ? v : [];
1312
+ }
1313
+ catch {
1314
+ return [];
1315
+ }
1316
+ }