@batadata/cli 0.1.17 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/dist/commands/compute.d.ts +13 -0
- package/dist/commands/compute.js +131 -1
- package/dist/commands/import.d.ts +197 -0
- package/dist/commands/import.js +1316 -0
- package/dist/commands/projects.d.ts +17 -1
- package/dist/commands/projects.js +82 -3
- package/dist/index.js +14 -2
- package/dist/pricing.d.ts +35 -0
- package/dist/pricing.js +44 -0
- package/package.json +5 -3
|
@@ -0,0 +1,1316 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* bata import — Neon (or any Postgres) → BataDB migration.
|
|
3
|
+
*
|
|
4
|
+
* bata import --source <postgres-uri> [--project <id> | --name <name>] [--pg <major>] [--yes] [--json]
|
|
5
|
+
*
|
|
6
|
+
* Agent-native: shells out to the standard `pg_dump` / `psql` client tools
|
|
7
|
+
* (zero runtime deps) and streams a dump straight into a fresh BataDB project.
|
|
8
|
+
*
|
|
9
|
+
* BataDB runs PostgreSQL 16 today; Neon defaults to 17. A PG17 → PG16
|
|
10
|
+
* dump/restore is clean for standard schemas EXCEPT that pg_dump 17 emits
|
|
11
|
+
* `SET transaction_timeout = 0;` — a GUC PG16 doesn't know — into the dump
|
|
12
|
+
* preamble. We strip that one line (and any future target-unknown GUC SETs)
|
|
13
|
+
* out of the stream. Genuinely PG17-only schema features still fail loudly at
|
|
14
|
+
* restore time, and we surface those errors honestly rather than pretending
|
|
15
|
+
* the migration succeeded.
|
|
16
|
+
*/
|
|
17
|
+
import { spawn, spawnSync, execSync } from "node:child_process";
|
|
18
|
+
import { URL } from "node:url";
|
|
19
|
+
import { api, apiError, resolveTeamId, asList } from "../api.js";
|
|
20
|
+
import { requireToken, loadConfig, isJsonMode, isYes } from "../config.js";
|
|
21
|
+
import { colors, log, json, success, warn, heading, kvList, table } from "../utils/logger.js";
|
|
22
|
+
import { emitError } from "../utils/errors.js";
|
|
23
|
+
// ── Pure, unit-tested helpers ────────────────────────────────────────────────
|
|
24
|
+
/**
|
|
25
|
+
* GUC SET statements pg_dump emits that a BataDB (PG16) target rejects. Kept as
|
|
26
|
+
* an extensible list so a future target-unknown GUC is a one-line addition.
|
|
27
|
+
* `transaction_timeout` is new in PG17 and is the only one seen in practice.
|
|
28
|
+
*/
|
|
29
|
+
export const TARGET_UNKNOWN_GUCS = ["transaction_timeout"];
|
|
30
|
+
/**
|
|
31
|
+
* A streaming line filter that drops target-unknown GUC `SET` statements from a
|
|
32
|
+
* pg_dump stream. It buffers a partial trailing line across chunk boundaries so
|
|
33
|
+
* a `SET transaction_timeout = 0;` split across two reads is still recognised
|
|
34
|
+
* and removed. `push()` returns the text to forward to psql for that chunk;
|
|
35
|
+
* `flush()` returns any buffered final line (a dump without a trailing newline).
|
|
36
|
+
*
|
|
37
|
+
* PREAMBLE-ONLY: pg_dump emits the target-unknown GUCs ONLY in the initial
|
|
38
|
+
* connection preamble (the block of `SET …;` / `SELECT pg_catalog.set_config(…)`
|
|
39
|
+
* statements at the very top). Filtering the whole stream is unsafe: a dumped
|
|
40
|
+
* function/procedure body can contain a standalone line `SET transaction_timeout
|
|
41
|
+
* = 0;` inside its dollar-quoted text, and a COPY data row can be literally
|
|
42
|
+
* anything — mutating those would silently corrupt the restore. So we only strip
|
|
43
|
+
* inside the preamble: the first line that is NOT one of {empty, a `--` comment,
|
|
44
|
+
* `SET …;`, `SELECT pg_catalog.set_config(…);`} permanently ends the preamble,
|
|
45
|
+
* after which every line passes through verbatim. A GUC-looking SET after the
|
|
46
|
+
* preamble is therefore NOT stripped — which is correct: transaction_timeout only
|
|
47
|
+
* ever appears in the preamble; were a future pg_dump to emit it mid-dump, the
|
|
48
|
+
* restore would fail loudly rather than be silently altered.
|
|
49
|
+
*/
|
|
50
|
+
export function makeGucLineFilter(gucs = TARGET_UNKNOWN_GUCS) {
|
|
51
|
+
// `^SET <guc> …;` — a standalone GUC SET statement. Anchored to the start of
|
|
52
|
+
// the line and requiring the trailing semicolon so it can't match a data row
|
|
53
|
+
// that merely begins with the same text.
|
|
54
|
+
const patterns = gucs.map((g) => new RegExp(`^SET\\s+${g}\\b[^\\n]*;\\s*$`, "i"));
|
|
55
|
+
const isStripped = (line) => patterns.some((re) => re.test(line));
|
|
56
|
+
// Lines that legitimately appear in the preamble; the first line that matches
|
|
57
|
+
// NONE of these ends the preamble for good. `\restrict <token>` is a psql
|
|
58
|
+
// meta-command pg_dump 17.x emits at the top (paired with `\unrestrict` at the
|
|
59
|
+
// very end) — it sits BEFORE the GUC SETs, so it must be recognised or the
|
|
60
|
+
// preamble would end early and the GUC we strip would sail through untouched.
|
|
61
|
+
const PREAMBLE_LINE = [
|
|
62
|
+
/^\s*$/, // blank
|
|
63
|
+
/^--/, // SQL comment
|
|
64
|
+
/^\\(un)?restrict\b/i, // psql \restrict / \unrestrict (PG17.x)
|
|
65
|
+
/^SET\s.+;\s*$/i, // any GUC SET (incl. the ones we strip)
|
|
66
|
+
/^SELECT\s+pg_catalog\.set_config\(.*\);\s*$/i, // set_config form
|
|
67
|
+
];
|
|
68
|
+
const isPreambleLine = (line) => PREAMBLE_LINE.some((re) => re.test(line));
|
|
69
|
+
let preambleDone = false;
|
|
70
|
+
let pending = "";
|
|
71
|
+
// Whether a preamble line survives (updating the preambleDone latch). Only
|
|
72
|
+
// called while still in the preamble.
|
|
73
|
+
const keepPreambleLine = (raw) => {
|
|
74
|
+
const line = raw.replace(/\r$/, "");
|
|
75
|
+
if (!isPreambleLine(line)) {
|
|
76
|
+
preambleDone = true; // first non-preamble line ends filtering permanently
|
|
77
|
+
return true;
|
|
78
|
+
}
|
|
79
|
+
return !isStripped(line);
|
|
80
|
+
};
|
|
81
|
+
return {
|
|
82
|
+
push(chunk) {
|
|
83
|
+
// Once out of the preamble, bypass line-splitting entirely and stream the
|
|
84
|
+
// chunk through verbatim. Buffering-until-newline here would hold a single
|
|
85
|
+
// huge COPY value (a multi-MB TOAST row on one line) entirely in memory and
|
|
86
|
+
// defeat backpressure — the body is never filtered, so there's no reason to
|
|
87
|
+
// reassemble its lines.
|
|
88
|
+
if (preambleDone)
|
|
89
|
+
return chunk;
|
|
90
|
+
pending += chunk;
|
|
91
|
+
const lines = pending.split("\n");
|
|
92
|
+
// The final element has no terminating newline yet — hold it for next push.
|
|
93
|
+
pending = lines.pop() ?? "";
|
|
94
|
+
let out = "";
|
|
95
|
+
for (let i = 0; i < lines.length; i++) {
|
|
96
|
+
if (keepPreambleLine(lines[i]))
|
|
97
|
+
out += lines[i] + "\n";
|
|
98
|
+
// The moment the preamble ends, everything remaining in this chunk is
|
|
99
|
+
// body — emit it verbatim (with its newlines) and stop line-processing.
|
|
100
|
+
if (preambleDone) {
|
|
101
|
+
const rest = lines.slice(i + 1);
|
|
102
|
+
if (rest.length > 0)
|
|
103
|
+
out += rest.join("\n") + "\n";
|
|
104
|
+
if (pending) {
|
|
105
|
+
out += pending;
|
|
106
|
+
pending = "";
|
|
107
|
+
}
|
|
108
|
+
break;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return out;
|
|
112
|
+
},
|
|
113
|
+
flush() {
|
|
114
|
+
// After the preamble, push() already emitted everything (pending is empty).
|
|
115
|
+
const last = pending;
|
|
116
|
+
pending = "";
|
|
117
|
+
if (!last)
|
|
118
|
+
return "";
|
|
119
|
+
// A dump that is ALL preamble (no trailing newline on the last line).
|
|
120
|
+
return keepPreambleLine(last) ? last : "";
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Extract a PostgreSQL major version from any of the shapes we encounter:
|
|
126
|
+
* - `server_version_num` (e.g. "170002", "160003", "90603")
|
|
127
|
+
* - dotted `server_version` (e.g. "17.2", "16.3 (Ubuntu 16.3-1.pgdg…)")
|
|
128
|
+
* - `pg_dump --version` output ("pg_dump (PostgreSQL) 16.3")
|
|
129
|
+
* Returns the integer major, or null if none can be found.
|
|
130
|
+
*/
|
|
131
|
+
export function parsePgMajor(v) {
|
|
132
|
+
const t = (v ?? "").trim();
|
|
133
|
+
// server_version_num form: all digits, wide enough to be an encoded version.
|
|
134
|
+
if (/^\d+$/.test(t) && t.length >= 5) {
|
|
135
|
+
return Math.floor(parseInt(t, 10) / 10000);
|
|
136
|
+
}
|
|
137
|
+
const m = t.match(/(\d+)/);
|
|
138
|
+
return m ? parseInt(m[1], 10) : null;
|
|
139
|
+
}
|
|
140
|
+
/** A client tool must be at least the source major (pg_dump can't dump a newer
|
|
141
|
+
* server than itself; psql can't parse a newer pg_dump's meta-commands). */
|
|
142
|
+
export function toolMajorSatisfies(toolMajor, sourceMajor) {
|
|
143
|
+
return toolMajor >= sourceMajor;
|
|
144
|
+
}
|
|
145
|
+
/** Clear, actionable message when a local client tool is too old for the source. */
|
|
146
|
+
export function toolVersionErrorMessage(tool, toolMajor, sourceMajor) {
|
|
147
|
+
return (`Your ${tool} is version ${toolMajor}, but the source server is PostgreSQL ${sourceMajor}. ` +
|
|
148
|
+
`A client older than the source can't reliably migrate it (e.g. psql ${sourceMajor - 1} can't parse ` +
|
|
149
|
+
`pg_dump ${sourceMajor}'s \\restrict meta-commands). Install PostgreSQL ${sourceMajor} (or newer) ` +
|
|
150
|
+
`client tools and re-run.`);
|
|
151
|
+
}
|
|
152
|
+
/** Message when psql can't restore the stream THIS pg_dump produces. */
|
|
153
|
+
export function psqlVsDumpErrorMessage(psqlMajor, dumpMajor) {
|
|
154
|
+
return (`Your psql is version ${psqlMajor}, but pg_dump is version ${dumpMajor} — psql can't reliably ` +
|
|
155
|
+
`restore a stream produced by a newer pg_dump (pg_dump ${dumpMajor} emits \\restrict meta-commands ` +
|
|
156
|
+
`psql ${psqlMajor} doesn't understand). Install matching PostgreSQL ${dumpMajor} client tools and re-run.`);
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Verdict on whether the local client tools can migrate a source of `sourceMajor`
|
|
160
|
+
* (null when the source version couldn't be parsed). Rules:
|
|
161
|
+
* - pg_dump must be ≥ source (can't dump a newer server than itself)
|
|
162
|
+
* - psql must be ≥ source AND ≥ pg_dump — the restore stream is produced by the
|
|
163
|
+
* LOCAL pg_dump, and pg_dump 17 emits \restrict/\unrestrict regardless of the
|
|
164
|
+
* source version, so psql 16 + pg_dump 17 fails even against a PG16 source.
|
|
165
|
+
* Effectively psql must satisfy max(sourceMajor, dumpMajor). Returns the install
|
|
166
|
+
* target so the caller can point the user at the right client package.
|
|
167
|
+
*/
|
|
168
|
+
export function checkClientVersions(dumpMajor, psqlMajor, sourceMajor) {
|
|
169
|
+
if (sourceMajor !== null && !toolMajorSatisfies(dumpMajor, sourceMajor)) {
|
|
170
|
+
return { ok: false, message: toolVersionErrorMessage("pg_dump", dumpMajor, sourceMajor), needMajor: sourceMajor };
|
|
171
|
+
}
|
|
172
|
+
if (sourceMajor !== null && !toolMajorSatisfies(psqlMajor, sourceMajor)) {
|
|
173
|
+
return { ok: false, message: toolVersionErrorMessage("psql", psqlMajor, sourceMajor), needMajor: sourceMajor };
|
|
174
|
+
}
|
|
175
|
+
if (!toolMajorSatisfies(psqlMajor, dumpMajor)) {
|
|
176
|
+
return { ok: false, message: psqlVsDumpErrorMessage(psqlMajor, dumpMajor), needMajor: dumpMajor };
|
|
177
|
+
}
|
|
178
|
+
return { ok: true };
|
|
179
|
+
}
|
|
180
|
+
/** Pull the first meaningful line out of a psql connection error for the user. */
|
|
181
|
+
export function sourcePreflightErrorMessage(stderr) {
|
|
182
|
+
const line = (stderr ?? "")
|
|
183
|
+
.split("\n")
|
|
184
|
+
.map((s) => s.trim())
|
|
185
|
+
.find((s) => s.length > 0);
|
|
186
|
+
return line
|
|
187
|
+
? line.replace(/^psql:\s*(error:\s*)?/i, "")
|
|
188
|
+
: "Could not connect to the source database.";
|
|
189
|
+
}
|
|
190
|
+
/** Default target project name = the source database name, else the URI path. */
|
|
191
|
+
export function defaultProjectName(preflightDb, sourceUri) {
|
|
192
|
+
if (preflightDb && preflightDb.trim())
|
|
193
|
+
return preflightDb.trim();
|
|
194
|
+
try {
|
|
195
|
+
const p = new URL(sourceUri).pathname.replace(/^\//, "");
|
|
196
|
+
if (p)
|
|
197
|
+
return p;
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
/* not a parseable URL — fall through */
|
|
201
|
+
}
|
|
202
|
+
return "imported-db";
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Compare two name→value maps (table row counts, or sequence last_values) and
|
|
206
|
+
* return only the mismatches. A name present on one side but not the other is a
|
|
207
|
+
* mismatch too (rendered as ∅). Deterministically sorted by name.
|
|
208
|
+
*/
|
|
209
|
+
export function diffCounts(source, target) {
|
|
210
|
+
const names = new Set([...source.keys(), ...target.keys()]);
|
|
211
|
+
const rows = [];
|
|
212
|
+
for (const name of [...names].sort()) {
|
|
213
|
+
const s = source.get(name) ?? "∅";
|
|
214
|
+
const t = target.get(name) ?? "∅";
|
|
215
|
+
if (s !== t)
|
|
216
|
+
rows.push({ name, source: s, target: t });
|
|
217
|
+
}
|
|
218
|
+
return rows;
|
|
219
|
+
}
|
|
220
|
+
/** Build the `table()` rows for a verify mismatch report (tables + sequences). */
|
|
221
|
+
export function mismatchRows(tableRows, seqRows) {
|
|
222
|
+
return [
|
|
223
|
+
...tableRows.map((r) => [`table ${r.name}`, r.source, r.target]),
|
|
224
|
+
...seqRows.map((r) => [`seq ${r.name}`, r.source, r.target]),
|
|
225
|
+
];
|
|
226
|
+
}
|
|
227
|
+
/** Double-quote a SQL identifier (escaping embedded quotes). */
|
|
228
|
+
export function quoteIdent(id) {
|
|
229
|
+
return `"${id.replace(/"/g, '""')}"`;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Map a child process 'close' `(code, signal)` to the numeric exit used for the
|
|
233
|
+
* migration verdict. A signal death (`code === null`, e.g. OOM-killer or an
|
|
234
|
+
* operator `kill`) is a FAILURE — it must never read as success just because its
|
|
235
|
+
* peer exited cleanly. The one exception: when WE deliberately SIGTERM'd the dump
|
|
236
|
+
* after a restore failure (`intentionalKill`), that path already has a nonzero
|
|
237
|
+
* restoreCode so the overall verdict is false regardless; we normalize to 0 to
|
|
238
|
+
* avoid a redundant, confusing error line.
|
|
239
|
+
*/
|
|
240
|
+
export function resolveChildExit(code, signal, opts = {}) {
|
|
241
|
+
if (code !== null)
|
|
242
|
+
return { code };
|
|
243
|
+
if (opts.intentionalKill)
|
|
244
|
+
return { code: 0 };
|
|
245
|
+
return { code: -1, error: `terminated by signal ${signal ?? "(unknown)"}` };
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Source extensions the target can't satisfy = source extensions not in the
|
|
249
|
+
* target's available set. `plpgsql` is built into every PostgreSQL and always
|
|
250
|
+
* present, so it's excluded — it can never be a real gap even though it shows up
|
|
251
|
+
* in pg_extension. Used both for the pre-restore availability gate (against
|
|
252
|
+
* `pg_available_extensions`) and the residual post-restore warning (against the
|
|
253
|
+
* target's installed `pg_extension`).
|
|
254
|
+
*/
|
|
255
|
+
export function missingExtensions(sourceExts, targetHasExts) {
|
|
256
|
+
const has = new Set(targetHasExts);
|
|
257
|
+
return sourceExts.filter((e) => e !== "plpgsql" && !has.has(e));
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* Objects BataDB's provisioning seeds into EVERY new project — platform furniture,
|
|
261
|
+
* not user data. Because a freshly created target ALWAYS contains these, they must
|
|
262
|
+
* be excluded from all TARGET-side "unexpected object" accounting: the
|
|
263
|
+
* created-project hard-fail, the existing-target leftover warnings/JSON, and the
|
|
264
|
+
* `--yes` non-empty guard (a target holding ONLY these is effectively empty). They
|
|
265
|
+
* are NEVER excluded on the SOURCE side — a source `public.health_check` (or a
|
|
266
|
+
* source `neon_migration` schema, e.g. a migration FROM another BataDB) is treated
|
|
267
|
+
* as real data and must verify normally.
|
|
268
|
+
*
|
|
269
|
+
* Two flavours:
|
|
270
|
+
* - Named objects: `public.health_check` (+ its sequence) — the data-path
|
|
271
|
+
* health-check row/table seeded on provision.
|
|
272
|
+
* - Whole schemas: `neon_migration` — our Neon-derived compute engine's
|
|
273
|
+
* compute_ctl records applied internal SQL migrations here (table
|
|
274
|
+
* `neon_migration.migration_id`) on EVERY compute startup. Real Neon hides this
|
|
275
|
+
* schema from customer `pg_tables`; our compute exposes it, so ANY object under
|
|
276
|
+
* `neon_migration.*` is platform furniture, never user data.
|
|
277
|
+
*/
|
|
278
|
+
export const PLATFORM_SCAFFOLDING_TABLES = ["public.health_check"];
|
|
279
|
+
export const PLATFORM_SCAFFOLDING_SEQUENCES = ["public.health_check_id_seq"];
|
|
280
|
+
/** Schemas whose ENTIRE contents are platform furniture (any `<schema>.*` object). */
|
|
281
|
+
export const PLATFORM_SCAFFOLDING_SCHEMAS = ["neon_migration"];
|
|
282
|
+
let PLATFORM_SCAFFOLDING_NAMES = new Set([
|
|
283
|
+
...PLATFORM_SCAFFOLDING_TABLES,
|
|
284
|
+
...PLATFORM_SCAFFOLDING_SEQUENCES,
|
|
285
|
+
]);
|
|
286
|
+
let PLATFORM_SCAFFOLDING_SCHEMA_PREFIXES = PLATFORM_SCAFFOLDING_SCHEMAS.map((s) => `${s}.`);
|
|
287
|
+
/**
|
|
288
|
+
* Merge the server-published scaffolding list (GET /v1/platform/import-scaffolding)
|
|
289
|
+
* into the baked-in defaults above. The server is the source of truth going
|
|
290
|
+
* forward — if provisioning grows new furniture, an already-installed CLI keeps
|
|
291
|
+
* verifying correctly instead of false-failing on it. Union (never replace): the
|
|
292
|
+
* baked defaults still apply against an older control plane, and a partial or
|
|
293
|
+
* malformed server response can only ever ADD exclusions it explicitly names.
|
|
294
|
+
* The exported const arrays are extended in place so every existing reference
|
|
295
|
+
* (the --yes guard, the created-target check, verify) sees the merged lists.
|
|
296
|
+
*/
|
|
297
|
+
export function applyServerScaffolding(server) {
|
|
298
|
+
const merge = (into, from) => {
|
|
299
|
+
if (!Array.isArray(from))
|
|
300
|
+
return;
|
|
301
|
+
for (const v of from) {
|
|
302
|
+
if (typeof v === "string" && v.length > 0 && !into.includes(v))
|
|
303
|
+
into.push(v);
|
|
304
|
+
}
|
|
305
|
+
};
|
|
306
|
+
merge(PLATFORM_SCAFFOLDING_TABLES, server.tables);
|
|
307
|
+
merge(PLATFORM_SCAFFOLDING_SEQUENCES, server.sequences);
|
|
308
|
+
merge(PLATFORM_SCAFFOLDING_SCHEMAS, server.schemas);
|
|
309
|
+
PLATFORM_SCAFFOLDING_NAMES = new Set([
|
|
310
|
+
...PLATFORM_SCAFFOLDING_TABLES,
|
|
311
|
+
...PLATFORM_SCAFFOLDING_SEQUENCES,
|
|
312
|
+
]);
|
|
313
|
+
PLATFORM_SCAFFOLDING_SCHEMA_PREFIXES = PLATFORM_SCAFFOLDING_SCHEMAS.map((s) => `${s}.`);
|
|
314
|
+
}
|
|
315
|
+
/** Is this qualified name one of BataDB's seeded platform-scaffolding objects —
|
|
316
|
+
* either an exact named object, or anything inside a scaffolding schema? */
|
|
317
|
+
export function isPlatformScaffolding(qualifiedName) {
|
|
318
|
+
if (PLATFORM_SCAFFOLDING_NAMES.has(qualifiedName))
|
|
319
|
+
return true;
|
|
320
|
+
return PLATFORM_SCAFFOLDING_SCHEMA_PREFIXES.some((p) => qualifiedName.startsWith(p));
|
|
321
|
+
}
|
|
322
|
+
/** Drop platform-scaffolding names — applied ONLY to TARGET-side object lists,
|
|
323
|
+
* never to source accounting. */
|
|
324
|
+
export function excludeScaffolding(names) {
|
|
325
|
+
return names.filter((n) => !isPlatformScaffolding(n));
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Scaffolding names that appear in a SOURCE object list — i.e. a user table that
|
|
329
|
+
* shares a name with BataDB's seeded platform table. On restore that `CREATE
|
|
330
|
+
* TABLE` clashes with the scaffolding already present on the target: with
|
|
331
|
+
* ON_ERROR_STOP the restore fails ("relation already exists" → RESTORE_FAILED),
|
|
332
|
+
* or, if the schemas happen to be compatible, rows append and the row-count verify
|
|
333
|
+
* catches the drift. Either way verify stays honest — the source name is NOT
|
|
334
|
+
* excluded from source accounting; we only warn the user up front.
|
|
335
|
+
*/
|
|
336
|
+
export function scaffoldingCollisions(sourceQualifiedNames) {
|
|
337
|
+
return sourceQualifiedNames.filter((n) => isPlatformScaffolding(n));
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Qualified names present on the TARGET but absent from the source — used for both
|
|
341
|
+
* tables and sequences. Verification only enumerates SOURCE objects, so on a
|
|
342
|
+
* `--project --yes` import into a non-empty target these are invisible to the
|
|
343
|
+
* count/value checks: they were left untouched and aren't covered by verify.
|
|
344
|
+
* Surfaced honestly (a warning for an existing target; unexpected for a freshly
|
|
345
|
+
* created one). Sorted, deduped by set.
|
|
346
|
+
*/
|
|
347
|
+
export function targetOnlyNames(sourceQualified, targetQualified) {
|
|
348
|
+
const source = new Set(sourceQualified);
|
|
349
|
+
return targetQualified.filter((t) => !source.has(t)).sort();
|
|
350
|
+
}
|
|
351
|
+
/** Single-quote a SQL string literal (escaping embedded quotes). */
|
|
352
|
+
function quoteLiteral(s) {
|
|
353
|
+
return `'${s.replace(/'/g, "''")}'`;
|
|
354
|
+
}
|
|
355
|
+
/** Split an array into fixed-size batches (used to bound verify query size). */
|
|
356
|
+
function chunk(items, size) {
|
|
357
|
+
const out = [];
|
|
358
|
+
for (let i = 0; i < items.length; i += size)
|
|
359
|
+
out.push(items.slice(i, i + size));
|
|
360
|
+
return out;
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Run a one-shot capture query with `psql -Atc`. `-X` skips ~/.psqlrc so a user's
|
|
364
|
+
* custom formatting / field separators / pager can't silently break the -Atc
|
|
365
|
+
* parsers; `-w` never prompts for a password.
|
|
366
|
+
*/
|
|
367
|
+
function runPsql(uri, sql) {
|
|
368
|
+
const res = spawnSync("psql", [uri, "-X", "-w", "-Atc", sql], {
|
|
369
|
+
encoding: "utf8",
|
|
370
|
+
env: { ...process.env, PGCONNECT_TIMEOUT: process.env.PGCONNECT_TIMEOUT ?? "10" },
|
|
371
|
+
maxBuffer: 128 * 1024 * 1024,
|
|
372
|
+
});
|
|
373
|
+
if (res.error)
|
|
374
|
+
return { code: -1, stdout: "", stderr: res.error.message };
|
|
375
|
+
return { code: res.status ?? -1, stdout: res.stdout ?? "", stderr: res.stderr ?? "" };
|
|
376
|
+
}
|
|
377
|
+
/** Is a client tool on PATH? Mirrors `bata connect`'s `which` probe. */
|
|
378
|
+
function hasTool(name) {
|
|
379
|
+
try {
|
|
380
|
+
execSync(`which ${name}`, { stdio: "ignore" });
|
|
381
|
+
return true;
|
|
382
|
+
}
|
|
383
|
+
catch {
|
|
384
|
+
return false;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
/** The local pg_dump major version, or null if it can't be determined. */
|
|
388
|
+
function pgDumpMajor() {
|
|
389
|
+
const res = spawnSync("pg_dump", ["--version"], { encoding: "utf8" });
|
|
390
|
+
if ((res.status ?? -1) !== 0 || !res.stdout)
|
|
391
|
+
return null;
|
|
392
|
+
return parsePgMajor(res.stdout);
|
|
393
|
+
}
|
|
394
|
+
/** The local psql major version, or null if it can't be determined. */
|
|
395
|
+
function psqlMajor() {
|
|
396
|
+
const res = spawnSync("psql", ["--version"], { encoding: "utf8" });
|
|
397
|
+
if ((res.status ?? -1) !== 0 || !res.stdout)
|
|
398
|
+
return null;
|
|
399
|
+
return parsePgMajor(res.stdout);
|
|
400
|
+
}
|
|
401
|
+
/** Parse `psql -At` `key|value` rows into a Map. */
|
|
402
|
+
function parseKvRows(stdout) {
|
|
403
|
+
const m = new Map();
|
|
404
|
+
for (const line of stdout.split("\n")) {
|
|
405
|
+
if (!line)
|
|
406
|
+
continue;
|
|
407
|
+
const idx = line.indexOf("|");
|
|
408
|
+
if (idx === -1)
|
|
409
|
+
continue;
|
|
410
|
+
m.set(line.slice(0, idx), line.slice(idx + 1));
|
|
411
|
+
}
|
|
412
|
+
return m;
|
|
413
|
+
}
|
|
414
|
+
function sleep(ms) {
|
|
415
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Stream `pg_dump <source>` → GUC filter → `psql <target> -v ON_ERROR_STOP=1`.
|
|
419
|
+
* Nothing is buffered whole; the dump flows chunk-by-chunk. Deliberately NOT
|
|
420
|
+
* --single-transaction: one giant transaction through a serverless proxy is
|
|
421
|
+
* fragile. Collects restore stderr (the actionable errors, e.g. a PG17-only
|
|
422
|
+
* feature) and resolves ok only when both processes exit 0.
|
|
423
|
+
*/
|
|
424
|
+
function runMigration(sourceUri, targetUri) {
|
|
425
|
+
return new Promise((resolve) => {
|
|
426
|
+
const filter = makeGucLineFilter();
|
|
427
|
+
const env = { ...process.env, PGCONNECT_TIMEOUT: process.env.PGCONNECT_TIMEOUT ?? "10" };
|
|
428
|
+
// `--encoding=UTF8` is load-bearing: the dump stream passes through JS strings
|
|
429
|
+
// here (dump.stdout is decoded as utf8 so the line filter can operate on text).
|
|
430
|
+
// Without it, a LATIN1 (or any non-UTF-8) source database's non-ASCII bytes
|
|
431
|
+
// would be replacement-mangled during that decode BEFORE reaching psql — silent
|
|
432
|
+
// data corruption that row-count verify wouldn't catch. Forcing UTF8 makes the
|
|
433
|
+
// server convert, guaranteeing a valid-UTF-8 stream; the dump carries
|
|
434
|
+
// `SET client_encoding = 'UTF8';`, which restores correctly into any target.
|
|
435
|
+
const dump = spawn("pg_dump", [sourceUri, "--no-owner", "--no-privileges", "--encoding=UTF8"], {
|
|
436
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
437
|
+
env,
|
|
438
|
+
});
|
|
439
|
+
// `-X` skips ~/.psqlrc: a user's custom settings (e.g. AUTOCOMMIT off, a
|
|
440
|
+
// different ON_ERROR_ROLLBACK, output tweaks) must not change restore
|
|
441
|
+
// semantics. `-w` never prompts for a password.
|
|
442
|
+
const restore = spawn("psql", [targetUri, "-X", "-w", "-v", "ON_ERROR_STOP=1"], {
|
|
443
|
+
stdio: ["pipe", "ignore", "pipe"],
|
|
444
|
+
env,
|
|
445
|
+
});
|
|
446
|
+
let restoreAlive = true;
|
|
447
|
+
const restoreErr = [];
|
|
448
|
+
let dumpErr = "";
|
|
449
|
+
let dumpCode = null;
|
|
450
|
+
let restoreCode = null;
|
|
451
|
+
let settled = false;
|
|
452
|
+
// Set true exactly where WE SIGTERM the dump after a restore failure, so the
|
|
453
|
+
// dump 'close' handler can tell our deliberate kill from an external one.
|
|
454
|
+
let intentionalDumpKill = false;
|
|
455
|
+
// Backpressure contract: psql can restore slower than pg_dump produces
|
|
456
|
+
// (index builds, TOAST writes, fsync). If we ignored write()'s return value,
|
|
457
|
+
// an unbounded backlog would buffer the ENTIRE dump in CLI memory and OOM on
|
|
458
|
+
// a multi-GB source. So when psql's stdin buffer is full (write() === false),
|
|
459
|
+
// pause the dump and resume only on 'drain'. This caps memory at ~one dump
|
|
460
|
+
// chunk plus psql's own buffer.
|
|
461
|
+
const safeWrite = (s) => {
|
|
462
|
+
if (!restoreAlive || !s)
|
|
463
|
+
return;
|
|
464
|
+
try {
|
|
465
|
+
if (!restore.stdin.write(s))
|
|
466
|
+
dump.stdout.pause();
|
|
467
|
+
}
|
|
468
|
+
catch {
|
|
469
|
+
/* psql closed its stdin (early exit) — stop feeding it */
|
|
470
|
+
}
|
|
471
|
+
};
|
|
472
|
+
restore.stdin.on("drain", () => {
|
|
473
|
+
if (restoreAlive)
|
|
474
|
+
dump.stdout.resume();
|
|
475
|
+
});
|
|
476
|
+
dump.stdout.setEncoding("utf8");
|
|
477
|
+
dump.stdout.on("data", (chunk) => safeWrite(filter.push(chunk)));
|
|
478
|
+
dump.stdout.on("end", () => {
|
|
479
|
+
safeWrite(filter.flush());
|
|
480
|
+
if (restoreAlive) {
|
|
481
|
+
try {
|
|
482
|
+
restore.stdin.end();
|
|
483
|
+
}
|
|
484
|
+
catch {
|
|
485
|
+
/* already closed */
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
});
|
|
489
|
+
dump.stderr.setEncoding("utf8");
|
|
490
|
+
dump.stderr.on("data", (d) => {
|
|
491
|
+
dumpErr += d;
|
|
492
|
+
});
|
|
493
|
+
restore.stderr.setEncoding("utf8");
|
|
494
|
+
restore.stderr.on("data", (d) => {
|
|
495
|
+
for (const line of d.split("\n")) {
|
|
496
|
+
const t = line.trim();
|
|
497
|
+
if (t)
|
|
498
|
+
restoreErr.push(t);
|
|
499
|
+
}
|
|
500
|
+
});
|
|
501
|
+
restore.stdin.on("error", () => {
|
|
502
|
+
/* EPIPE when psql exits before we finish writing — expected on failure */
|
|
503
|
+
});
|
|
504
|
+
const maybeSettle = () => {
|
|
505
|
+
if (settled || dumpCode === null || restoreCode === null)
|
|
506
|
+
return;
|
|
507
|
+
settled = true;
|
|
508
|
+
const ok = dumpCode === 0 && restoreCode === 0;
|
|
509
|
+
const errors = restoreErr.slice(0, 12);
|
|
510
|
+
if (dumpCode !== 0 && dumpErr.trim()) {
|
|
511
|
+
errors.push(...dumpErr
|
|
512
|
+
.trim()
|
|
513
|
+
.split("\n")
|
|
514
|
+
.map((s) => s.trim())
|
|
515
|
+
.filter(Boolean)
|
|
516
|
+
.slice(0, 4));
|
|
517
|
+
}
|
|
518
|
+
resolve({ ok, errors });
|
|
519
|
+
};
|
|
520
|
+
dump.on("error", (e) => {
|
|
521
|
+
dumpErr += `pg_dump failed to start: ${e.message}\n`;
|
|
522
|
+
dumpCode = dumpCode ?? -1;
|
|
523
|
+
restoreAlive = false;
|
|
524
|
+
try {
|
|
525
|
+
restore.stdin.end();
|
|
526
|
+
}
|
|
527
|
+
catch {
|
|
528
|
+
/* noop */
|
|
529
|
+
}
|
|
530
|
+
maybeSettle();
|
|
531
|
+
});
|
|
532
|
+
restore.on("error", (e) => {
|
|
533
|
+
restoreErr.push(`psql failed to start: ${e.message}`);
|
|
534
|
+
restoreCode = restoreCode ?? -1;
|
|
535
|
+
maybeSettle();
|
|
536
|
+
});
|
|
537
|
+
dump.on("close", (code, signal) => {
|
|
538
|
+
// A signal death (code === null) is a FAILURE — an OOM-killed or externally
|
|
539
|
+
// terminated pg_dump must not read as success just because psql exited
|
|
540
|
+
// cleanly. The ONLY exception is our own deliberate SIGTERM after a restore
|
|
541
|
+
// failure: restoreCode is already nonzero there so `ok` stays false anyway,
|
|
542
|
+
// and we normalize to 0 to avoid a redundant error line. On restore SUCCESS
|
|
543
|
+
// we never kill the dump — it closes naturally and its REAL exit code is
|
|
544
|
+
// honored here, so a late pg_dump failure can't be masked into success.
|
|
545
|
+
const r = resolveChildExit(code, signal, { intentionalKill: intentionalDumpKill });
|
|
546
|
+
if (r.error)
|
|
547
|
+
dumpErr += `pg_dump ${r.error}\n`;
|
|
548
|
+
dumpCode = r.code;
|
|
549
|
+
maybeSettle();
|
|
550
|
+
});
|
|
551
|
+
restore.on("close", (code, signal) => {
|
|
552
|
+
restoreAlive = false;
|
|
553
|
+
// A signal death is likewise a failure (never success on an external kill).
|
|
554
|
+
const r = resolveChildExit(code, signal);
|
|
555
|
+
if (r.error)
|
|
556
|
+
restoreErr.push(`psql ${r.error}`);
|
|
557
|
+
restoreCode = r.code;
|
|
558
|
+
// Kill the dump ONLY when the restore FAILED: a failed psql means no 'drain'
|
|
559
|
+
// will ever come, so a backpressure-paused pg_dump would hang forever and
|
|
560
|
+
// maybeSettle() would wait on dumpCode indefinitely — and its remaining
|
|
561
|
+
// output is useless anyway. On restore SUCCESS we deliberately do NOT kill:
|
|
562
|
+
// let pg_dump finish and report its real exit code, so maybeSettle (which
|
|
563
|
+
// waits for both) catches a late dump failure instead of hiding it.
|
|
564
|
+
if (restoreCode !== 0) {
|
|
565
|
+
intentionalDumpKill = true;
|
|
566
|
+
try {
|
|
567
|
+
dump.kill("SIGTERM");
|
|
568
|
+
}
|
|
569
|
+
catch {
|
|
570
|
+
/* already gone */
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
maybeSettle();
|
|
574
|
+
});
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
/**
|
|
578
|
+
* Delete a project we created for this import (DELETE /v1/projects/:id — the same
|
|
579
|
+
* endpoint `bata projects delete` uses). Returns whether it was actually removed,
|
|
580
|
+
* so the caller can tell the user to clean up manually if it wasn't. Only ever
|
|
581
|
+
* called for a project THIS run created — never a pre-existing --project target.
|
|
582
|
+
*/
|
|
583
|
+
async function deleteCreatedProject(projectId, token, teamId) {
|
|
584
|
+
const query = {};
|
|
585
|
+
if (teamId)
|
|
586
|
+
query.team_id = teamId;
|
|
587
|
+
try {
|
|
588
|
+
const res = await api.del(`/v1/projects/${projectId}`, token, query);
|
|
589
|
+
return res.ok;
|
|
590
|
+
}
|
|
591
|
+
catch {
|
|
592
|
+
return false;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
/** Fetch the primary branch's usable (password-revealed) direct URI — the same
|
|
596
|
+
* path `bata connect` / `bata db url` use. Never fabricated locally. */
|
|
597
|
+
async function getTargetUri(projectId, token) {
|
|
598
|
+
const res = await api.get(`/v1/connection-info/${projectId}`, token, { reveal: "true" });
|
|
599
|
+
if (!res.ok)
|
|
600
|
+
return null;
|
|
601
|
+
const conns = res.data?.connections ?? [];
|
|
602
|
+
const c = conns.find((x) => x.is_primary) ?? conns[0];
|
|
603
|
+
return c?.direct ?? null;
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Poll GET /v1/branches?project_id= until the primary compute is ready. Cold
|
|
607
|
+
* attach can take 40s+, so the budget is patient (~180s). Falls back to a real
|
|
608
|
+
* `SELECT 1` probe once past the warm-up window, since a working query is the
|
|
609
|
+
* true readiness signal even if the `ready` flag lags.
|
|
610
|
+
*/
|
|
611
|
+
async function waitForReady(projectId, targetUri, token, teamId, onTick) {
|
|
612
|
+
const budget = 180_000;
|
|
613
|
+
const interval = 3_000;
|
|
614
|
+
const probeAfter = 30_000;
|
|
615
|
+
const start = Date.now();
|
|
616
|
+
while (Date.now() - start < budget) {
|
|
617
|
+
const query = { project_id: projectId };
|
|
618
|
+
if (teamId)
|
|
619
|
+
query.team_id = teamId;
|
|
620
|
+
const res = await api.get("/v1/branches", token, query);
|
|
621
|
+
if (res.ok) {
|
|
622
|
+
const branches = asList(res.data);
|
|
623
|
+
const primary = branches.find((b) => b.isPrimary ?? b.is_primary) ?? branches[0];
|
|
624
|
+
if (primary?.ready === true)
|
|
625
|
+
return true;
|
|
626
|
+
}
|
|
627
|
+
if (Date.now() - start > probeAfter && runPsql(targetUri, "SELECT 1").code === 0) {
|
|
628
|
+
return true;
|
|
629
|
+
}
|
|
630
|
+
onTick(Math.round((Date.now() - start) / 1000));
|
|
631
|
+
await sleep(interval);
|
|
632
|
+
}
|
|
633
|
+
// Last chance: a successful query means ready regardless of the flag.
|
|
634
|
+
return runPsql(targetUri, "SELECT 1").code === 0;
|
|
635
|
+
}
|
|
636
|
+
// ── Command ──────────────────────────────────────────────────────────────────
|
|
637
|
+
export function parseImportArgs(args) {
|
|
638
|
+
let source;
|
|
639
|
+
let projectId;
|
|
640
|
+
let name;
|
|
641
|
+
let pg;
|
|
642
|
+
let help = false;
|
|
643
|
+
for (let i = 0; i < args.length; i++) {
|
|
644
|
+
const a = args[i];
|
|
645
|
+
const takeVal = () => {
|
|
646
|
+
const next = args[i + 1];
|
|
647
|
+
if (next === undefined || next.startsWith("-"))
|
|
648
|
+
return undefined;
|
|
649
|
+
i++;
|
|
650
|
+
return next;
|
|
651
|
+
};
|
|
652
|
+
if (a === "--help" || a === "-h")
|
|
653
|
+
help = true;
|
|
654
|
+
else if (a === "--source") {
|
|
655
|
+
const v = takeVal();
|
|
656
|
+
if (v === undefined)
|
|
657
|
+
return { error: "--source requires a value (a postgres:// connection URI)." };
|
|
658
|
+
source = v;
|
|
659
|
+
}
|
|
660
|
+
else if (a.startsWith("--source="))
|
|
661
|
+
source = a.slice("--source=".length);
|
|
662
|
+
else if (a === "--project") {
|
|
663
|
+
const v = takeVal();
|
|
664
|
+
if (v === undefined)
|
|
665
|
+
return { error: "--project requires a value (a project id)." };
|
|
666
|
+
projectId = v;
|
|
667
|
+
}
|
|
668
|
+
else if (a.startsWith("--project="))
|
|
669
|
+
projectId = a.slice("--project=".length);
|
|
670
|
+
else if (a === "--name") {
|
|
671
|
+
const v = takeVal();
|
|
672
|
+
if (v === undefined)
|
|
673
|
+
return { error: "--name requires a value." };
|
|
674
|
+
name = v;
|
|
675
|
+
}
|
|
676
|
+
else if (a.startsWith("--name="))
|
|
677
|
+
name = a.slice("--name=".length);
|
|
678
|
+
else if (a === "--pg") {
|
|
679
|
+
const v = takeVal();
|
|
680
|
+
if (v === undefined)
|
|
681
|
+
return { error: "--pg requires a value (a PostgreSQL major, e.g. 17)." };
|
|
682
|
+
pg = v;
|
|
683
|
+
}
|
|
684
|
+
else if (a.startsWith("--pg="))
|
|
685
|
+
pg = a.slice("--pg=".length);
|
|
686
|
+
// Unknown tokens / positionals are ignored — --source is the only input.
|
|
687
|
+
}
|
|
688
|
+
if (pg !== undefined && !/^\d+$/.test(pg)) {
|
|
689
|
+
return { error: `--pg must be a PostgreSQL major version number (got "${pg}").` };
|
|
690
|
+
}
|
|
691
|
+
// --pg picks the version of a NEW project; an existing project's version is fixed.
|
|
692
|
+
if (pg !== undefined && projectId !== undefined && !help) {
|
|
693
|
+
return { error: "--pg only applies when creating a new project — it cannot change an existing --project's version." };
|
|
694
|
+
}
|
|
695
|
+
// --project (import into an existing project) and --name (create a new one) are
|
|
696
|
+
// opposite intents. Silently taking --project was the riskier read (mutating an
|
|
697
|
+
// existing project while ignoring the requested name) — make it an explicit error.
|
|
698
|
+
if (projectId !== undefined && name !== undefined && !help) {
|
|
699
|
+
return {
|
|
700
|
+
error: "--project and --name are mutually exclusive (import into an existing project OR create a new one).",
|
|
701
|
+
};
|
|
702
|
+
}
|
|
703
|
+
return { source, projectId, name, pg, help };
|
|
704
|
+
}
|
|
705
|
+
/** Highest PostgreSQL major BataDB can create today — used only to pick the
|
|
706
|
+
* DEFAULT version for a new import target (min(source major, this), floored at
|
|
707
|
+
* 16). An explicit --pg is passed through as-is and validated by the API's
|
|
708
|
+
* allowlist, so a newer server accepts newer majors without a CLI release. */
|
|
709
|
+
const HIGHEST_SUPPORTED_PG = 17;
|
|
710
|
+
/** Default target major for a new project: match the source so features survive,
|
|
711
|
+
* capped at what BataDB ships (a PG18 source → 17, a PG16 or older source → 16). */
|
|
712
|
+
export function defaultTargetPg(sourceMajor) {
|
|
713
|
+
return String(Math.max(16, Math.min(sourceMajor, HIGHEST_SUPPORTED_PG)));
|
|
714
|
+
}
|
|
715
|
+
function printHelp() {
|
|
716
|
+
log();
|
|
717
|
+
log(` ${colors.bold("bata import")} ${colors.dim("--source <postgres-uri> [--project <id> | --name <name>] [--pg <major>]")}`);
|
|
718
|
+
log();
|
|
719
|
+
log(` Migrate a Neon (or any Postgres) database into BataDB via pg_dump | psql.`);
|
|
720
|
+
log();
|
|
721
|
+
log(` ${colors.bold("Options")}`);
|
|
722
|
+
log(` ${colors.dim("--source <uri>")} Source Postgres connection URI ${colors.dim("(required)")}`);
|
|
723
|
+
log(` ${colors.dim("--project <id>")} Import into an existing BataDB project`);
|
|
724
|
+
log(` ${colors.dim("--name <name>")} Create a new project with this name ${colors.dim("(default: source db name)")}`);
|
|
725
|
+
log(` ${colors.dim("--pg <major>")} PostgreSQL major for a NEW project ${colors.dim("(default: match the source, capped at 17)")}`);
|
|
726
|
+
log(` ${colors.dim("--yes, -y")} Proceed even if the target already has tables`);
|
|
727
|
+
log(` ${colors.dim("--json")} Machine-readable result`);
|
|
728
|
+
log();
|
|
729
|
+
log(` ${colors.bold("Prerequisites")}`);
|
|
730
|
+
log(` Local ${colors.cyan("pg_dump")} + ${colors.cyan("psql")} whose major ≥ the source server's.`);
|
|
731
|
+
log(` ${colors.dim("macOS:")} ${colors.cyan("brew install libpq")} ${colors.dim("Ubuntu:")} ${colors.cyan("sudo apt install postgresql-client-17")}`);
|
|
732
|
+
log();
|
|
733
|
+
log(` ${colors.bold("Examples")}`);
|
|
734
|
+
log(` ${colors.cyan('bata import --source "postgresql://…@ep-x.neon.tech/neondb"')}`);
|
|
735
|
+
log(` ${colors.cyan('bata import --source "$NEON_URL" --name my-app --yes --json')}`);
|
|
736
|
+
log();
|
|
737
|
+
log(` ${colors.dim("BataDB runs PostgreSQL 16 and 17. A new target defaults to the source's")}`);
|
|
738
|
+
log(` ${colors.dim("major (capped at 17); features newer than the target fail loudly during")}`);
|
|
739
|
+
log(` ${colors.dim("restore (nothing is hidden).")}`);
|
|
740
|
+
log();
|
|
741
|
+
}
|
|
742
|
+
export async function importDb(args) {
|
|
743
|
+
const parsed = parseImportArgs(args);
|
|
744
|
+
if ("error" in parsed) {
|
|
745
|
+
emitError("INVALID_FLAG", parsed.error, "Run `bata import --help` for usage.");
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
if (parsed.help) {
|
|
749
|
+
printHelp();
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
if (!parsed.source) {
|
|
753
|
+
emitError("MISSING_ARG", "A source database is required.", "Pass --source <postgres-uri>. Run `bata import --help` for usage.");
|
|
754
|
+
return;
|
|
755
|
+
}
|
|
756
|
+
const jsonMode = isJsonMode();
|
|
757
|
+
const token = requireToken();
|
|
758
|
+
const config = loadConfig();
|
|
759
|
+
const source = parsed.source;
|
|
760
|
+
const phase = (msg) => {
|
|
761
|
+
if (!jsonMode)
|
|
762
|
+
log(` ${colors.cyan("›")} ${msg}`);
|
|
763
|
+
};
|
|
764
|
+
if (!jsonMode)
|
|
765
|
+
heading("Import into BataDB");
|
|
766
|
+
// Ask the control plane for the current platform-scaffolding list (tables/
|
|
767
|
+
// sequences/schemas provisioning seeds into every project) and merge it into
|
|
768
|
+
// the baked-in defaults, so every downstream consumer — the source-collision
|
|
769
|
+
// warning, the --yes non-empty guard, and verification — keeps excluding the
|
|
770
|
+
// platform's own furniture even when it grows after this CLI was installed.
|
|
771
|
+
// Best-effort: any failure — older server (404), network error, or a transport
|
|
772
|
+
// THROW (timeout/DNS) — silently keeps the defaults; it must never abort the run.
|
|
773
|
+
try {
|
|
774
|
+
const scaffRes = await api.get("/v1/platform/import-scaffolding", token);
|
|
775
|
+
if (scaffRes.ok && scaffRes.data && typeof scaffRes.data === "object") {
|
|
776
|
+
applyServerScaffolding(scaffRes.data);
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
catch {
|
|
780
|
+
// baked-in defaults stay in effect
|
|
781
|
+
}
|
|
782
|
+
// The client tools are needed to even inspect the source, so verify they're on
|
|
783
|
+
// PATH up front (the pg_dump-major-vs-source-major gate stays in phase 2, once
|
|
784
|
+
// the source major is known).
|
|
785
|
+
if (!hasTool("pg_dump") || !hasTool("psql")) {
|
|
786
|
+
emitError("TOOLS_MISSING", "pg_dump and psql are required but were not found on PATH.", "macOS: brew install libpq · Ubuntu: sudo apt install postgresql-client", 1);
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
// ── Phase 1: preflight the source ──────────────────────────────────────────
|
|
790
|
+
phase("Inspecting source database");
|
|
791
|
+
const summarySql = "SELECT json_build_object(" +
|
|
792
|
+
"'database', current_database()," +
|
|
793
|
+
"'server_version', current_setting('server_version')," +
|
|
794
|
+
"'server_version_num', current_setting('server_version_num')," +
|
|
795
|
+
"'size', pg_size_pretty(pg_database_size(current_database()))," +
|
|
796
|
+
"'tables', (SELECT count(*) FROM pg_tables WHERE schemaname NOT IN ('pg_catalog','information_schema'))" +
|
|
797
|
+
")";
|
|
798
|
+
const srcRes = runPsql(source, summarySql);
|
|
799
|
+
if (srcRes.code !== 0) {
|
|
800
|
+
emitError("SOURCE_UNREACHABLE", `Could not read the source database: ${sourcePreflightErrorMessage(srcRes.stderr)}`, "Check the --source URI (host, credentials, sslmode) and that the server is reachable.", 1);
|
|
801
|
+
return;
|
|
802
|
+
}
|
|
803
|
+
let sourceSummary;
|
|
804
|
+
try {
|
|
805
|
+
sourceSummary = JSON.parse(srcRes.stdout.trim());
|
|
806
|
+
}
|
|
807
|
+
catch {
|
|
808
|
+
emitError("SOURCE_UNREACHABLE", "Unexpected response from the source database.", "", 1);
|
|
809
|
+
return;
|
|
810
|
+
}
|
|
811
|
+
const extRes = runPsql(source, "SELECT extname FROM pg_extension ORDER BY extname");
|
|
812
|
+
const sourceExtensions = extRes.code === 0
|
|
813
|
+
? extRes.stdout.split("\n").map((s) => s.trim()).filter(Boolean)
|
|
814
|
+
: [];
|
|
815
|
+
const sourceMajor = parsePgMajor(sourceSummary.server_version_num || sourceSummary.server_version);
|
|
816
|
+
phase(`Source: ${sourceSummary.database} · PostgreSQL ${sourceSummary.server_version} · ` +
|
|
817
|
+
`${sourceSummary.size} · ${sourceSummary.tables} tables`);
|
|
818
|
+
// Warn up front if the SOURCE contains anything sharing a name with BataDB's
|
|
819
|
+
// seeded platform scaffolding — the named public.health_check table, or ANY
|
|
820
|
+
// object in a scaffolding schema (neon_migration), e.g. when migrating FROM
|
|
821
|
+
// another BataDB. Its CREATE will clash with the furniture the target is
|
|
822
|
+
// provisioned with, so the restore will likely fail later with "relation already
|
|
823
|
+
// exists" — flagging it now makes that error unsurprising. We do NOT touch source
|
|
824
|
+
// accounting; a source health_check / neon_migration object is treated as real
|
|
825
|
+
// data and verifies normally if it does restore.
|
|
826
|
+
const scaffoldingBareTables = PLATFORM_SCAFFOLDING_TABLES.filter((n) => n.startsWith("public."))
|
|
827
|
+
.map((n) => quoteLiteral(n.slice("public.".length)))
|
|
828
|
+
.join(",");
|
|
829
|
+
const scaffoldingSchemaLits = PLATFORM_SCAFFOLDING_SCHEMAS.map((s) => quoteLiteral(s)).join(",");
|
|
830
|
+
const srcCollisionRes = runPsql(source, "SELECT schemaname||'.'||tablename FROM pg_tables " +
|
|
831
|
+
`WHERE (schemaname = 'public' AND tablename IN (${scaffoldingBareTables})) ` +
|
|
832
|
+
`OR schemaname IN (${scaffoldingSchemaLits}) ORDER BY 1`);
|
|
833
|
+
const sourceScaffoldingHits = srcCollisionRes.code === 0
|
|
834
|
+
? scaffoldingCollisions(srcCollisionRes.stdout.split("\n").map((s) => s.trim()).filter(Boolean))
|
|
835
|
+
: [];
|
|
836
|
+
if (sourceScaffoldingHits.length > 0 && !jsonMode) {
|
|
837
|
+
warn(`Source contains ${sourceScaffoldingHits.join(", ")}, which shares a name with BataDB's seeded ` +
|
|
838
|
+
'platform scaffolding on the target. The restore will likely fail with "relation already ' +
|
|
839
|
+
'exists" — rename or exclude it on the source to migrate cleanly.');
|
|
840
|
+
}
|
|
841
|
+
// ── Phase 2: check pg_dump AND psql are new enough ──────────────────────────
|
|
842
|
+
// BOTH matter: pg_dump can't dump a newer server than itself, and psql must be
|
|
843
|
+
// able to restore the stream THIS pg_dump produces — pg_dump 17 emits
|
|
844
|
+
// \restrict/\unrestrict regardless of the source version, so psql 16 + pg_dump
|
|
845
|
+
// 17 fails even against a PG16 source. checkClientVersions() encodes the full
|
|
846
|
+
// rule (psql ≥ max(source, pg_dump)); without it the restore would fail late,
|
|
847
|
+
// after a target was already created and warmed.
|
|
848
|
+
phase("Checking pg_dump / psql versions");
|
|
849
|
+
const dumpMajor = pgDumpMajor();
|
|
850
|
+
if (dumpMajor === null) {
|
|
851
|
+
emitError("TOOLS_MISSING", "Could not determine the pg_dump version.", "", 1);
|
|
852
|
+
return;
|
|
853
|
+
}
|
|
854
|
+
const psqlVer = psqlMajor();
|
|
855
|
+
if (psqlVer === null) {
|
|
856
|
+
emitError("TOOLS_MISSING", "Could not determine the psql version.", "", 1);
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
859
|
+
const versionVerdict = checkClientVersions(dumpMajor, psqlVer, sourceMajor);
|
|
860
|
+
if (!versionVerdict.ok) {
|
|
861
|
+
emitError("PG_VERSION_TOO_OLD", versionVerdict.message, "macOS: brew install libpq · Ubuntu: sudo apt install postgresql-client-" + versionVerdict.needMajor, 1);
|
|
862
|
+
return;
|
|
863
|
+
}
|
|
864
|
+
// ── Phase 3: resolve / create the target ───────────────────────────────────
|
|
865
|
+
const teamId = await resolveTeamId(token);
|
|
866
|
+
let targetProjectId;
|
|
867
|
+
let created = false;
|
|
868
|
+
// The target's PostgreSQL major (for messaging + the JSON result). For a new
|
|
869
|
+
// project this is what we asked for; for --project it's read from the record.
|
|
870
|
+
let targetPg;
|
|
871
|
+
if (parsed.projectId) {
|
|
872
|
+
phase(`Resolving target project ${parsed.projectId}`);
|
|
873
|
+
const q = {};
|
|
874
|
+
if (teamId)
|
|
875
|
+
q.team_id = teamId;
|
|
876
|
+
const projRes = await api.get(`/v1/projects/${parsed.projectId}`, token, q);
|
|
877
|
+
if (!projRes.ok) {
|
|
878
|
+
emitError(projRes.status === 404 ? "NOT_FOUND"
|
|
879
|
+
: projRes.status === 401 || projRes.status === 403 ? "INVALID_KEY"
|
|
880
|
+
: projRes.status >= 500 || projRes.status === 0 ? "API_UNAVAILABLE"
|
|
881
|
+
: "CLI_ERROR", apiError(projRes, "Target project not found"), "");
|
|
882
|
+
return;
|
|
883
|
+
}
|
|
884
|
+
targetProjectId = parsed.projectId;
|
|
885
|
+
// Older control planes may omit pgVersion; fall back to the platform's
|
|
886
|
+
// historical default rather than guessing anything newer.
|
|
887
|
+
targetPg = projRes.data.pgVersion ?? "16";
|
|
888
|
+
}
|
|
889
|
+
else {
|
|
890
|
+
const name = parsed.name ?? defaultProjectName(sourceSummary.database, source);
|
|
891
|
+
// An unparseable source version keeps the conservative historical default.
|
|
892
|
+
targetPg = parsed.pg ?? (sourceMajor === null ? "16" : defaultTargetPg(sourceMajor));
|
|
893
|
+
phase(`Creating project ${name} (serverless · PostgreSQL ${targetPg})`);
|
|
894
|
+
if (sourceMajor !== null && sourceMajor > Number(targetPg) && !jsonMode) {
|
|
895
|
+
warn(`source is PG${sourceMajor}, BataDB target is PG${targetPg} — standard schemas migrate cleanly; ` +
|
|
896
|
+
`PG${sourceMajor}-only features will fail loudly during restore`);
|
|
897
|
+
}
|
|
898
|
+
const body = {
|
|
899
|
+
name,
|
|
900
|
+
region: "us-east-1",
|
|
901
|
+
compute: { tier: "serverless", size_cu: 1 },
|
|
902
|
+
pg_version: targetPg,
|
|
903
|
+
};
|
|
904
|
+
if (teamId)
|
|
905
|
+
body.team_id = teamId;
|
|
906
|
+
const res = await api.post("/v1/projects", body, token);
|
|
907
|
+
if (!res.ok) {
|
|
908
|
+
const explicitPg = parsed.pg !== undefined;
|
|
909
|
+
emitError(res.status === 401 || res.status === 403 ? "INVALID_KEY"
|
|
910
|
+
: res.status >= 500 || res.status === 0 ? "API_UNAVAILABLE"
|
|
911
|
+
: "CLI_ERROR", apiError(res, "Failed to create target project"),
|
|
912
|
+
// A 400 on an explicit --pg is almost always an unsupported major.
|
|
913
|
+
explicitPg && res.status === 400 ? "Try --pg 16 (or omit --pg for the default)." : "");
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
targetProjectId = res.data.id;
|
|
917
|
+
created = true;
|
|
918
|
+
}
|
|
919
|
+
// Resolve the usable target URI (the deliverable) via the API — never local.
|
|
920
|
+
const targetUri = await getTargetUri(targetProjectId, token);
|
|
921
|
+
if (!targetUri) {
|
|
922
|
+
emitError("API_UNAVAILABLE", "Could not fetch the target connection string.", "", 6);
|
|
923
|
+
return;
|
|
924
|
+
}
|
|
925
|
+
phase("Waiting for target compute to be ready");
|
|
926
|
+
const ready = await waitForReady(targetProjectId, targetUri, token, teamId, (secs) => {
|
|
927
|
+
if (!jsonMode && secs > 0 && secs % 15 === 0)
|
|
928
|
+
phase(` …still waiting (${secs}s)`);
|
|
929
|
+
});
|
|
930
|
+
if (!ready) {
|
|
931
|
+
emitError("COMPUTE_STARTING", "Target compute did not become ready within 180s.", `Retry once it warms: bata status --project ${targetProjectId}`, 6);
|
|
932
|
+
return;
|
|
933
|
+
}
|
|
934
|
+
// For an existing target, ANY pre-existing user OBJECT risks a collision in a
|
|
935
|
+
// non-transactional restore — not just relations. This ONE fail-closed query
|
|
936
|
+
// counts: relations (table/partitioned/view/matview/sequence/foreign table),
|
|
937
|
+
// user-created schemas (excluding public), user types (enums, domains, and
|
|
938
|
+
// free-standing composites — NOT a table's implicit row type), and user
|
|
939
|
+
// functions (excluding extension-owned ones). If the query itself errors we must
|
|
940
|
+
// NOT treat the target as empty and silently proceed.
|
|
941
|
+
if (parsed.projectId) {
|
|
942
|
+
// BataDB seeds public.health_check (+ its sequence) into every project and our
|
|
943
|
+
// compute exposes an internal neon_migration schema, so a target holding ONLY
|
|
944
|
+
// scaffolding is effectively empty — exclude those from the counts so the --yes
|
|
945
|
+
// guard doesn't trip on our own furniture. Derived from the same constants as
|
|
946
|
+
// the rest of the scaffolding handling.
|
|
947
|
+
const scaffoldingPublicRelnames = [...PLATFORM_SCAFFOLDING_TABLES, ...PLATFORM_SCAFFOLDING_SEQUENCES]
|
|
948
|
+
.filter((n) => n.startsWith("public."))
|
|
949
|
+
.map((n) => quoteLiteral(n.slice("public.".length)))
|
|
950
|
+
.join(",");
|
|
951
|
+
const scaffoldingSchemaLiterals = PLATFORM_SCAFFOLDING_SCHEMAS.map((s) => quoteLiteral(s)).join(",");
|
|
952
|
+
const cnt = runPsql(targetUri, "SELECT" +
|
|
953
|
+
// relations, in any non-system namespace (incl. public), minus scaffolding:
|
|
954
|
+
// named public objects AND everything in a scaffolding schema.
|
|
955
|
+
" (SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace" +
|
|
956
|
+
" WHERE c.relkind IN ('r','p','v','m','S','f')" +
|
|
957
|
+
" AND n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_%'" +
|
|
958
|
+
` AND n.nspname NOT IN (${scaffoldingSchemaLiterals})` +
|
|
959
|
+
` AND NOT (n.nspname = 'public' AND c.relname IN (${scaffoldingPublicRelnames})))` +
|
|
960
|
+
// user-created schemas (public is a default, not user-created; scaffolding
|
|
961
|
+
// schemas are platform furniture)
|
|
962
|
+
" + (SELECT count(*) FROM pg_namespace n" +
|
|
963
|
+
" WHERE n.nspname NOT IN ('public','pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_%'" +
|
|
964
|
+
` AND n.nspname NOT IN (${scaffoldingSchemaLiterals}))` +
|
|
965
|
+
// user types: enums, domains, and standalone composites (relkind 'c'),
|
|
966
|
+
// but not a table's implicit row type
|
|
967
|
+
" + (SELECT count(*) FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace" +
|
|
968
|
+
" WHERE n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_%'" +
|
|
969
|
+
` AND n.nspname NOT IN (${scaffoldingSchemaLiterals})` +
|
|
970
|
+
" AND (t.typtype IN ('e','d')" +
|
|
971
|
+
" OR (t.typtype = 'c' AND EXISTS (SELECT 1 FROM pg_class cc WHERE cc.oid = t.typrelid AND cc.relkind = 'c'))))" +
|
|
972
|
+
// user functions, excluding extension-owned (pg_depend deptype 'e')
|
|
973
|
+
" + (SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace" +
|
|
974
|
+
" WHERE n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_%'" +
|
|
975
|
+
` AND n.nspname NOT IN (${scaffoldingSchemaLiterals})` +
|
|
976
|
+
" AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = p.oid AND d.classid = 'pg_proc'::regclass AND d.deptype = 'e'))" +
|
|
977
|
+
" AS n");
|
|
978
|
+
if (cnt.code !== 0) {
|
|
979
|
+
emitError("CLI_ERROR", "Could not verify whether the target project is empty before importing.", "Retry once the target compute is warm, or check the target connection.", 1);
|
|
980
|
+
return;
|
|
981
|
+
}
|
|
982
|
+
const existingObjects = parseInt(cnt.stdout.trim(), 10) || 0;
|
|
983
|
+
if (existingObjects > 0 && !isYes()) {
|
|
984
|
+
emitError("INVALID_FLAG", `Target project already has ${existingObjects} user object(s). Importing may collide with existing objects.`, "Re-run with --yes to import into it anyway.");
|
|
985
|
+
return;
|
|
986
|
+
}
|
|
987
|
+
if (existingObjects > 0 && !jsonMode) {
|
|
988
|
+
warn(`Target already has ${existingObjects} user object(s) — proceeding because --yes was set.`);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
// Extension preflight — BEFORE the dump. With ON_ERROR_STOP=1 the restore
|
|
992
|
+
// would die at the first `CREATE EXTENSION <x>;` the target can't provide, so
|
|
993
|
+
// checking availability up front lets us fail loudly with the full list
|
|
994
|
+
// instead of a cryptic mid-restore error (and a half-imported target). Fail
|
|
995
|
+
// CLOSED: if we can't even query availability, don't start a restore we know
|
|
996
|
+
// might half-apply.
|
|
997
|
+
if (sourceExtensions.length > 0) {
|
|
998
|
+
phase("Checking target extension availability");
|
|
999
|
+
// We reach this AFTER (possibly) creating the project and waiting for its
|
|
1000
|
+
// compute — so a failure here would strand an empty project. If THIS run
|
|
1001
|
+
// created it, delete it before exiting and say so; if the delete fails, tell
|
|
1002
|
+
// the user the id to clean up. A pre-existing --project target is never
|
|
1003
|
+
// touched (created === false).
|
|
1004
|
+
const failWithCleanup = async (message, baseHint) => {
|
|
1005
|
+
let hint = baseHint;
|
|
1006
|
+
if (created) {
|
|
1007
|
+
const removed = await deleteCreatedProject(targetProjectId, token, teamId);
|
|
1008
|
+
hint += removed
|
|
1009
|
+
? " Removed the empty project that was created for this import."
|
|
1010
|
+
: ` Could not remove the project created for this import — delete it manually: bata projects delete ${targetProjectId}`;
|
|
1011
|
+
}
|
|
1012
|
+
emitError("EXTENSION_UNSUPPORTED", message, hint, 1);
|
|
1013
|
+
throw new Error("unreachable"); // emitError exits; satisfy the `never` return
|
|
1014
|
+
};
|
|
1015
|
+
const availRes = runPsql(targetUri, "SELECT name FROM pg_available_extensions ORDER BY name");
|
|
1016
|
+
if (availRes.code !== 0) {
|
|
1017
|
+
await failWithCleanup("Could not verify which extensions the target supports.", "Retry once the target compute is warm — a restore isn't safe to start until this is known.");
|
|
1018
|
+
}
|
|
1019
|
+
const available = availRes.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
|
|
1020
|
+
const unsupported = missingExtensions(sourceExtensions, available);
|
|
1021
|
+
if (unsupported.length > 0) {
|
|
1022
|
+
await failWithCleanup(`The source uses extension(s) the target can't provide: ${unsupported.join(", ")}.`, "BataDB may not offer these yet. Drop them on the source (DROP EXTENSION …) before importing, or contact BataDB support to request them.");
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
// Once we're past target creation + restore start, a failure leaves a
|
|
1026
|
+
// partially-populated target. We DELIBERATELY do NOT auto-delete it here
|
|
1027
|
+
// (unlike the extension preflight, which fails before any data moves): a
|
|
1028
|
+
// partial restore can represent significant transfer time and is useful for
|
|
1029
|
+
// diagnosis and retry, so destroying it silently would be worse. Instead every
|
|
1030
|
+
// post-restore failure path NAMES the project and gives actionable next steps.
|
|
1031
|
+
const strandedTargetJson = () => ({
|
|
1032
|
+
project_id: targetProjectId,
|
|
1033
|
+
created,
|
|
1034
|
+
connection: targetUri,
|
|
1035
|
+
});
|
|
1036
|
+
const printStrandedTargetHints = () => {
|
|
1037
|
+
log(` ${colors.dim("The target project was left in place for diagnosis/retry:")}`);
|
|
1038
|
+
kvList([["Project ID", colors.dim(targetProjectId)]]);
|
|
1039
|
+
log();
|
|
1040
|
+
log(` ${colors.dim("Retry into the SAME target after fixing the issue:")}`);
|
|
1041
|
+
log(` ${colors.cyan(`bata import --source <source> --project ${targetProjectId} --yes`)}`);
|
|
1042
|
+
if (created) {
|
|
1043
|
+
log(` ${colors.dim("…or remove the partial project:")}`);
|
|
1044
|
+
log(` ${colors.cyan(`bata projects delete ${targetProjectId} --yes`)}`);
|
|
1045
|
+
}
|
|
1046
|
+
log();
|
|
1047
|
+
};
|
|
1048
|
+
const strandedHintJson = "The target project was left in place (NOT deleted) for diagnosis/retry — re-run with " +
|
|
1049
|
+
`--project ${targetProjectId} --yes after fixing` +
|
|
1050
|
+
(created ? `, or remove it with: bata projects delete ${targetProjectId} --yes.` : ".");
|
|
1051
|
+
// ── Phase 4: migrate ───────────────────────────────────────────────────────
|
|
1052
|
+
phase("Migrating schema + data (pg_dump | psql)");
|
|
1053
|
+
const migration = await runMigration(source, targetUri);
|
|
1054
|
+
if (!migration.ok) {
|
|
1055
|
+
if (jsonMode) {
|
|
1056
|
+
json({
|
|
1057
|
+
error: "Restore failed — the target rejected part of the dump.",
|
|
1058
|
+
code: "RESTORE_FAILED",
|
|
1059
|
+
hint: `Schema features newer than the target's PostgreSQL ${targetPg} aren't supported there. ` +
|
|
1060
|
+
"Review the errors below. " +
|
|
1061
|
+
strandedHintJson,
|
|
1062
|
+
errors: migration.errors,
|
|
1063
|
+
target: strandedTargetJson(),
|
|
1064
|
+
});
|
|
1065
|
+
}
|
|
1066
|
+
else {
|
|
1067
|
+
log();
|
|
1068
|
+
emitErrorHuman("Restore failed — the target rejected part of the dump.", migration.errors);
|
|
1069
|
+
printStrandedTargetHints();
|
|
1070
|
+
}
|
|
1071
|
+
process.exit(1);
|
|
1072
|
+
return;
|
|
1073
|
+
}
|
|
1074
|
+
// ── Phase 5: verify ────────────────────────────────────────────────────────
|
|
1075
|
+
// Fail CLOSED: a verify query that can't run must never read as "no mismatch"
|
|
1076
|
+
// (that would report a false success). Any failed query, or an unexpectedly
|
|
1077
|
+
// empty result for a non-empty input, is a hard failure that tells the user
|
|
1078
|
+
// verification couldn't run — never silent success.
|
|
1079
|
+
phase("Verifying row counts + sequences");
|
|
1080
|
+
let verifyFailed = null;
|
|
1081
|
+
const tableListRes = runPsql(source, "SELECT COALESCE(json_agg(json_build_object('schema', schemaname, 'table', tablename) " +
|
|
1082
|
+
"ORDER BY schemaname, tablename), '[]') " +
|
|
1083
|
+
"FROM pg_tables WHERE schemaname NOT IN ('pg_catalog','information_schema')");
|
|
1084
|
+
if (tableListRes.code !== 0)
|
|
1085
|
+
verifyFailed = "could not list source tables";
|
|
1086
|
+
const sourceTables = tableListRes.code === 0 ? safeJsonArray(tableListRes.stdout) : [];
|
|
1087
|
+
const srcCounts = new Map();
|
|
1088
|
+
const tgtCounts = new Map();
|
|
1089
|
+
if (!verifyFailed && sourceTables.length > 0) {
|
|
1090
|
+
// Chunk into batches so a huge table list can't blow psql's statement limit
|
|
1091
|
+
// (which would fail the whole UNION ALL and leave both maps empty).
|
|
1092
|
+
for (const batch of chunk(sourceTables, 100)) {
|
|
1093
|
+
const countsSql = batch
|
|
1094
|
+
.map((t) => `SELECT ${quoteLiteral(`${t.schema}.${t.table}`)} AS t, count(*)::text AS c ` +
|
|
1095
|
+
`FROM ${quoteIdent(t.schema)}.${quoteIdent(t.table)}`)
|
|
1096
|
+
.join("\nUNION ALL\n");
|
|
1097
|
+
const sRes = runPsql(source, countsSql);
|
|
1098
|
+
const tRes = runPsql(targetUri, countsSql);
|
|
1099
|
+
if (sRes.code !== 0 || tRes.code !== 0) {
|
|
1100
|
+
verifyFailed = "row-count query failed";
|
|
1101
|
+
break;
|
|
1102
|
+
}
|
|
1103
|
+
const sMap = parseKvRows(sRes.stdout);
|
|
1104
|
+
const tMap = parseKvRows(tRes.stdout);
|
|
1105
|
+
// A query that returned fewer rows than tables didn't run cleanly.
|
|
1106
|
+
if (sMap.size !== batch.length || tMap.size !== batch.length) {
|
|
1107
|
+
verifyFailed = "row-count query returned an incomplete result";
|
|
1108
|
+
break;
|
|
1109
|
+
}
|
|
1110
|
+
for (const [k, v] of sMap)
|
|
1111
|
+
srcCounts.set(k, v);
|
|
1112
|
+
for (const [k, v] of tMap)
|
|
1113
|
+
tgtCounts.set(k, v);
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
const tableMismatches = verifyFailed ? [] : diffCounts(srcCounts, tgtCounts);
|
|
1117
|
+
const seqSql = "SELECT schemaname||'.'||sequencename AS s, COALESCE(last_value::text,'') AS v " +
|
|
1118
|
+
"FROM pg_sequences ORDER BY 1";
|
|
1119
|
+
const srcSeq = new Map();
|
|
1120
|
+
const tgtSeq = new Map();
|
|
1121
|
+
if (!verifyFailed) {
|
|
1122
|
+
const sSeqRes = runPsql(source, seqSql);
|
|
1123
|
+
const tSeqRes = runPsql(targetUri, seqSql);
|
|
1124
|
+
if (sSeqRes.code !== 0 || tSeqRes.code !== 0) {
|
|
1125
|
+
verifyFailed = "sequence query failed";
|
|
1126
|
+
}
|
|
1127
|
+
else {
|
|
1128
|
+
for (const [k, v] of parseKvRows(sSeqRes.stdout))
|
|
1129
|
+
srcSeq.set(k, v);
|
|
1130
|
+
for (const [k, v] of parseKvRows(tSeqRes.stdout))
|
|
1131
|
+
tgtSeq.set(k, v);
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
// Target-only sequences (present on target, absent from source) get the SAME
|
|
1135
|
+
// treatment as target-only tables: on a --project --yes existing target they're
|
|
1136
|
+
// expected leftovers (e.g. a pre-existing serial/identity table contributes one),
|
|
1137
|
+
// so they're excluded from the mismatch diff and surfaced as untouched — NOT a
|
|
1138
|
+
// failure. Source-only sequences and value drift on shared ones stay mismatches.
|
|
1139
|
+
let targetOnlySequences = [];
|
|
1140
|
+
let seqMismatches = [];
|
|
1141
|
+
if (!verifyFailed) {
|
|
1142
|
+
// The RAW set (incl. BataDB scaffolding) is what we exclude from the mismatch
|
|
1143
|
+
// diff — a target-only sequence must never read as a mismatch. The REPORTED
|
|
1144
|
+
// list + the created-project hard-fail then drop platform scaffolding (BataDB
|
|
1145
|
+
// seeds public.health_check_id_seq into every project, so it's expected on any
|
|
1146
|
+
// target and must not false-positive a freshly created one).
|
|
1147
|
+
const rawTargetOnlySeq = targetOnlyNames([...srcSeq.keys()], [...tgtSeq.keys()]);
|
|
1148
|
+
const rawTargetOnlySet = new Set(rawTargetOnlySeq);
|
|
1149
|
+
seqMismatches = diffCounts(srcSeq, tgtSeq).filter((r) => !rawTargetOnlySet.has(r.name));
|
|
1150
|
+
targetOnlySequences = excludeScaffolding(rawTargetOnlySeq);
|
|
1151
|
+
// A freshly created project must have no NON-scaffolding sequences the source lacks.
|
|
1152
|
+
if (created && targetOnlySequences.length > 0) {
|
|
1153
|
+
verifyFailed =
|
|
1154
|
+
`freshly created target unexpectedly has ${targetOnlySequences.length} sequence(s) absent ` +
|
|
1155
|
+
`from the source: ${targetOnlySequences.join(", ")}`;
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
// Enumerate the TARGET's tables too — verify only counts SOURCE tables, so a
|
|
1159
|
+
// --project --yes import into a non-empty target has pre-existing target-only
|
|
1160
|
+
// tables our row-count check never sees. Fail closed like the rest of verify.
|
|
1161
|
+
let targetOnly = [];
|
|
1162
|
+
if (!verifyFailed) {
|
|
1163
|
+
const tgtTablesRes = runPsql(targetUri, "SELECT schemaname||'.'||tablename FROM pg_tables " +
|
|
1164
|
+
"WHERE schemaname NOT IN ('pg_catalog','information_schema') ORDER BY 1");
|
|
1165
|
+
if (tgtTablesRes.code !== 0) {
|
|
1166
|
+
verifyFailed = "could not list target tables";
|
|
1167
|
+
}
|
|
1168
|
+
else {
|
|
1169
|
+
const targetQualified = tgtTablesRes.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
|
|
1170
|
+
const sourceQualified = sourceTables.map((t) => `${t.schema}.${t.table}`);
|
|
1171
|
+
// Drop BataDB scaffolding (public.health_check + the neon_migration schema) —
|
|
1172
|
+
// seeded/created on every project, so never a real "target-only" leftover and
|
|
1173
|
+
// must not false-positive a freshly created target. (Target-only tables never
|
|
1174
|
+
// enter the count mismatch diff, so no raw/reported split is needed here.)
|
|
1175
|
+
targetOnly = excludeScaffolding(targetOnlyNames(sourceQualified, targetQualified));
|
|
1176
|
+
// A FRESHLY CREATED project should have no NON-scaffolding tables the source
|
|
1177
|
+
// lacks. If it does, something is genuinely wrong (wrong target, contaminated
|
|
1178
|
+
// template) — hard-fail rather than wave it through as an expected leftover.
|
|
1179
|
+
if (created && targetOnly.length > 0) {
|
|
1180
|
+
verifyFailed =
|
|
1181
|
+
`freshly created target unexpectedly has ${targetOnly.length} table(s) absent ` +
|
|
1182
|
+
`from the source: ${targetOnly.join(", ")}`;
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
const tgtExtRes = runPsql(targetUri, "SELECT extname FROM pg_extension ORDER BY extname");
|
|
1187
|
+
const targetExtensions = tgtExtRes.code === 0
|
|
1188
|
+
? tgtExtRes.stdout.split("\n").map((s) => s.trim()).filter(Boolean)
|
|
1189
|
+
: [];
|
|
1190
|
+
// Residual check only: unsupported extensions were already hard-failed in the
|
|
1191
|
+
// pre-restore preflight, so this now catches only the oddity of an extension
|
|
1192
|
+
// that IS available on the target but somehow didn't get installed.
|
|
1193
|
+
const residualMissingExtensions = missingExtensions(sourceExtensions, targetExtensions);
|
|
1194
|
+
// Verification couldn't run — the data was restored but we can't confirm it
|
|
1195
|
+
// matches. Hard-fail rather than imply success.
|
|
1196
|
+
if (verifyFailed) {
|
|
1197
|
+
if (jsonMode) {
|
|
1198
|
+
json({
|
|
1199
|
+
error: `Verification could not run: ${verifyFailed}.`,
|
|
1200
|
+
code: "VERIFY_FAILED",
|
|
1201
|
+
hint: "The dump/restore reported success, but the CLI could not confirm row counts/sequences match. " +
|
|
1202
|
+
"Verify manually before relying on the target. " +
|
|
1203
|
+
strandedHintJson,
|
|
1204
|
+
target: strandedTargetJson(),
|
|
1205
|
+
});
|
|
1206
|
+
}
|
|
1207
|
+
else {
|
|
1208
|
+
log();
|
|
1209
|
+
log(` ${colors.red("Verification could not run")} — ${verifyFailed}.`);
|
|
1210
|
+
log(` ${colors.dim("The data was restored, but counts/sequences were NOT confirmed. Verify manually before relying on it.")}`);
|
|
1211
|
+
log();
|
|
1212
|
+
printStrandedTargetHints();
|
|
1213
|
+
}
|
|
1214
|
+
process.exit(1);
|
|
1215
|
+
return;
|
|
1216
|
+
}
|
|
1217
|
+
if (tableMismatches.length > 0 || seqMismatches.length > 0) {
|
|
1218
|
+
if (jsonMode) {
|
|
1219
|
+
json({
|
|
1220
|
+
error: "Verification failed — source and target differ after import.",
|
|
1221
|
+
code: "VERIFY_MISMATCH",
|
|
1222
|
+
hint: "The dump/restore reported success but row counts or sequences don't match. " + strandedHintJson,
|
|
1223
|
+
mismatches: {
|
|
1224
|
+
tables: tableMismatches,
|
|
1225
|
+
sequences: seqMismatches,
|
|
1226
|
+
},
|
|
1227
|
+
target: strandedTargetJson(),
|
|
1228
|
+
});
|
|
1229
|
+
}
|
|
1230
|
+
else {
|
|
1231
|
+
log();
|
|
1232
|
+
log(` ${colors.red("Verification failed")} — source and target differ:`);
|
|
1233
|
+
log();
|
|
1234
|
+
table(["OBJECT", "SOURCE", "TARGET"], mismatchRows(tableMismatches, seqMismatches));
|
|
1235
|
+
log();
|
|
1236
|
+
printStrandedTargetHints();
|
|
1237
|
+
}
|
|
1238
|
+
process.exit(1);
|
|
1239
|
+
return;
|
|
1240
|
+
}
|
|
1241
|
+
// ── Phase 6: done ──────────────────────────────────────────────────────────
|
|
1242
|
+
if (jsonMode) {
|
|
1243
|
+
json({
|
|
1244
|
+
ok: true,
|
|
1245
|
+
source: {
|
|
1246
|
+
database: sourceSummary.database,
|
|
1247
|
+
server_version: sourceSummary.server_version,
|
|
1248
|
+
server_major: sourceMajor,
|
|
1249
|
+
size: sourceSummary.size,
|
|
1250
|
+
tables: sourceSummary.tables,
|
|
1251
|
+
extensions: sourceExtensions,
|
|
1252
|
+
},
|
|
1253
|
+
target: {
|
|
1254
|
+
project_id: targetProjectId,
|
|
1255
|
+
created,
|
|
1256
|
+
pg_version: targetPg,
|
|
1257
|
+
connection: targetUri,
|
|
1258
|
+
},
|
|
1259
|
+
verify: {
|
|
1260
|
+
tables_checked: sourceTables.length,
|
|
1261
|
+
sequences_checked: srcSeq.size,
|
|
1262
|
+
extensions_missing_on_target: residualMissingExtensions,
|
|
1263
|
+
// Pre-existing target tables/sequences the source doesn't have — left
|
|
1264
|
+
// untouched and NOT covered by verification (only possible on --project
|
|
1265
|
+
// --yes into a non-empty target).
|
|
1266
|
+
target_only_tables: targetOnly,
|
|
1267
|
+
target_only_sequences: targetOnlySequences,
|
|
1268
|
+
},
|
|
1269
|
+
});
|
|
1270
|
+
return;
|
|
1271
|
+
}
|
|
1272
|
+
if (targetOnly.length > 0) {
|
|
1273
|
+
warn(`${targetOnly.length} pre-existing table(s) on the target were left untouched and are ` +
|
|
1274
|
+
`NOT covered by verification: ${targetOnly.join(", ")}`);
|
|
1275
|
+
}
|
|
1276
|
+
if (targetOnlySequences.length > 0) {
|
|
1277
|
+
warn(`${targetOnlySequences.length} pre-existing sequence(s) on the target were left untouched and are ` +
|
|
1278
|
+
`NOT covered by verification: ${targetOnlySequences.join(", ")}`);
|
|
1279
|
+
}
|
|
1280
|
+
if (residualMissingExtensions.length > 0) {
|
|
1281
|
+
warn(`Extensions on the source not installed on the target: ${residualMissingExtensions.join(", ")} ` +
|
|
1282
|
+
"(available but not created during restore — install them manually if needed).");
|
|
1283
|
+
}
|
|
1284
|
+
log();
|
|
1285
|
+
success(`Imported into ${colors.cyan(targetProjectId)} — ${sourceSummary.tables} tables verified.`);
|
|
1286
|
+
log();
|
|
1287
|
+
kvList([
|
|
1288
|
+
["Project ID", colors.dim(targetProjectId)],
|
|
1289
|
+
["PostgreSQL", targetPg],
|
|
1290
|
+
["Connection", colors.dim(targetUri)],
|
|
1291
|
+
]);
|
|
1292
|
+
log();
|
|
1293
|
+
log(` ${colors.dim("Connect:")} ${colors.cyan(`bata connect ${targetProjectId}`)}`);
|
|
1294
|
+
log();
|
|
1295
|
+
}
|
|
1296
|
+
/** Print a restore failure with its first errors (human mode). */
|
|
1297
|
+
function emitErrorHuman(message, errors) {
|
|
1298
|
+
log(` ${colors.red("Error:")} ${message}`);
|
|
1299
|
+
if (errors.length > 0) {
|
|
1300
|
+
log();
|
|
1301
|
+
for (const e of errors)
|
|
1302
|
+
log(` ${colors.dim(e)}`);
|
|
1303
|
+
}
|
|
1304
|
+
log();
|
|
1305
|
+
log(` ${colors.dim("Schema features newer than the target's PostgreSQL major aren't supported there.")}`);
|
|
1306
|
+
log();
|
|
1307
|
+
}
|
|
1308
|
+
function safeJsonArray(stdout) {
|
|
1309
|
+
try {
|
|
1310
|
+
const v = JSON.parse(stdout.trim());
|
|
1311
|
+
return Array.isArray(v) ? v : [];
|
|
1312
|
+
}
|
|
1313
|
+
catch {
|
|
1314
|
+
return [];
|
|
1315
|
+
}
|
|
1316
|
+
}
|