@ichibase/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -0
- package/dist/auth-map.js +106 -0
- package/dist/bundle.js +39 -0
- package/dist/firebase.js +271 -0
- package/dist/index.js +135 -0
- package/dist/infer.js +171 -0
- package/dist/mongo.js +157 -0
- package/dist/postgres.js +283 -0
- package/dist/push.js +107 -0
- package/package.json +31 -0
package/dist/infer.js
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
// Schema inference for NoSQL → Postgres migrations.
|
|
2
|
+
//
|
|
3
|
+
// A Mongo collection / Firestore collection is schemaless, but the user wants
|
|
4
|
+
// real typed tables, not one JSONB blob. So we scan EVERY document in a
|
|
5
|
+
// collection, build a per-field type profile, and emit a Postgres BundleTable
|
|
6
|
+
// the existing importer can CREATE + load:
|
|
7
|
+
// - the document id (Mongo _id / Firestore docId) → `id text primary key`
|
|
8
|
+
// - each top-level field → a typed column (bigint / double precision /
|
|
9
|
+
// boolean / timestamptz / text), or `jsonb` when the field holds nested
|
|
10
|
+
// objects/arrays, or `text` when a field's scalar type is inconsistent
|
|
11
|
+
// across documents.
|
|
12
|
+
//
|
|
13
|
+
// Callers (mongo.ts, firebase.ts) pre-normalize source-specific values into
|
|
14
|
+
// plain JS (BSON ObjectId→string, Mongo/Firestore timestamps→Date, etc.) so this
|
|
15
|
+
// module only reasons about: null, boolean, number, string, Date, array, object.
|
|
16
|
+
function classify(v) {
|
|
17
|
+
if (v === null || v === undefined)
|
|
18
|
+
return null;
|
|
19
|
+
if (typeof v === 'boolean')
|
|
20
|
+
return 'bool';
|
|
21
|
+
if (typeof v === 'bigint')
|
|
22
|
+
return 'int';
|
|
23
|
+
if (typeof v === 'number')
|
|
24
|
+
return Number.isInteger(v) ? 'int' : 'float';
|
|
25
|
+
if (v instanceof Date)
|
|
26
|
+
return 'date';
|
|
27
|
+
if (Array.isArray(v))
|
|
28
|
+
return 'array';
|
|
29
|
+
if (typeof v === 'string')
|
|
30
|
+
return 'string';
|
|
31
|
+
if (typeof v === 'object')
|
|
32
|
+
return 'object';
|
|
33
|
+
return 'string';
|
|
34
|
+
}
|
|
35
|
+
// Resolve a field's observed scalar classes to a single Postgres column type.
|
|
36
|
+
// Nested objects/arrays → jsonb; an inconsistent scalar mix → text (safe).
|
|
37
|
+
function resolveType(classes) {
|
|
38
|
+
if (classes.size === 0)
|
|
39
|
+
return 'text'; // field was always null/absent
|
|
40
|
+
if (classes.has('object') || classes.has('array'))
|
|
41
|
+
return 'jsonb';
|
|
42
|
+
if (classes.size === 1) {
|
|
43
|
+
if (classes.has('bool'))
|
|
44
|
+
return 'boolean';
|
|
45
|
+
if (classes.has('int'))
|
|
46
|
+
return 'bigint';
|
|
47
|
+
if (classes.has('float'))
|
|
48
|
+
return 'double precision';
|
|
49
|
+
if (classes.has('date'))
|
|
50
|
+
return 'timestamptz';
|
|
51
|
+
if (classes.has('string'))
|
|
52
|
+
return 'text';
|
|
53
|
+
}
|
|
54
|
+
// int + float together → a single numeric column.
|
|
55
|
+
if ([...classes].every((c) => c === 'int' || c === 'float'))
|
|
56
|
+
return 'double precision';
|
|
57
|
+
// any other scalar mix (e.g. string + number) → text.
|
|
58
|
+
return 'text';
|
|
59
|
+
}
|
|
60
|
+
function sanitizeIdent(name, used) {
|
|
61
|
+
let s = name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
|
|
62
|
+
if (!/^[a-z_]/.test(s))
|
|
63
|
+
s = '_' + s;
|
|
64
|
+
s = s.slice(0, 63);
|
|
65
|
+
if (!s)
|
|
66
|
+
s = 'field';
|
|
67
|
+
let candidate = s;
|
|
68
|
+
let i = 2;
|
|
69
|
+
while (used.has(candidate)) {
|
|
70
|
+
candidate = `${s.slice(0, 60)}_${i}`;
|
|
71
|
+
i++;
|
|
72
|
+
}
|
|
73
|
+
used.add(candidate);
|
|
74
|
+
return candidate;
|
|
75
|
+
}
|
|
76
|
+
/** Sanitize a collection name into a Postgres table name. */
|
|
77
|
+
export function sanitizeTableName(name) {
|
|
78
|
+
return sanitizeIdent(name, new Set());
|
|
79
|
+
}
|
|
80
|
+
// Two-pass over an in-memory collection: pass 1 here builds the type profile and
|
|
81
|
+
// the table; coerceRow (below) is pass 2, called per row by the caller.
|
|
82
|
+
export function inferCollection(tableName, rows) {
|
|
83
|
+
// First-seen order, union of all field names across every document.
|
|
84
|
+
const fieldOrder = [];
|
|
85
|
+
const classesByField = new Map();
|
|
86
|
+
const seenField = new Set();
|
|
87
|
+
const nullableField = new Set();
|
|
88
|
+
for (const row of rows) {
|
|
89
|
+
const keys = Object.keys(row.data);
|
|
90
|
+
const present = new Set(keys);
|
|
91
|
+
for (const k of keys) {
|
|
92
|
+
if (!classesByField.has(k)) {
|
|
93
|
+
classesByField.set(k, new Set());
|
|
94
|
+
fieldOrder.push(k);
|
|
95
|
+
}
|
|
96
|
+
const cls = classify(row.data[k]);
|
|
97
|
+
if (cls === null)
|
|
98
|
+
nullableField.add(k);
|
|
99
|
+
else
|
|
100
|
+
classesByField.get(k).add(cls);
|
|
101
|
+
}
|
|
102
|
+
// A field missing from this doc makes the column nullable.
|
|
103
|
+
for (const k of seenField)
|
|
104
|
+
if (!present.has(k))
|
|
105
|
+
nullableField.add(k);
|
|
106
|
+
for (const k of keys)
|
|
107
|
+
seenField.add(k);
|
|
108
|
+
}
|
|
109
|
+
// Fields that appeared only after other rows existed are nullable too.
|
|
110
|
+
if (rows.length) {
|
|
111
|
+
for (const k of fieldOrder) {
|
|
112
|
+
const count = rows.reduce((n, r) => n + (k in r.data ? 1 : 0), 0);
|
|
113
|
+
if (count < rows.length)
|
|
114
|
+
nullableField.add(k);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
const usedNames = new Set();
|
|
118
|
+
const idCol = { name: 'id', type: 'text', nullable: false };
|
|
119
|
+
usedNames.add('id');
|
|
120
|
+
const columns = [{ pgName: 'id', sourceField: null, type: 'text' }];
|
|
121
|
+
const bundleColumns = [idCol];
|
|
122
|
+
for (const field of fieldOrder) {
|
|
123
|
+
const type = resolveType(classesByField.get(field) ?? new Set());
|
|
124
|
+
const pgName = sanitizeIdent(field, usedNames);
|
|
125
|
+
columns.push({ pgName, sourceField: field, type });
|
|
126
|
+
bundleColumns.push({ name: pgName, type, nullable: nullableField.has(field) || true });
|
|
127
|
+
}
|
|
128
|
+
const table = {
|
|
129
|
+
name: tableName,
|
|
130
|
+
columns: bundleColumns,
|
|
131
|
+
primary_key: ['id'],
|
|
132
|
+
};
|
|
133
|
+
return { table, columns };
|
|
134
|
+
}
|
|
135
|
+
function coerceValue(v, pgType) {
|
|
136
|
+
if (v === null || v === undefined)
|
|
137
|
+
return null;
|
|
138
|
+
switch (pgType) {
|
|
139
|
+
case 'jsonb':
|
|
140
|
+
return v; // importer JSON-encodes + casts ::jsonb
|
|
141
|
+
case 'boolean':
|
|
142
|
+
return Boolean(v);
|
|
143
|
+
case 'bigint':
|
|
144
|
+
return typeof v === 'bigint' ? Number(v) : typeof v === 'number' ? v : Number(v);
|
|
145
|
+
case 'double precision':
|
|
146
|
+
return typeof v === 'number' ? v : Number(v);
|
|
147
|
+
case 'timestamptz':
|
|
148
|
+
return v instanceof Date ? v.toISOString() : String(v);
|
|
149
|
+
default: // text
|
|
150
|
+
if (typeof v === 'string')
|
|
151
|
+
return v;
|
|
152
|
+
if (v instanceof Date)
|
|
153
|
+
return v.toISOString();
|
|
154
|
+
if (typeof v === 'object')
|
|
155
|
+
return JSON.stringify(v);
|
|
156
|
+
return String(v);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
/** Pass 2: produce a NDJSON-ready row object keyed by Postgres column name. */
|
|
160
|
+
export function coerceRow(row, inf) {
|
|
161
|
+
const out = {};
|
|
162
|
+
for (const col of inf.columns) {
|
|
163
|
+
if (col.sourceField === null) {
|
|
164
|
+
out[col.pgName] = row.id;
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
out[col.pgName] = coerceValue(row.data[col.sourceField], col.type);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return out;
|
|
171
|
+
}
|
package/dist/mongo.js
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
// Extract a MongoDB / Atlas source.
|
|
2
|
+
//
|
|
3
|
+
// --target mongo (default) → native `mongodump --archive --gzip`. Highest
|
|
4
|
+
// fidelity; the server restores it with mongorestore and
|
|
5
|
+
// remaps namespaces onto db_<slug>. Needs mongodump on PATH.
|
|
6
|
+
// --target postgres → read every document with the driver, infer a typed
|
|
7
|
+
// Postgres schema per collection (one table each), and emit
|
|
8
|
+
// a Postgres-shaped bundle the existing importer consumes.
|
|
9
|
+
import { spawnSync } from 'node:child_process';
|
|
10
|
+
import { inferCollection, coerceRow, sanitizeTableName } from './infer.js';
|
|
11
|
+
import { mapAuthRecord } from './auth-map.js';
|
|
12
|
+
export async function extractMongo(conn, dbName, out, source, target, authMap) {
|
|
13
|
+
if (!dbName) {
|
|
14
|
+
throw new Error('--db <name> is required for a MongoDB/Atlas source (the source database to import)');
|
|
15
|
+
}
|
|
16
|
+
const base = target === 'postgres'
|
|
17
|
+
? await extractMongoToPostgres(conn, dbName, out, source)
|
|
18
|
+
: await extractMongoDump(conn, dbName, out, source);
|
|
19
|
+
// Bring-your-own-users: map a chosen collection into ichibase Auth (works for
|
|
20
|
+
// both targets — auth.ndjson is imported regardless of where the data landed).
|
|
21
|
+
let users = 0;
|
|
22
|
+
if (authMap) {
|
|
23
|
+
users = await dumpMongoMappedAuth(conn, dbName, authMap, out);
|
|
24
|
+
process.stderr.write(` • ${authMap.source} (custom auth): ${users} users\n`);
|
|
25
|
+
}
|
|
26
|
+
return { ...base, users };
|
|
27
|
+
}
|
|
28
|
+
async function dumpMongoMappedAuth(conn, dbName, authMap, out) {
|
|
29
|
+
const { MongoClient } = await import('mongodb');
|
|
30
|
+
const client = new MongoClient(conn);
|
|
31
|
+
await client.connect();
|
|
32
|
+
try {
|
|
33
|
+
const docs = await client.db(dbName).collection(authMap.source).find({}).toArray();
|
|
34
|
+
let n = 0;
|
|
35
|
+
for (const d of docs) {
|
|
36
|
+
const rec = mapAuthRecord(normalizeDoc(d), authMap);
|
|
37
|
+
if (rec) {
|
|
38
|
+
await out.writeJsonLine('auth.ndjson', rec);
|
|
39
|
+
n++;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return n;
|
|
43
|
+
}
|
|
44
|
+
finally {
|
|
45
|
+
await client.close();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
// ── target=mongo: native dump ─────────────────────────────────────────────
|
|
49
|
+
function extractMongoDump(conn, dbName, out, source) {
|
|
50
|
+
const archivePath = out.dataPath('mongo.archive');
|
|
51
|
+
process.stderr.write(` • mongodump --db ${dbName} → data/mongo.archive\n`);
|
|
52
|
+
const res = spawnSync('mongodump', [`--uri=${conn}`, `--db=${dbName}`, `--archive=${archivePath}`, '--gzip'], { stdio: ['ignore', 'inherit', 'inherit'] });
|
|
53
|
+
if (res.error) {
|
|
54
|
+
throw new Error(`could not run mongodump (${res.error.message}). Install the MongoDB Database Tools: https://www.mongodb.com/docs/database-tools/`);
|
|
55
|
+
}
|
|
56
|
+
if (res.status !== 0)
|
|
57
|
+
throw new Error(`mongodump exited with code ${res.status}`);
|
|
58
|
+
return out
|
|
59
|
+
.writeManifest({
|
|
60
|
+
source_kind: source,
|
|
61
|
+
created_at: new Date().toISOString(),
|
|
62
|
+
target_flavor: 'mongo',
|
|
63
|
+
mongo_source_db: dbName,
|
|
64
|
+
})
|
|
65
|
+
.then(() => ({ db: dbName }));
|
|
66
|
+
}
|
|
67
|
+
// ── target=postgres: driver read + schema inference ───────────────────────
|
|
68
|
+
async function extractMongoToPostgres(conn, dbName, out, source) {
|
|
69
|
+
// Imported lazily so a `--target mongo` run never needs the driver installed.
|
|
70
|
+
const { MongoClient } = await import('mongodb');
|
|
71
|
+
const client = new MongoClient(conn);
|
|
72
|
+
await client.connect();
|
|
73
|
+
try {
|
|
74
|
+
const db = client.db(dbName);
|
|
75
|
+
const collInfos = await db.listCollections({}, { nameOnly: true }).toArray();
|
|
76
|
+
const tables = [];
|
|
77
|
+
const counts = {};
|
|
78
|
+
let rowTotal = 0;
|
|
79
|
+
for (const ci of collInfos) {
|
|
80
|
+
if (ci.name.startsWith('system.'))
|
|
81
|
+
continue;
|
|
82
|
+
const docs = await db.collection(ci.name).find({}).toArray();
|
|
83
|
+
const rows = docs.map((d) => {
|
|
84
|
+
const { _id, ...rest } = d;
|
|
85
|
+
return { id: stringifyId(_id), data: normalizeDoc(rest) };
|
|
86
|
+
});
|
|
87
|
+
const tableName = sanitizeTableName(ci.name);
|
|
88
|
+
const inf = inferCollection(tableName, rows);
|
|
89
|
+
tables.push(inf.table);
|
|
90
|
+
const stream = out.ndjsonStream(`data/${tableName}.ndjson`);
|
|
91
|
+
for (const r of rows)
|
|
92
|
+
stream.write(JSON.stringify(coerceRow(r, inf)) + '\n');
|
|
93
|
+
await new Promise((res) => stream.end(res));
|
|
94
|
+
counts[tableName] = rows.length;
|
|
95
|
+
rowTotal += rows.length;
|
|
96
|
+
process.stderr.write(` • ${ci.name} → ${tableName}: ${rows.length} rows, ${inf.table.columns.length} cols\n`);
|
|
97
|
+
}
|
|
98
|
+
await out.writeSchema({ schema: 'public', tables });
|
|
99
|
+
await out.writeManifest({
|
|
100
|
+
source_kind: source,
|
|
101
|
+
created_at: new Date().toISOString(),
|
|
102
|
+
target_flavor: 'postgres',
|
|
103
|
+
counts,
|
|
104
|
+
});
|
|
105
|
+
return { db: dbName, tables: tables.length, rows: rowTotal };
|
|
106
|
+
}
|
|
107
|
+
finally {
|
|
108
|
+
await client.close();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
function stringifyId(id) {
|
|
112
|
+
const v = normalizeDoc(id);
|
|
113
|
+
if (v === null || v === undefined)
|
|
114
|
+
return '';
|
|
115
|
+
return typeof v === 'string' ? v : String(v);
|
|
116
|
+
}
|
|
117
|
+
// Convert BSON values (ObjectId, Long, Decimal128, Binary, …) into plain JS so
|
|
118
|
+
// the inference engine only sees null/bool/number/string/Date/array/object.
|
|
119
|
+
// Native Date is preserved (it carries timestamp typing).
|
|
120
|
+
export function normalizeDoc(v) {
|
|
121
|
+
if (v === null || v === undefined)
|
|
122
|
+
return null;
|
|
123
|
+
if (v instanceof Date)
|
|
124
|
+
return v;
|
|
125
|
+
if (Array.isArray(v))
|
|
126
|
+
return v.map(normalizeDoc);
|
|
127
|
+
if (typeof v === 'object') {
|
|
128
|
+
const bt = v._bsontype;
|
|
129
|
+
if (bt) {
|
|
130
|
+
const anyv = v;
|
|
131
|
+
switch (bt) {
|
|
132
|
+
case 'ObjectId':
|
|
133
|
+
case 'ObjectID':
|
|
134
|
+
case 'UUID':
|
|
135
|
+
case 'Timestamp':
|
|
136
|
+
return anyv.toString();
|
|
137
|
+
case 'Long':
|
|
138
|
+
return typeof anyv.toNumber === 'function' ? anyv.toNumber() : Number(anyv.toString());
|
|
139
|
+
case 'Decimal128':
|
|
140
|
+
return anyv.toString(); // keep precision as text
|
|
141
|
+
case 'Int32':
|
|
142
|
+
case 'Double':
|
|
143
|
+
return Number(anyv.toString());
|
|
144
|
+
case 'Binary':
|
|
145
|
+
return anyv.buffer ? Buffer.from(anyv.buffer).toString('base64') : anyv.toString();
|
|
146
|
+
default:
|
|
147
|
+
return anyv.toString();
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
const o = {};
|
|
151
|
+
for (const k of Object.keys(v)) {
|
|
152
|
+
o[k] = normalizeDoc(v[k]);
|
|
153
|
+
}
|
|
154
|
+
return o;
|
|
155
|
+
}
|
|
156
|
+
return v;
|
|
157
|
+
}
|
package/dist/postgres.js
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
// Extract a Postgres source (generic Postgres OR Supabase) into a bundle.
|
|
2
|
+
// Runs entirely on the user's machine against their own connection string.
|
|
3
|
+
import pg from 'pg';
|
|
4
|
+
import { mapAuthRecord } from './auth-map.js';
|
|
5
|
+
const ON_DELETE = { a: 'no action', r: 'restrict', c: 'cascade', n: 'set null', d: 'set default' };
|
|
6
|
+
export async function extractPostgres(conn, out, source, authMap) {
|
|
7
|
+
const client = new pg.Client({ connectionString: conn });
|
|
8
|
+
await client.connect();
|
|
9
|
+
try {
|
|
10
|
+
const tables = await introspect(client);
|
|
11
|
+
const types = await extractEnums(client);
|
|
12
|
+
const extensions = await extractExtensions(client);
|
|
13
|
+
const functions = await extractFunctions(client);
|
|
14
|
+
const schema = { schema: 'public', types, extensions, functions, tables };
|
|
15
|
+
await out.writeSchema(schema);
|
|
16
|
+
if (types.length)
|
|
17
|
+
process.stderr.write(` • ${types.length} custom enum type(s)\n`);
|
|
18
|
+
if (extensions.length)
|
|
19
|
+
process.stderr.write(` • ${extensions.length} extension(s): ${extensions.join(', ')}\n`);
|
|
20
|
+
if (functions.length)
|
|
21
|
+
process.stderr.write(` • ${functions.length} function(s)\n`);
|
|
22
|
+
const policyCount = tables.reduce((n, t) => n + (t.policies?.length ?? 0), 0);
|
|
23
|
+
if (policyCount)
|
|
24
|
+
process.stderr.write(` • ${policyCount} RLS policy(ies)\n`);
|
|
25
|
+
const triggerCount = tables.reduce((n, t) => n + (t.triggers?.length ?? 0), 0);
|
|
26
|
+
if (triggerCount)
|
|
27
|
+
process.stderr.write(` • ${triggerCount} trigger(s)\n`);
|
|
28
|
+
const counts = {};
|
|
29
|
+
let rowTotal = 0;
|
|
30
|
+
for (const t of tables) {
|
|
31
|
+
const n = await dumpTable(client, t.name, out);
|
|
32
|
+
counts[t.name] = n;
|
|
33
|
+
rowTotal += n;
|
|
34
|
+
process.stderr.write(` • ${t.name}: ${n} rows\n`);
|
|
35
|
+
}
|
|
36
|
+
let users = 0;
|
|
37
|
+
if (source === 'supabase') {
|
|
38
|
+
users = await dumpSupabaseAuth(client, out);
|
|
39
|
+
process.stderr.write(` • auth.users: ${users} users\n`);
|
|
40
|
+
}
|
|
41
|
+
// Bring-your-own-users: a custom table mapped into ichibase Auth (works for
|
|
42
|
+
// generic Postgres that has no Supabase auth.users; additive for Supabase).
|
|
43
|
+
if (authMap) {
|
|
44
|
+
const mapped = await dumpMappedAuth(client, authMap, out);
|
|
45
|
+
users += mapped;
|
|
46
|
+
process.stderr.write(` • ${authMap.source} (custom auth): ${mapped} users\n`);
|
|
47
|
+
}
|
|
48
|
+
await out.writeManifest({
|
|
49
|
+
source_kind: source,
|
|
50
|
+
created_at: new Date().toISOString(),
|
|
51
|
+
counts,
|
|
52
|
+
});
|
|
53
|
+
return { tables: tables.length, rows: rowTotal, users };
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
await client.end();
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// Custom enum types in the public schema (Supabase's own enums live in
|
|
60
|
+
// auth/storage/etc., so public ones are the user's). Recreated before tables.
|
|
61
|
+
async function extractEnums(client) {
|
|
62
|
+
const r = await client.query(`SELECT t.typname AS name,
|
|
63
|
+
array_agg(e.enumlabel ORDER BY e.enumsortorder)::text[] AS values
|
|
64
|
+
FROM pg_type t
|
|
65
|
+
JOIN pg_enum e ON e.enumtypid = t.oid
|
|
66
|
+
JOIN pg_namespace n ON n.oid = t.typnamespace
|
|
67
|
+
WHERE n.nspname = 'public'
|
|
68
|
+
GROUP BY t.typname
|
|
69
|
+
ORDER BY t.typname`);
|
|
70
|
+
return r.rows.map((row) => ({ name: row.name, kind: 'enum', values: row.values }));
|
|
71
|
+
}
|
|
72
|
+
// Installed extensions minus the ones ichibase always provisions.
|
|
73
|
+
async function extractExtensions(client) {
|
|
74
|
+
const r = await client.query(`SELECT extname FROM pg_extension
|
|
75
|
+
WHERE extname NOT IN ('plpgsql', 'pgcrypto', 'citext', 'pg_trgm')
|
|
76
|
+
ORDER BY extname`);
|
|
77
|
+
return r.rows.map((x) => x.extname);
|
|
78
|
+
}
|
|
79
|
+
// Full definitions of public functions/procedures (not extension-owned, not
|
|
80
|
+
// aggregates/window). pg_get_functiondef emits CREATE OR REPLACE … runnable as-is.
|
|
81
|
+
async function extractFunctions(client) {
|
|
82
|
+
const r = await client.query(`SELECT pg_get_functiondef(p.oid) AS def
|
|
83
|
+
FROM pg_proc p
|
|
84
|
+
JOIN pg_namespace n ON n.oid = p.pronamespace
|
|
85
|
+
WHERE n.nspname = 'public'
|
|
86
|
+
AND p.prokind IN ('f', 'p')
|
|
87
|
+
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = p.oid AND d.deptype = 'e')
|
|
88
|
+
ORDER BY p.proname`);
|
|
89
|
+
return r.rows.map((x) => x.def);
|
|
90
|
+
}
|
|
91
|
+
async function introspect(client) {
|
|
92
|
+
const tnames = await client.query(`SELECT c.relname AS name
|
|
93
|
+
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
94
|
+
WHERE n.nspname = 'public' AND c.relkind = 'r'
|
|
95
|
+
ORDER BY c.relname`);
|
|
96
|
+
const out = [];
|
|
97
|
+
for (const { name } of tnames.rows) {
|
|
98
|
+
const rel = `public.${quote(name)}`;
|
|
99
|
+
const cols = await client.query(`SELECT a.attname AS name,
|
|
100
|
+
format_type(a.atttypid, a.atttypmod) AS type,
|
|
101
|
+
a.attnotnull AS notnull,
|
|
102
|
+
pg_get_expr(d.adbin, d.adrelid) AS default,
|
|
103
|
+
a.attidentity <> '' AS identity
|
|
104
|
+
FROM pg_attribute a
|
|
105
|
+
LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum
|
|
106
|
+
WHERE a.attrelid = $1::regclass AND a.attnum > 0 AND NOT a.attisdropped
|
|
107
|
+
ORDER BY a.attnum`, [rel]);
|
|
108
|
+
const columns = cols.rows.map((c) => ({
|
|
109
|
+
name: c.name,
|
|
110
|
+
type: c.type,
|
|
111
|
+
nullable: !c.notnull,
|
|
112
|
+
default: c.default,
|
|
113
|
+
identity: c.identity,
|
|
114
|
+
}));
|
|
115
|
+
const pk = await client.query(`SELECT (SELECT array_agg(att.attname ORDER BY k.ord)::text[]
|
|
116
|
+
FROM unnest(con.conkey) WITH ORDINALITY k(attnum, ord)
|
|
117
|
+
JOIN pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = k.attnum) AS cols
|
|
118
|
+
FROM pg_constraint con
|
|
119
|
+
WHERE con.conrelid = $1::regclass AND con.contype = 'p'`, [rel]);
|
|
120
|
+
const uniqs = await client.query(`SELECT (SELECT array_agg(att.attname ORDER BY k.ord)::text[]
|
|
121
|
+
FROM unnest(con.conkey) WITH ORDINALITY k(attnum, ord)
|
|
122
|
+
JOIN pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = k.attnum) AS cols
|
|
123
|
+
FROM pg_constraint con
|
|
124
|
+
WHERE con.conrelid = $1::regclass AND con.contype = 'u'`, [rel]);
|
|
125
|
+
const fks = await client.query(`SELECT
|
|
126
|
+
(SELECT array_agg(att.attname ORDER BY k.ord)::text[]
|
|
127
|
+
FROM unnest(con.conkey) WITH ORDINALITY k(attnum, ord)
|
|
128
|
+
JOIN pg_attribute att ON att.attrelid = con.conrelid AND att.attnum = k.attnum) AS columns,
|
|
129
|
+
nsp.nspname AS ref_schema,
|
|
130
|
+
refcl.relname AS ref_table,
|
|
131
|
+
(SELECT array_agg(att.attname ORDER BY k.ord)::text[]
|
|
132
|
+
FROM unnest(con.confkey) WITH ORDINALITY k(attnum, ord)
|
|
133
|
+
JOIN pg_attribute att ON att.attrelid = con.confrelid AND att.attnum = k.attnum) AS ref_columns,
|
|
134
|
+
con.confdeltype AS on_delete
|
|
135
|
+
FROM pg_constraint con
|
|
136
|
+
JOIN pg_class refcl ON refcl.oid = con.confrelid
|
|
137
|
+
JOIN pg_namespace nsp ON nsp.oid = refcl.relnamespace
|
|
138
|
+
WHERE con.conrelid = $1::regclass AND con.contype = 'f'`, [rel]);
|
|
139
|
+
// Standalone indexes via pg_get_indexdef (covers expression/partial). Skip
|
|
140
|
+
// the PK + any index that backs a constraint (those are recreated as table
|
|
141
|
+
// constraints, so re-running their CREATE INDEX would duplicate).
|
|
142
|
+
const idx = await client.query(`SELECT i.relname AS name, pg_get_indexdef(ix.indexrelid) AS def
|
|
143
|
+
FROM pg_index ix
|
|
144
|
+
JOIN pg_class i ON i.oid = ix.indexrelid
|
|
145
|
+
WHERE ix.indrelid = $1::regclass AND NOT ix.indisprimary
|
|
146
|
+
AND NOT EXISTS (SELECT 1 FROM pg_constraint con WHERE con.conindid = ix.indexrelid)`, [rel]);
|
|
147
|
+
// User triggers (skip internal FK/constraint triggers). pg_get_triggerdef
|
|
148
|
+
// emits a runnable CREATE TRIGGER … EXECUTE FUNCTION … (the function itself
|
|
149
|
+
// is captured separately by extractFunctions).
|
|
150
|
+
const trg = await client.query(`SELECT tg.tgname AS name, pg_get_triggerdef(tg.oid) AS def
|
|
151
|
+
FROM pg_trigger tg
|
|
152
|
+
WHERE tg.tgrelid = $1::regclass AND NOT tg.tgisinternal`, [rel]);
|
|
153
|
+
// RLS policies on this table.
|
|
154
|
+
const pol = await client.query(`SELECT policyname AS name, cmd AS command, roles::text[] AS roles,
|
|
155
|
+
qual AS using, with_check AS check
|
|
156
|
+
FROM pg_policies WHERE schemaname = 'public' AND tablename = $1`, [name]);
|
|
157
|
+
const table = {
|
|
158
|
+
name,
|
|
159
|
+
columns,
|
|
160
|
+
primary_key: pk.rows[0]?.cols ?? undefined,
|
|
161
|
+
uniques: uniqs.rows.map((u) => u.cols).filter(Boolean),
|
|
162
|
+
foreign_keys: fks.rows.map((f) => ({
|
|
163
|
+
columns: f.columns,
|
|
164
|
+
ref_schema: f.ref_schema,
|
|
165
|
+
ref_table: f.ref_table,
|
|
166
|
+
ref_columns: f.ref_columns,
|
|
167
|
+
on_delete: ON_DELETE[f.on_delete] ?? 'no action',
|
|
168
|
+
})),
|
|
169
|
+
indexes: idx.rows.map((i) => ({ name: i.name, def: i.def })),
|
|
170
|
+
policies: pol.rows.map((p) => ({
|
|
171
|
+
name: p.name,
|
|
172
|
+
command: p.command,
|
|
173
|
+
roles: p.roles,
|
|
174
|
+
using: p.using ?? undefined,
|
|
175
|
+
check: p.check ?? undefined,
|
|
176
|
+
})),
|
|
177
|
+
triggers: trg.rows.map((t) => ({ name: t.name, def: t.def })),
|
|
178
|
+
};
|
|
179
|
+
out.push(table);
|
|
180
|
+
}
|
|
181
|
+
return out;
|
|
182
|
+
}
|
|
183
|
+
// Stream a table's rows to data/<table>.ndjson via a server-side cursor, writing
|
|
184
|
+
// row_to_json()::text verbatim (no client-side JSON.parse → full fidelity).
|
|
185
|
+
async function dumpTable(client, table, out) {
|
|
186
|
+
const stream = out.ndjsonStream(`data/${table}.ndjson`);
|
|
187
|
+
let n = 0;
|
|
188
|
+
await client.query('BEGIN');
|
|
189
|
+
try {
|
|
190
|
+
await client.query(`DECLARE ich_cur NO SCROLL CURSOR FOR SELECT row_to_json(t)::text AS j FROM public.${quote(table)} t`);
|
|
191
|
+
for (;;) {
|
|
192
|
+
const batch = await client.query('FETCH 1000 FROM ich_cur');
|
|
193
|
+
if (batch.rowCount === 0)
|
|
194
|
+
break;
|
|
195
|
+
for (const r of batch.rows) {
|
|
196
|
+
stream.write(r.j + '\n');
|
|
197
|
+
n++;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
await client.query('CLOSE ich_cur');
|
|
201
|
+
await client.query('COMMIT');
|
|
202
|
+
}
|
|
203
|
+
catch (e) {
|
|
204
|
+
await client.query('ROLLBACK').catch(() => { });
|
|
205
|
+
throw e;
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
await new Promise((res) => stream.end(res));
|
|
209
|
+
}
|
|
210
|
+
return n;
|
|
211
|
+
}
|
|
212
|
+
// Supabase auth.users → auth.ndjson. encrypted_password is bcrypt → passed
|
|
213
|
+
// verbatim (the server stores it tagged and login lazily re-hashes to Argon2id).
|
|
214
|
+
// Original user UUIDs are preserved so migrated data FKs to auth.users stay valid.
|
|
215
|
+
async function dumpSupabaseAuth(client, out) {
|
|
216
|
+
// OAuth identities (provider + provider uid), grouped by user. Best-effort —
|
|
217
|
+
// the table/columns vary across Supabase versions.
|
|
218
|
+
const idByUser = new Map();
|
|
219
|
+
try {
|
|
220
|
+
const ids = await client.query(`SELECT user_id::text, provider, COALESCE(provider_id, id::text) AS provider_uid FROM auth.identities`);
|
|
221
|
+
for (const r of ids.rows) {
|
|
222
|
+
const list = idByUser.get(r.user_id) ?? [];
|
|
223
|
+
list.push({ provider: r.provider, provider_uid: r.provider_uid });
|
|
224
|
+
idByUser.set(r.user_id, list);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
/* no identities table / different shape — skip OAuth links */
|
|
229
|
+
}
|
|
230
|
+
const users = await client.query(`SELECT id::text,
|
|
231
|
+
email::text,
|
|
232
|
+
encrypted_password AS password_hash,
|
|
233
|
+
(email_confirmed_at IS NOT NULL) AS email_verified,
|
|
234
|
+
COALESCE(raw_user_meta_data, '{}'::jsonb)::text AS metadata,
|
|
235
|
+
created_at
|
|
236
|
+
FROM auth.users
|
|
237
|
+
WHERE email IS NOT NULL`);
|
|
238
|
+
let n = 0;
|
|
239
|
+
for (const u of users.rows) {
|
|
240
|
+
let metadata = {};
|
|
241
|
+
try {
|
|
242
|
+
metadata = JSON.parse(u.metadata);
|
|
243
|
+
}
|
|
244
|
+
catch {
|
|
245
|
+
metadata = {};
|
|
246
|
+
}
|
|
247
|
+
await out.writeJsonLine('auth.ndjson', {
|
|
248
|
+
id: u.id,
|
|
249
|
+
email: u.email,
|
|
250
|
+
password_hash: u.password_hash || null,
|
|
251
|
+
email_verified: u.email_verified,
|
|
252
|
+
metadata,
|
|
253
|
+
created_at: u.created_at,
|
|
254
|
+
oauth: idByUser.get(u.id) ?? [],
|
|
255
|
+
});
|
|
256
|
+
n++;
|
|
257
|
+
}
|
|
258
|
+
return n;
|
|
259
|
+
}
|
|
260
|
+
// Bring-your-own-users from a custom Postgres table. row_to_json gives plain JS
|
|
261
|
+
// objects the shared mapper turns into auth records (id/email/hash/verified/…).
|
|
262
|
+
async function dumpMappedAuth(client, authMap, out) {
|
|
263
|
+
const res = await client.query(`SELECT row_to_json(t) AS j FROM ${quoteRel(authMap.source)} t`);
|
|
264
|
+
let n = 0;
|
|
265
|
+
for (const row of res.rows) {
|
|
266
|
+
const rec = mapAuthRecord(row.j, authMap);
|
|
267
|
+
if (rec) {
|
|
268
|
+
await out.writeJsonLine('auth.ndjson', rec);
|
|
269
|
+
n++;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return n;
|
|
273
|
+
}
|
|
274
|
+
function quote(ident) {
|
|
275
|
+
return '"' + ident.replace(/"/g, '""') + '"';
|
|
276
|
+
}
|
|
277
|
+
// Quote a possibly schema-qualified relation ("schema.table" or "table").
|
|
278
|
+
function quoteRel(rel) {
|
|
279
|
+
return rel
|
|
280
|
+
.split('.')
|
|
281
|
+
.map((p) => quote(p))
|
|
282
|
+
.join('.');
|
|
283
|
+
}
|