@ichibase/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -0
- package/dist/auth-map.js +106 -0
- package/dist/bundle.js +39 -0
- package/dist/firebase.js +271 -0
- package/dist/index.js +135 -0
- package/dist/infer.js +171 -0
- package/dist/mongo.js +157 -0
- package/dist/postgres.js +283 -0
- package/dist/push.js +107 -0
- package/package.json +31 -0
package/README.md
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# @ichibase/cli
|
|
2
|
+
|
|
3
|
+
The `ichibase` CLI. Today it ships the **migration** commands — extract an
|
|
4
|
+
existing app from Supabase / Postgres / MongoDB / Atlas into an ichibase
|
|
5
|
+
migration bundle and push it to your project. (The broader local-dev surface —
|
|
6
|
+
`init` / `db push` / `functions deploy` — is still future work.)
|
|
7
|
+
|
|
8
|
+
Extraction runs entirely on **your** machine against **your** source. Your source
|
|
9
|
+
credentials never leave your computer; only the resulting migration bundle is
|
|
10
|
+
uploaded to your ichibase project.
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
npm i -g @ichibase/cli # or: npx @ichibase/cli …
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
For MongoDB/Atlas sources you also need the
|
|
19
|
+
[MongoDB Database Tools](https://www.mongodb.com/docs/database-tools/)
|
|
20
|
+
(`mongodump`) on your `PATH`.
|
|
21
|
+
|
|
22
|
+
## 1. Extract a bundle
|
|
23
|
+
|
|
24
|
+
```sh
|
|
25
|
+
# Supabase (tables + data + auth users; bcrypt passwords come across)
|
|
26
|
+
# Use the SESSION POOLER string (Settings → Database → Connection string → Session pooler);
|
|
27
|
+
# the direct db.<ref>.supabase.co host is IPv6-only and won't resolve on most machines.
|
|
28
|
+
ichibase migrate supabase --conn "postgresql://postgres.<ref>:PASSWORD@aws-0-<region>.pooler.supabase.com:5432/postgres" --out ./bundle
|
|
29
|
+
|
|
30
|
+
# Generic Postgres (schema + data)
|
|
31
|
+
ichibase migrate postgres --conn "postgresql://user:pw@host:5432/dbname" --out ./bundle
|
|
32
|
+
|
|
33
|
+
# MongoDB / Atlas (collections + documents)
|
|
34
|
+
ichibase migrate mongodb --conn "mongodb://user:pw@host:27017" --db myapp --out ./bundle
|
|
35
|
+
ichibase migrate atlas --conn "mongodb+srv://user:pw@cluster.mongodb.net" --db myapp --out ./bundle
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
This writes an **ichibase migration bundle** to `./bundle`:
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
manifest.json source kind, row counts, (mongo) source db name
|
|
42
|
+
schema.json Postgres: tables, columns, PK, uniques, indexes, FKs
|
|
43
|
+
data/<table>.ndjson Postgres rows (one JSON object per line)
|
|
44
|
+
data/mongo.archive Mongo: native mongodump --archive --gzip
|
|
45
|
+
auth.ndjson users (Supabase): id, email, password_hash, oauth links
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## 2. Push it to your project
|
|
49
|
+
|
|
50
|
+
```sh
|
|
51
|
+
ichibase migrate push --project <slug> --bundle ./bundle \
|
|
52
|
+
--api https://api.ichibase.com --token "$ICHIBASE_TOKEN"
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`--token` is your ichibase owner token / API key (or set `ICHIBASE_TOKEN`).
|
|
56
|
+
The CLI uploads each bundle part to storage via presigned URLs, starts the
|
|
57
|
+
import, and streams progress until it completes. The final report lists row /
|
|
58
|
+
document counts and any manual follow-ups.
|
|
59
|
+
|
|
60
|
+
## Notes & limitations (v1)
|
|
61
|
+
|
|
62
|
+
- **Same-paradigm only:** Supabase/Postgres → Postgres, MongoDB/Atlas → MongoDB.
|
|
63
|
+
Cross-paradigm (SQL→Mongo, Firestore) is coming later.
|
|
64
|
+
- **Passwords:** imported users keep working — ichibase verifies the source
|
|
65
|
+
hash (bcrypt) on first login and transparently upgrades it to Argon2id. Users
|
|
66
|
+
whose hash can't be verified must use "forgot password".
|
|
67
|
+
- **User IDs are preserved**, so your data's foreign keys to `auth.users` stay valid.
|
|
68
|
+
- **RLS:** imported tables have Row-Level Security **enabled with no policies** —
|
|
69
|
+
only your service key can read them until you add policies (Dashboard →
|
|
70
|
+
Database → RLS). Source policies are reported, not auto-translated.
|
|
71
|
+
- **Paid plans only.**
|
|
72
|
+
- Integers larger than 2^53 should be stored as strings in the source to avoid
|
|
73
|
+
JSON precision loss; standard `bigint` ids within that range are fine.
|
package/dist/auth-map.js
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// "Bring your own users" — a source-agnostic mapper that turns an arbitrary
|
|
2
|
+
// users table / collection into auth.ndjson records the server imports into
|
|
3
|
+
// ichibase Auth. Works for ANY source (generic Postgres, Mongo/Atlas, a Firestore
|
|
4
|
+
// users collection), in ADDITION to the built-in extractors (Supabase auth.users,
|
|
5
|
+
// Firebase Auth export).
|
|
6
|
+
//
|
|
7
|
+
// The owner declares which fields hold the email / password hash / id / etc. and
|
|
8
|
+
// what scheme the hash is in:
|
|
9
|
+
// bcrypt | argon2 → passed verbatim (login verifies, then lazy-upgrades)
|
|
10
|
+
// fbscrypt → wrapped into a $fbscrypt$ tag (needs per-user salt field +
|
|
11
|
+
// project params; see services/auth/fbscrypt.go)
|
|
12
|
+
// plain | none → no hash stored → user lands as needs_reset (must reset /
|
|
13
|
+
// use OAuth). Plaintext is never stored.
|
|
14
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
15
|
+
// Assemble the self-describing tag that services/auth verifies (matches the
|
|
16
|
+
// parser in fbscrypt.go). Shared with the Firebase Auth export path.
|
|
17
|
+
export function buildFbscryptTag(fb, saltB64, hashB64) {
|
|
18
|
+
return `$fbscrypt$ln=${fb.memCost},r=${fb.rounds},p=1$${fb.saltSeparator}$${fb.signerKey}$${saltB64}$${hashB64}`;
|
|
19
|
+
}
|
|
20
|
+
function asString(v) {
|
|
21
|
+
if (v === null || v === undefined)
|
|
22
|
+
return null;
|
|
23
|
+
if (typeof v === 'string')
|
|
24
|
+
return v;
|
|
25
|
+
if (v instanceof Date)
|
|
26
|
+
return v.toISOString();
|
|
27
|
+
return String(v);
|
|
28
|
+
}
|
|
29
|
+
function truthy(v) {
|
|
30
|
+
if (typeof v === 'boolean')
|
|
31
|
+
return v;
|
|
32
|
+
if (typeof v === 'number')
|
|
33
|
+
return v !== 0;
|
|
34
|
+
if (typeof v === 'string')
|
|
35
|
+
return ['true', 't', '1', 'yes', 'y'].includes(v.trim().toLowerCase());
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
// Map one source row/document → an AuthRecord, or null to skip (no usable email).
|
|
39
|
+
export function mapAuthRecord(doc, m) {
|
|
40
|
+
const email = (asString(doc[m.emailField]) ?? '').trim().toLowerCase();
|
|
41
|
+
if (!email || !email.includes('@'))
|
|
42
|
+
return null;
|
|
43
|
+
let password_hash = null;
|
|
44
|
+
if (m.passField && m.hashScheme !== 'none' && m.hashScheme !== 'plain') {
|
|
45
|
+
const raw = asString(doc[m.passField]);
|
|
46
|
+
if (raw) {
|
|
47
|
+
if (m.hashScheme === 'bcrypt' || m.hashScheme === 'argon2') {
|
|
48
|
+
password_hash = raw; // already a verifiable, self-identifying tagged hash
|
|
49
|
+
}
|
|
50
|
+
else if (m.hashScheme === 'fbscrypt' && m.fb && m.saltField) {
|
|
51
|
+
const salt = asString(doc[m.saltField]);
|
|
52
|
+
if (salt)
|
|
53
|
+
password_hash = buildFbscryptTag(m.fb, salt, raw);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const metadata = {};
|
|
58
|
+
let id;
|
|
59
|
+
if (m.idField) {
|
|
60
|
+
const idRaw = asString(doc[m.idField]);
|
|
61
|
+
if (idRaw) {
|
|
62
|
+
// The server preserves a valid UUID; otherwise it mints a new one, so keep
|
|
63
|
+
// the original around for reference / data reconciliation.
|
|
64
|
+
if (UUID_RE.test(idRaw))
|
|
65
|
+
id = idRaw;
|
|
66
|
+
else
|
|
67
|
+
metadata.source_id = idRaw;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
id,
|
|
72
|
+
email,
|
|
73
|
+
password_hash,
|
|
74
|
+
email_verified: m.verifiedField ? truthy(doc[m.verifiedField]) : false,
|
|
75
|
+
metadata,
|
|
76
|
+
created_at: m.createdField ? asString(doc[m.createdField]) : null,
|
|
77
|
+
oauth: [],
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
// Parse the common --auth-* flags into a field map, or undefined when the owner
|
|
81
|
+
// didn't ask to import users from a custom source.
|
|
82
|
+
export function parseAuthMap(f) {
|
|
83
|
+
const source = f['auth-collection'] ?? f['auth-table'];
|
|
84
|
+
if (!source)
|
|
85
|
+
return undefined;
|
|
86
|
+
const hashScheme = (f['auth-hash-scheme'] ?? 'none');
|
|
87
|
+
const map = {
|
|
88
|
+
source,
|
|
89
|
+
emailField: f['auth-email-field'] ?? 'email',
|
|
90
|
+
passField: f['auth-pass-field'],
|
|
91
|
+
idField: f['auth-id-field'],
|
|
92
|
+
verifiedField: f['auth-verified-field'],
|
|
93
|
+
createdField: f['auth-created-field'],
|
|
94
|
+
saltField: f['auth-salt-field'],
|
|
95
|
+
hashScheme,
|
|
96
|
+
};
|
|
97
|
+
if (hashScheme === 'fbscrypt') {
|
|
98
|
+
map.fb = {
|
|
99
|
+
signerKey: f['auth-fb-signer-key'] ?? '',
|
|
100
|
+
saltSeparator: f['auth-fb-salt-separator'] ?? '',
|
|
101
|
+
rounds: Number(f['auth-fb-rounds'] ?? '8'),
|
|
102
|
+
memCost: Number(f['auth-fb-mem-cost'] ?? '14'),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
return map;
|
|
106
|
+
}
|
package/dist/bundle.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// Shapes + writers for the "ichibase migration bundle" (IMB) the server imports.
|
|
2
|
+
// Layout on disk (and in R2 under migrations/<slug>/<job>/):
|
|
3
|
+
// manifest.json { source_kind, mongo_source_db?, counts? }
|
|
4
|
+
// schema.json Postgres: { schema, tables[...] }
|
|
5
|
+
// data/<t>.ndjson Postgres rows (one JSON object per line)
|
|
6
|
+
// data/mongo.archive Mongo: native mongodump --archive --gzip
|
|
7
|
+
// auth.ndjson users (one JSON object per line)
|
|
8
|
+
import { createWriteStream, mkdirSync } from 'node:fs';
|
|
9
|
+
import { writeFile } from 'node:fs/promises';
|
|
10
|
+
import path from 'node:path';
|
|
11
|
+
export class BundleWriter {
|
|
12
|
+
dir;
|
|
13
|
+
constructor(dir) {
|
|
14
|
+
this.dir = dir;
|
|
15
|
+
mkdirSync(path.join(dir, 'data'), { recursive: true });
|
|
16
|
+
}
|
|
17
|
+
async writeManifest(m) {
|
|
18
|
+
await writeFile(path.join(this.dir, 'manifest.json'), JSON.stringify(m, null, 2));
|
|
19
|
+
}
|
|
20
|
+
async writeSchema(s) {
|
|
21
|
+
await writeFile(path.join(this.dir, 'schema.json'), JSON.stringify(s, null, 2));
|
|
22
|
+
}
|
|
23
|
+
async writeJsonLine(file, obj) {
|
|
24
|
+
const full = path.join(this.dir, file);
|
|
25
|
+
mkdirSync(path.dirname(full), { recursive: true });
|
|
26
|
+
await writeFile(full, JSON.stringify(obj) + '\n', { flag: 'a' });
|
|
27
|
+
}
|
|
28
|
+
// Stream-append raw NDJSON text (already one-object-per-line) — used for table
|
|
29
|
+
// row dumps so we never JSON.parse on the way out (preserves fidelity + memory).
|
|
30
|
+
// Creates parent dirs (e.g. data/mongo/) so nested collection files work.
|
|
31
|
+
ndjsonStream(file) {
|
|
32
|
+
const full = path.join(this.dir, file);
|
|
33
|
+
mkdirSync(path.dirname(full), { recursive: true });
|
|
34
|
+
return createWriteStream(full, { flags: 'w' });
|
|
35
|
+
}
|
|
36
|
+
dataPath(file) {
|
|
37
|
+
return path.join(this.dir, 'data', file);
|
|
38
|
+
}
|
|
39
|
+
}
|
package/dist/firebase.js
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
// Extract a Firebase project: Firestore data + Firebase Auth users.
|
|
2
|
+
//
|
|
3
|
+
// ichibase migrate firebase --service-account ./sa.json \
|
|
4
|
+
// --target <mongo|postgres> \
|
|
5
|
+
// [--auth-export ./users.json --auth-fb-signer-key … --auth-fb-salt-separator … \
|
|
6
|
+
// --auth-fb-rounds 8 --auth-fb-mem-cost 14] [--out <dir>]
|
|
7
|
+
//
|
|
8
|
+
// Firestore is read with the Admin SDK (a service-account JSON). Subcollections
|
|
9
|
+
// are flattened into separate tables/collections named `parent__child` with a
|
|
10
|
+
// `_parent_id` reference. Firebase Auth is migrated from a `firebase auth:export`
|
|
11
|
+
// JSON; password hashes are wrapped into a $fbscrypt$ tag (using the project's
|
|
12
|
+
// hash params from the console) so users keep their passwords (lazy-upgraded to
|
|
13
|
+
// Argon2id on first login).
|
|
14
|
+
import { readFileSync } from 'node:fs';
|
|
15
|
+
import { createHash } from 'node:crypto';
|
|
16
|
+
import { initializeApp, cert } from 'firebase-admin/app';
|
|
17
|
+
import { getFirestore, Timestamp, GeoPoint, DocumentReference, } from 'firebase-admin/firestore';
|
|
18
|
+
import { inferCollection, coerceRow, sanitizeTableName } from './infer.js';
|
|
19
|
+
import { buildFbscryptTag, mapAuthRecord, } from './auth-map.js';
|
|
20
|
+
export async function extractFirebase(opts) {
|
|
21
|
+
const sa = JSON.parse(readFileSync(opts.serviceAccountPath, 'utf8'));
|
|
22
|
+
const app = initializeApp({ credential: cert(sa) });
|
|
23
|
+
const db = getFirestore(app);
|
|
24
|
+
// Build the Firebase-UID → deterministic-UUID map from the auth export FIRST,
|
|
25
|
+
// so document fields that reference users (posterUID, ownerId, doc ids, …) can
|
|
26
|
+
// be rewritten to the SAME UUIDs the users import under. Firebase UIDs aren't
|
|
27
|
+
// UUIDs, so without this the references would dangle after migration.
|
|
28
|
+
let authRecords = [];
|
|
29
|
+
let uidMap = new Map();
|
|
30
|
+
if (opts.authExportPath) {
|
|
31
|
+
({ records: authRecords, uidMap } = loadFirebaseAuth(opts.authExportPath, opts.fbHash));
|
|
32
|
+
process.stderr.write(` • Firebase Auth export: ${authRecords.length} users (UID→UUID remap on)\n`);
|
|
33
|
+
}
|
|
34
|
+
const { tables, counts, docTotal, ejsonCounts } = await extractFirestore(db, opts.out, opts.target, uidMap);
|
|
35
|
+
if (opts.target === 'postgres') {
|
|
36
|
+
await opts.out.writeSchema({ schema: 'public', tables });
|
|
37
|
+
}
|
|
38
|
+
let users = 0;
|
|
39
|
+
if (authRecords.length) {
|
|
40
|
+
for (const rec of authRecords)
|
|
41
|
+
await opts.out.writeJsonLine('auth.ndjson', rec);
|
|
42
|
+
users = authRecords.length;
|
|
43
|
+
process.stderr.write(` • Firebase Auth: ${users} users written\n`);
|
|
44
|
+
}
|
|
45
|
+
else if (opts.authMap) {
|
|
46
|
+
users = await dumpFirestoreCollectionAuth(db, opts.authMap, opts.out);
|
|
47
|
+
process.stderr.write(` • ${opts.authMap.source} (custom auth): ${users} users\n`);
|
|
48
|
+
}
|
|
49
|
+
await opts.out.writeManifest({
|
|
50
|
+
source_kind: 'firebase',
|
|
51
|
+
created_at: new Date().toISOString(),
|
|
52
|
+
target_flavor: opts.target,
|
|
53
|
+
counts: opts.target === 'postgres' ? counts : ejsonCounts,
|
|
54
|
+
});
|
|
55
|
+
const collCount = opts.target === 'postgres' ? tables.length : Object.keys(ejsonCounts).length;
|
|
56
|
+
return { collections: collCount, docs: docTotal, users };
|
|
57
|
+
}
|
|
58
|
+
// Deterministic UUIDv5 (SHA-1) so a Firebase UID always maps to the same UUID —
|
|
59
|
+
// across the auth import AND every document reference to that user.
|
|
60
|
+
const ICHIBASE_FB_NAMESPACE = '6f4d8e2a-1c3b-4f5a-9e7d-2b8c1a0f3e6d';
|
|
61
|
+
function uuidv5(name) {
|
|
62
|
+
const ns = Buffer.from(ICHIBASE_FB_NAMESPACE.replace(/-/g, ''), 'hex');
|
|
63
|
+
const h = createHash('sha1').update(ns).update(Buffer.from(name, 'utf8')).digest();
|
|
64
|
+
const b = h.subarray(0, 16);
|
|
65
|
+
b[6] = (b[6] & 0x0f) | 0x50; // version 5
|
|
66
|
+
b[8] = (b[8] & 0x3f) | 0x80; // RFC-4122 variant
|
|
67
|
+
const x = b.toString('hex');
|
|
68
|
+
return `${x.slice(0, 8)}-${x.slice(8, 12)}-${x.slice(12, 16)}-${x.slice(16, 20)}-${x.slice(20, 32)}`;
|
|
69
|
+
}
|
|
70
|
+
// ── Firestore ─────────────────────────────────────────────────────────────
|
|
71
|
+
async function extractFirestore(db, out, target, uidMap) {
|
|
72
|
+
// Gather first: subcollections of different parent docs share a logical
|
|
73
|
+
// collection (users/{a}/posts + users/{b}/posts → one `users__posts` table),
|
|
74
|
+
// so we can't write per-parent or we'd clobber the file.
|
|
75
|
+
const buckets = new Map();
|
|
76
|
+
async function gather(coll, baseName, parentId) {
|
|
77
|
+
const bucket = buckets.get(baseName) ?? { rows: [], ejson: [] };
|
|
78
|
+
buckets.set(baseName, bucket);
|
|
79
|
+
const snap = await coll.get();
|
|
80
|
+
process.stderr.write(` • read ${baseName}: ${snap.size} docs\n`);
|
|
81
|
+
for (const doc of snap.docs) {
|
|
82
|
+
const data = doc.data();
|
|
83
|
+
// If the doc id IS a Firebase UID (e.g. users/{uid}), use the user's UUID
|
|
84
|
+
// so the doc lines up with the migrated auth user + any references.
|
|
85
|
+
const docId = uidMap.get(doc.id) ?? doc.id;
|
|
86
|
+
if (target === 'postgres') {
|
|
87
|
+
const plain = toPlainJs(data, uidMap);
|
|
88
|
+
if (parentId !== null)
|
|
89
|
+
plain._parent_id = parentId;
|
|
90
|
+
bucket.rows.push({ id: docId, data: plain });
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
const ejson = toEjson(data, uidMap);
|
|
94
|
+
ejson._id = docId;
|
|
95
|
+
if (parentId !== null)
|
|
96
|
+
ejson._parent_id = parentId;
|
|
97
|
+
bucket.ejson.push(JSON.stringify(ejson));
|
|
98
|
+
}
|
|
99
|
+
// Subcollection parent ref = the (remapped) parent doc id.
|
|
100
|
+
for (const sub of await doc.ref.listCollections()) {
|
|
101
|
+
await gather(sub, `${baseName}__${sub.id}`, docId);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
process.stderr.write(' • connecting to Firestore (listing collections)…\n');
|
|
106
|
+
const topCollections = await db.listCollections();
|
|
107
|
+
process.stderr.write(` • ${topCollections.length} top-level collection(s)\n`);
|
|
108
|
+
for (const c of topCollections)
|
|
109
|
+
await gather(c, c.id, null);
|
|
110
|
+
const tables = [];
|
|
111
|
+
const counts = {};
|
|
112
|
+
const ejsonCounts = {};
|
|
113
|
+
let docTotal = 0;
|
|
114
|
+
const usedNames = new Set();
|
|
115
|
+
for (const [baseName, bucket] of buckets) {
|
|
116
|
+
let name = sanitizeTableName(baseName);
|
|
117
|
+
while (usedNames.has(name))
|
|
118
|
+
name = `${name}_x`;
|
|
119
|
+
usedNames.add(name);
|
|
120
|
+
if (target === 'postgres') {
|
|
121
|
+
const inf = inferCollection(name, bucket.rows);
|
|
122
|
+
tables.push(inf.table);
|
|
123
|
+
const stream = out.ndjsonStream(`data/${name}.ndjson`);
|
|
124
|
+
for (const r of bucket.rows)
|
|
125
|
+
stream.write(JSON.stringify(coerceRow(r, inf)) + '\n');
|
|
126
|
+
await new Promise((res) => stream.end(res));
|
|
127
|
+
counts[name] = bucket.rows.length;
|
|
128
|
+
docTotal += bucket.rows.length;
|
|
129
|
+
process.stderr.write(` • ${baseName} → ${name}: ${bucket.rows.length} docs, ${inf.table.columns.length} cols\n`);
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
const stream = out.ndjsonStream(`data/mongo/${name}.ndjson`);
|
|
133
|
+
for (const l of bucket.ejson)
|
|
134
|
+
stream.write(l + '\n');
|
|
135
|
+
await new Promise((res) => stream.end(res));
|
|
136
|
+
ejsonCounts[name] = bucket.ejson.length;
|
|
137
|
+
docTotal += bucket.ejson.length;
|
|
138
|
+
process.stderr.write(` • ${baseName} → ${name}: ${bucket.ejson.length} docs\n`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return { tables, counts, ejsonCounts, docTotal };
|
|
142
|
+
}
|
|
143
|
+
// Firestore value → plain JS (for Postgres inference): Timestamp→Date,
|
|
144
|
+
// GeoPoint→{latitude,longitude}, DocumentReference→path, Bytes→base64 string.
|
|
145
|
+
// Any string that is a known Firebase UID is rewritten to the user's UUID so
|
|
146
|
+
// references (posterUID, ownerId, …) stay valid post-migration.
|
|
147
|
+
function toPlainJs(v, uidMap) {
|
|
148
|
+
if (v === null || v === undefined)
|
|
149
|
+
return null;
|
|
150
|
+
if (v instanceof Timestamp)
|
|
151
|
+
return v.toDate();
|
|
152
|
+
if (v instanceof GeoPoint)
|
|
153
|
+
return { latitude: v.latitude, longitude: v.longitude };
|
|
154
|
+
if (v instanceof DocumentReference)
|
|
155
|
+
return v.path;
|
|
156
|
+
if (Buffer.isBuffer(v))
|
|
157
|
+
return v.toString('base64');
|
|
158
|
+
if (v instanceof Date)
|
|
159
|
+
return v;
|
|
160
|
+
if (Array.isArray(v))
|
|
161
|
+
return v.map((x) => toPlainJs(x, uidMap));
|
|
162
|
+
if (typeof v === 'object') {
|
|
163
|
+
const o = {};
|
|
164
|
+
for (const k of Object.keys(v))
|
|
165
|
+
o[k] = toPlainJs(v[k], uidMap);
|
|
166
|
+
return o;
|
|
167
|
+
}
|
|
168
|
+
if (typeof v === 'string')
|
|
169
|
+
return uidMap.get(v) ?? v;
|
|
170
|
+
return v;
|
|
171
|
+
}
|
|
172
|
+
// Firestore value → MongoDB Extended JSON (for mongoimport --type json).
|
|
173
|
+
function toEjson(v, uidMap) {
|
|
174
|
+
if (v === null || v === undefined)
|
|
175
|
+
return null;
|
|
176
|
+
if (v instanceof Timestamp)
|
|
177
|
+
return { $date: v.toDate().toISOString() };
|
|
178
|
+
if (v instanceof GeoPoint)
|
|
179
|
+
return { type: 'Point', coordinates: [v.longitude, v.latitude] };
|
|
180
|
+
if (v instanceof DocumentReference)
|
|
181
|
+
return v.path;
|
|
182
|
+
if (Buffer.isBuffer(v))
|
|
183
|
+
return { $binary: { base64: v.toString('base64'), subType: '00' } };
|
|
184
|
+
if (v instanceof Date)
|
|
185
|
+
return { $date: v.toISOString() };
|
|
186
|
+
if (Array.isArray(v))
|
|
187
|
+
return v.map((x) => toEjson(x, uidMap));
|
|
188
|
+
if (typeof v === 'object') {
|
|
189
|
+
const o = {};
|
|
190
|
+
for (const k of Object.keys(v))
|
|
191
|
+
o[k] = toEjson(v[k], uidMap);
|
|
192
|
+
return o;
|
|
193
|
+
}
|
|
194
|
+
if (typeof v === 'string')
|
|
195
|
+
return uidMap.get(v) ?? v;
|
|
196
|
+
return v;
|
|
197
|
+
}
|
|
198
|
+
// ── Firebase Auth ───────────────────────────────────────────────────────────
|
|
199
|
+
const PROVIDER_MAP = {
|
|
200
|
+
'google.com': 'google',
|
|
201
|
+
'apple.com': 'apple',
|
|
202
|
+
'github.com': 'github',
|
|
203
|
+
'facebook.com': 'facebook',
|
|
204
|
+
'twitter.com': 'twitter',
|
|
205
|
+
'microsoft.com': 'microsoft',
|
|
206
|
+
};
|
|
207
|
+
// Parse a `firebase auth:export` JSON into auth records + a UID→UUID map.
|
|
208
|
+
// Each user's ichibase id is a DETERMINISTIC uuidv5(localId) so the same id is
|
|
209
|
+
// used by the auth import AND by every document reference to that user. Password
|
|
210
|
+
// hash + per-user salt are wrapped into a $fbscrypt$ tag; OAuth identities become
|
|
211
|
+
// oauth[] links; the Firebase localId is also kept in metadata.
|
|
212
|
+
function loadFirebaseAuth(path, fb) {
|
|
213
|
+
const parsed = JSON.parse(readFileSync(path, 'utf8'));
|
|
214
|
+
const list = parsed.users ?? [];
|
|
215
|
+
const records = [];
|
|
216
|
+
const uidMap = new Map();
|
|
217
|
+
for (const u of list) {
|
|
218
|
+
const email = (u.email ?? '').trim().toLowerCase();
|
|
219
|
+
if (!email || !email.includes('@'))
|
|
220
|
+
continue;
|
|
221
|
+
const id = u.localId ? uuidv5(u.localId) : undefined;
|
|
222
|
+
if (u.localId && id)
|
|
223
|
+
uidMap.set(u.localId, id);
|
|
224
|
+
let password_hash = null;
|
|
225
|
+
if (fb && u.passwordHash && u.salt) {
|
|
226
|
+
password_hash = buildFbscryptTag(fb, u.salt, u.passwordHash);
|
|
227
|
+
}
|
|
228
|
+
const oauth = [];
|
|
229
|
+
for (const p of u.providerUserInfo ?? []) {
|
|
230
|
+
const prov = p.providerId ? PROVIDER_MAP[p.providerId] : undefined;
|
|
231
|
+
if (prov && p.rawId)
|
|
232
|
+
oauth.push({ provider: prov, provider_uid: p.rawId });
|
|
233
|
+
}
|
|
234
|
+
const metadata = {};
|
|
235
|
+
if (u.localId)
|
|
236
|
+
metadata.firebase_uid = u.localId;
|
|
237
|
+
if (u.displayName)
|
|
238
|
+
metadata.name = u.displayName;
|
|
239
|
+
if (u.photoUrl)
|
|
240
|
+
metadata.avatar_url = u.photoUrl;
|
|
241
|
+
if (u.phoneNumber)
|
|
242
|
+
metadata.phone = u.phoneNumber;
|
|
243
|
+
const createdMs = u.createdAt ? Number(u.createdAt) : NaN;
|
|
244
|
+
records.push({
|
|
245
|
+
id,
|
|
246
|
+
email,
|
|
247
|
+
password_hash,
|
|
248
|
+
email_verified: Boolean(u.emailVerified),
|
|
249
|
+
metadata,
|
|
250
|
+
created_at: Number.isFinite(createdMs) ? new Date(createdMs).toISOString() : null,
|
|
251
|
+
oauth,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
return { records, uidMap };
|
|
255
|
+
}
|
|
256
|
+
// Alternative: users stored in a Firestore collection (no Firebase Auth).
|
|
257
|
+
async function dumpFirestoreCollectionAuth(db, authMap, out) {
|
|
258
|
+
const snap = await db.collection(authMap.source).get();
|
|
259
|
+
let n = 0;
|
|
260
|
+
for (const doc of snap.docs) {
|
|
261
|
+
const data = toPlainJs(doc.data(), new Map());
|
|
262
|
+
if (authMap.idField && !(authMap.idField in data))
|
|
263
|
+
data[authMap.idField] = doc.id;
|
|
264
|
+
const rec = mapAuthRecord(data, authMap);
|
|
265
|
+
if (rec) {
|
|
266
|
+
await out.writeJsonLine('auth.ndjson', rec);
|
|
267
|
+
n++;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return n;
|
|
271
|
+
}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// @ichibase/cli — migrate an existing app into an ichibase project.
|
|
3
|
+
//
|
|
4
|
+
// ichibase migrate supabase --conn "postgresql://…" --out ./bundle
|
|
5
|
+
// ichibase migrate postgres --conn "postgresql://…" --out ./bundle
|
|
6
|
+
// ichibase migrate mongodb --conn "mongodb://…" --db myapp --out ./bundle
|
|
7
|
+
// ichibase migrate atlas --conn "mongodb+srv://…" --db myapp --out ./bundle
|
|
8
|
+
// ichibase migrate push --project <slug> --bundle ./bundle [--api URL] [--token T]
|
|
9
|
+
//
|
|
10
|
+
// Extraction runs on YOUR machine against YOUR source — credentials never leave
|
|
11
|
+
// it. `push` uploads the bundle to your project and runs the import.
|
|
12
|
+
import { BundleWriter } from './bundle.js';
|
|
13
|
+
import { extractPostgres } from './postgres.js';
|
|
14
|
+
import { extractMongo } from './mongo.js';
|
|
15
|
+
import { extractFirebase } from './firebase.js';
|
|
16
|
+
import { parseAuthMap } from './auth-map.js';
|
|
17
|
+
import { push } from './push.js';
|
|
18
|
+
const SOURCES = ['supabase', 'postgres', 'mongodb', 'atlas', 'firebase'];
|
|
19
|
+
function flags(argv) {
|
|
20
|
+
const out = {};
|
|
21
|
+
for (let i = 0; i < argv.length; i++) {
|
|
22
|
+
const a = argv[i];
|
|
23
|
+
if (a.startsWith('--')) {
|
|
24
|
+
const key = a.slice(2);
|
|
25
|
+
const next = argv[i + 1];
|
|
26
|
+
if (next && !next.startsWith('--')) {
|
|
27
|
+
out[key] = next;
|
|
28
|
+
i++;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
out[key] = 'true';
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return out;
|
|
36
|
+
}
|
|
37
|
+
function usage() {
|
|
38
|
+
process.stderr.write([
|
|
39
|
+
'Usage:',
|
|
40
|
+
' ichibase migrate <supabase|postgres|mongodb|atlas> --conn <connstr> [--db <name>] [--out <dir>]',
|
|
41
|
+
' ichibase migrate <mongodb|atlas> --conn <connstr> --db <name> --target <mongo|postgres> [--out <dir>]',
|
|
42
|
+
' ichibase migrate firebase --service-account <key.json> --target <mongo|postgres> [--auth-export <users.json>] [--out <dir>]',
|
|
43
|
+
' ichibase migrate push --project <slug> --bundle <dir> [--api <url>] [--token <token>]',
|
|
44
|
+
'',
|
|
45
|
+
' --target mongo (default) imports 1:1 into MongoDB; --target postgres infers',
|
|
46
|
+
' typed tables from your documents and imports into Postgres.',
|
|
47
|
+
' Firebase Auth: pass --auth-export plus --auth-fb-signer-key / --auth-fb-salt-separator',
|
|
48
|
+
' / --auth-fb-rounds / --auth-fb-mem-cost (Console → Password hash parameters).',
|
|
49
|
+
'',
|
|
50
|
+
'Env: ICHIBASE_API_URL, ICHIBASE_TOKEN',
|
|
51
|
+
].join('\n') + '\n');
|
|
52
|
+
process.exit(2);
|
|
53
|
+
}
|
|
54
|
+
async function main() {
|
|
55
|
+
const [, , cmd, sub, ...rest] = process.argv;
|
|
56
|
+
if (cmd !== 'migrate' || !sub)
|
|
57
|
+
usage();
|
|
58
|
+
const f = flags(rest);
|
|
59
|
+
if (sub === 'push') {
|
|
60
|
+
const project = f.project;
|
|
61
|
+
const bundleDir = f.bundle ?? './ichibase-bundle';
|
|
62
|
+
const api = f.api ?? process.env.ICHIBASE_API_URL ?? 'https://api.ichibase.com';
|
|
63
|
+
const token = f.token ?? process.env.ICHIBASE_TOKEN ?? '';
|
|
64
|
+
if (!project)
|
|
65
|
+
usage();
|
|
66
|
+
if (!token) {
|
|
67
|
+
process.stderr.write('error: --token (or ICHIBASE_TOKEN) is required — your ichibase owner token/API key\n');
|
|
68
|
+
process.exit(2);
|
|
69
|
+
}
|
|
70
|
+
await push({ project, bundleDir, api, token });
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
if (!SOURCES.includes(sub))
|
|
74
|
+
usage();
|
|
75
|
+
const source = sub;
|
|
76
|
+
const outDir = f.out ?? './ichibase-bundle';
|
|
77
|
+
const out = new BundleWriter(outDir);
|
|
78
|
+
const authMap = parseAuthMap(f);
|
|
79
|
+
// Firebase uses a service-account JSON (Admin SDK), not a --conn string.
|
|
80
|
+
if (source === 'firebase') {
|
|
81
|
+
const serviceAccountPath = f['service-account'] ?? f.key;
|
|
82
|
+
if (!serviceAccountPath) {
|
|
83
|
+
process.stderr.write('error: --service-account <path-to-key.json> is required for firebase\n');
|
|
84
|
+
process.exit(2);
|
|
85
|
+
}
|
|
86
|
+
const target = f.target === 'postgres' ? 'postgres' : 'mongo';
|
|
87
|
+
const fbHash = f['auth-fb-signer-key']
|
|
88
|
+
? {
|
|
89
|
+
signerKey: f['auth-fb-signer-key'],
|
|
90
|
+
saltSeparator: f['auth-fb-salt-separator'] ?? '',
|
|
91
|
+
rounds: Number(f['auth-fb-rounds'] ?? '8'),
|
|
92
|
+
memCost: Number(f['auth-fb-mem-cost'] ?? '14'),
|
|
93
|
+
}
|
|
94
|
+
: undefined;
|
|
95
|
+
process.stderr.write(`Extracting firebase (→ ${target}) → ${outDir}\n`);
|
|
96
|
+
const r = await extractFirebase({
|
|
97
|
+
serviceAccountPath,
|
|
98
|
+
target,
|
|
99
|
+
out,
|
|
100
|
+
authExportPath: f['auth-export'],
|
|
101
|
+
fbHash,
|
|
102
|
+
authMap,
|
|
103
|
+
});
|
|
104
|
+
process.stderr.write(`✓ ${r.collections} collections, ${r.docs} docs${r.users ? `, ${r.users} users` : ''} → ${outDir}\n`);
|
|
105
|
+
process.stderr.write(`\nNext: ichibase migrate push --project <slug> --bundle ${outDir} --token <token>\n`);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
const conn = f.conn;
|
|
109
|
+
if (!conn) {
|
|
110
|
+
process.stderr.write('error: --conn <connection string> is required\n');
|
|
111
|
+
process.exit(2);
|
|
112
|
+
}
|
|
113
|
+
process.stderr.write(`Extracting ${source} → ${outDir}\n`);
|
|
114
|
+
if (source === 'supabase' || source === 'postgres') {
|
|
115
|
+
const r = await extractPostgres(conn, out, source, authMap);
|
|
116
|
+
process.stderr.write(`✓ ${r.tables} tables, ${r.rows} rows${r.users ? `, ${r.users} users` : ''} → ${outDir}\n`);
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
// NoSQL sources land in either Mongo (1:1) or Postgres (auto-schema).
|
|
120
|
+
const target = f.target === 'postgres' ? 'postgres' : 'mongo';
|
|
121
|
+
const r = await extractMongo(conn, f.db ?? '', out, source, target, authMap);
|
|
122
|
+
const usersNote = r.users ? `, ${r.users} users` : '';
|
|
123
|
+
if (target === 'postgres') {
|
|
124
|
+
process.stderr.write(`✓ ${r.tables} collections → ${r.rows} rows across inferred Postgres tables${usersNote} → ${outDir}\n`);
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
process.stderr.write(`✓ mongodump of "${r.db}"${usersNote} → ${outDir}/data/mongo.archive\n`);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
process.stderr.write(`\nNext: ichibase migrate push --project <slug> --bundle ${outDir} --token <token>\n`);
|
|
131
|
+
}
|
|
132
|
+
main().catch((err) => {
|
|
133
|
+
process.stderr.write(`\nerror: ${err.message}\n`);
|
|
134
|
+
process.exit(1);
|
|
135
|
+
});
|