admindb 2.5.0 → 2.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -20
- package/dist/app.js +13 -1
- package/dist/core/diff/data-differ.d.ts +1 -0
- package/dist/core/diff/data-differ.js +243 -0
- package/dist/core/diff/patch-generator.d.ts +1 -0
- package/dist/core/diff/patch-generator.js +77 -0
- package/dist/core/diff/schema-differ.d.ts +1 -0
- package/dist/core/diff/schema-differ.js +229 -0
- package/dist/core/router.d.ts +1 -1
- package/dist/core/transfer/dialect-mapper.d.ts +1 -0
- package/dist/core/transfer/dialect-mapper.js +303 -0
- package/dist/core/transfer/masking.d.ts +1 -0
- package/dist/core/transfer/masking.js +49 -0
- package/dist/core/transfer/sync-engine.d.ts +1 -0
- package/dist/core/transfer/sync-engine.js +234 -0
- package/dist/core/transfer/topological-sort.d.ts +1 -0
- package/dist/core/transfer/topological-sort.js +55 -0
- package/dist/db/dialects/postgres/introspector.js +71 -71
- package/dist/db/dialects/sqlite/introspector.js +6 -3
- package/dist/modules/index.d.ts +1 -1
- package/dist/modules/index.js +9 -1
- package/dist/modules/search/index.d.ts +1 -0
- package/dist/modules/search/index.js +19 -0
- package/dist/modules/search/search.routes.d.ts +1 -0
- package/dist/modules/search/search.routes.js +38 -0
- package/dist/modules/search/search.service.d.ts +1 -0
- package/dist/modules/search/search.service.js +458 -0
- package/dist/modules/search/search.types.d.ts +1 -0
- package/dist/modules/search/search.types.js +2 -0
- package/dist/modules/sync/index.d.ts +1 -0
- package/dist/modules/sync/index.js +18 -0
- package/dist/modules/sync/sync.routes.d.ts +1 -0
- package/dist/modules/sync/sync.routes.js +135 -0
- package/dist/modules/sync/sync.service.d.ts +1 -0
- package/dist/modules/sync/sync.service.js +41 -0
- package/dist/public/css/app.css +2 -1
- package/dist/public/js/browse.js +1 -1
- package/dist/public/js/search.js +1 -0
- package/dist/public/js/sync.js +1 -0
- package/dist/utils/icons.js +3 -0
- package/dist/views/layouts/main.hbs +1 -1
- package/dist/views/pages/error.hbs +1 -10
- package/dist/views/pages/seed-select.hbs +252 -1
- package/dist/views/pages/seed.hbs +307 -1
- package/dist/views/pages/sync.hbs +1 -0
- package/dist/views/pages/table.hbs +1 -1
- package/dist/views/partials/navbar.hbs +1 -1
- package/dist/views/partials/sidebar.hbs +1 -1
- package/docs/EXAMPLES.md +579 -579
- package/docs/SECURITY.md +265 -265
- package/package.json +14 -9
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.categorizeType = categorizeType;
|
|
4
|
+
exports.mapColumnToDialect = mapColumnToDialect;
|
|
5
|
+
exports.normalizeTableInfo = normalizeTableInfo;
|
|
6
|
+
exports.convertValueForDialect = convertValueForDialect;
|
|
7
|
+
exports.mapDefaultValueForDialect = mapDefaultValueForDialect;
|
|
8
|
+
exports.generateCreateTableForDialect = generateCreateTableForDialect;
|
|
9
|
+
function categorizeType(rawType) {
|
|
10
|
+
const t = (rawType || '').toLowerCase().trim();
|
|
11
|
+
if (t.includes('int') || t.includes('serial') || t === 'smallint' || t === 'bigint')
|
|
12
|
+
return 'integer';
|
|
13
|
+
if (t.includes('bool'))
|
|
14
|
+
return 'boolean';
|
|
15
|
+
if (t.includes('float') || t.includes('double') || t.includes('real') || t.includes('numeric') || t.includes('decimal'))
|
|
16
|
+
return 'real';
|
|
17
|
+
if (t.includes('blob') || t.includes('bytea') || t.includes('binary'))
|
|
18
|
+
return 'blob';
|
|
19
|
+
if (t.includes('date') || t.includes('time'))
|
|
20
|
+
return 'datetime';
|
|
21
|
+
if (t.includes('json'))
|
|
22
|
+
return 'json';
|
|
23
|
+
return 'text';
|
|
24
|
+
}
|
|
25
|
+
function mapColumnToDialect(col, targetDialect) {
|
|
26
|
+
if (targetDialect === 'postgres') {
|
|
27
|
+
if (col.autoIncrement) {
|
|
28
|
+
return col.type.toLowerCase().includes('big') ? 'BIGSERIAL' : 'SERIAL';
|
|
29
|
+
}
|
|
30
|
+
switch (col.genericType) {
|
|
31
|
+
case 'integer':
|
|
32
|
+
return col.type.toLowerCase().includes('big') ? 'BIGINT' : 'INTEGER';
|
|
33
|
+
case 'boolean':
|
|
34
|
+
return 'BOOLEAN';
|
|
35
|
+
case 'real':
|
|
36
|
+
return 'DOUBLE PRECISION';
|
|
37
|
+
case 'blob':
|
|
38
|
+
return 'BYTEA';
|
|
39
|
+
case 'datetime':
|
|
40
|
+
return 'TIMESTAMP WITH TIME ZONE';
|
|
41
|
+
case 'json':
|
|
42
|
+
return 'JSONB';
|
|
43
|
+
case 'text':
|
|
44
|
+
default:
|
|
45
|
+
return 'TEXT';
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (col.primaryKey && col.autoIncrement) {
|
|
49
|
+
return 'INTEGER PRIMARY KEY AUTOINCREMENT';
|
|
50
|
+
}
|
|
51
|
+
switch (col.genericType) {
|
|
52
|
+
case 'integer':
|
|
53
|
+
return 'INTEGER';
|
|
54
|
+
case 'boolean':
|
|
55
|
+
return 'INTEGER';
|
|
56
|
+
case 'real':
|
|
57
|
+
return 'REAL';
|
|
58
|
+
case 'blob':
|
|
59
|
+
return 'BLOB';
|
|
60
|
+
case 'datetime':
|
|
61
|
+
case 'json':
|
|
62
|
+
case 'text':
|
|
63
|
+
default:
|
|
64
|
+
return 'TEXT';
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
function normalizeTableInfo(info, dialect) {
|
|
68
|
+
const uniqueCols = new Set();
|
|
69
|
+
const pks = new Set(info.primaryKey || []);
|
|
70
|
+
for (const idx of info.indexes || []) {
|
|
71
|
+
const cols = Array.isArray(idx.columns) ? idx.columns.filter(Boolean) : [];
|
|
72
|
+
if (idx.unique && cols.length === 1 && !pks.has(cols[0])) {
|
|
73
|
+
uniqueCols.add(cols[0]);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const columns = (info.columns || []).map((col) => {
|
|
77
|
+
const isPk = col.pk > 0 || pks.has(col.name);
|
|
78
|
+
const rawType = (col.type || '').toUpperCase();
|
|
79
|
+
const genericType = categorizeType(rawType);
|
|
80
|
+
const isAutoIncrement = (isPk && genericType === 'integer' && dialect === 'sqlite' && Boolean(info.sql?.toLowerCase().includes('autoincrement'))) ||
|
|
81
|
+
rawType.includes('SERIAL') ||
|
|
82
|
+
Boolean(col.dflt_value?.includes('nextval(')) ||
|
|
83
|
+
Boolean(info.sql?.toLowerCase().includes('generated always as identity'));
|
|
84
|
+
const fk = (info.foreignKeys || []).find((f) => f.from === col.name);
|
|
85
|
+
return {
|
|
86
|
+
name: col.name,
|
|
87
|
+
type: col.type,
|
|
88
|
+
genericType,
|
|
89
|
+
primaryKey: isPk,
|
|
90
|
+
autoIncrement: isAutoIncrement,
|
|
91
|
+
notNull: Boolean(col.notnull),
|
|
92
|
+
defaultValue: col.dflt_value,
|
|
93
|
+
unique: uniqueCols.has(col.name),
|
|
94
|
+
foreignKey: fk
|
|
95
|
+
? {
|
|
96
|
+
table: fk.table,
|
|
97
|
+
column: fk.to || '',
|
|
98
|
+
onDelete: fk.on_delete,
|
|
99
|
+
onUpdate: fk.on_update,
|
|
100
|
+
}
|
|
101
|
+
: null,
|
|
102
|
+
};
|
|
103
|
+
});
|
|
104
|
+
const indexes = (info.indexes || [])
|
|
105
|
+
.filter((ix) => !ix.name?.startsWith('sqlite_') && ix.origin !== 'pk' && (!ix.name?.endsWith('_pkey') || !ix.unique))
|
|
106
|
+
.map((ix) => {
|
|
107
|
+
let cols = [];
|
|
108
|
+
if (Array.isArray(ix.columns)) {
|
|
109
|
+
cols = ix.columns.filter((c) => typeof c === 'string' && c.trim().length > 0);
|
|
110
|
+
}
|
|
111
|
+
else if (typeof ix.columns === 'string') {
|
|
112
|
+
cols = [ix.columns];
|
|
113
|
+
}
|
|
114
|
+
return {
|
|
115
|
+
name: ix.name || `idx_${info.table}_${cols.join('_') || 'col'}`,
|
|
116
|
+
unique: Boolean(ix.unique),
|
|
117
|
+
columns: cols,
|
|
118
|
+
};
|
|
119
|
+
})
|
|
120
|
+
.filter((ix) => ix.columns.length > 0);
|
|
121
|
+
return {
|
|
122
|
+
name: info.table,
|
|
123
|
+
columns,
|
|
124
|
+
primaryKeys: info.primaryKey || [],
|
|
125
|
+
foreignKeys: info.foreignKeys || [],
|
|
126
|
+
indexes,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
function convertValueForDialect(value, genericType, targetDialect) {
|
|
130
|
+
if (value === null || value === undefined)
|
|
131
|
+
return null;
|
|
132
|
+
if (targetDialect === 'postgres') {
|
|
133
|
+
if (genericType === 'boolean') {
|
|
134
|
+
if (typeof value === 'boolean')
|
|
135
|
+
return value;
|
|
136
|
+
if (typeof value === 'number')
|
|
137
|
+
return value !== 0;
|
|
138
|
+
if (typeof value === 'string')
|
|
139
|
+
return value === '1' || value.toLowerCase() === 'true';
|
|
140
|
+
return Boolean(value);
|
|
141
|
+
}
|
|
142
|
+
if (genericType === 'json') {
|
|
143
|
+
if (typeof value === 'object')
|
|
144
|
+
return JSON.stringify(value);
|
|
145
|
+
return String(value);
|
|
146
|
+
}
|
|
147
|
+
if (genericType === 'blob') {
|
|
148
|
+
if (Buffer.isBuffer(value))
|
|
149
|
+
return value;
|
|
150
|
+
if (value instanceof Uint8Array)
|
|
151
|
+
return Buffer.from(value);
|
|
152
|
+
return Buffer.from(String(value));
|
|
153
|
+
}
|
|
154
|
+
return value;
|
|
155
|
+
}
|
|
156
|
+
if (genericType === 'boolean') {
|
|
157
|
+
if (typeof value === 'boolean')
|
|
158
|
+
return value ? 1 : 0;
|
|
159
|
+
if (typeof value === 'number')
|
|
160
|
+
return value ? 1 : 0;
|
|
161
|
+
if (typeof value === 'string')
|
|
162
|
+
return value === 'true' || value === '1' ? 1 : 0;
|
|
163
|
+
return value ? 1 : 0;
|
|
164
|
+
}
|
|
165
|
+
if (genericType === 'json') {
|
|
166
|
+
if (typeof value === 'object')
|
|
167
|
+
return JSON.stringify(value);
|
|
168
|
+
return String(value);
|
|
169
|
+
}
|
|
170
|
+
if (genericType === 'datetime') {
|
|
171
|
+
if (value instanceof Date)
|
|
172
|
+
return value.toISOString();
|
|
173
|
+
return String(value);
|
|
174
|
+
}
|
|
175
|
+
if (genericType === 'blob') {
|
|
176
|
+
if (Buffer.isBuffer(value))
|
|
177
|
+
return value;
|
|
178
|
+
if (value instanceof Uint8Array)
|
|
179
|
+
return Buffer.from(value);
|
|
180
|
+
}
|
|
181
|
+
return value;
|
|
182
|
+
}
|
|
183
|
+
function mapDefaultValueForDialect(defaultValue, genericType, targetDialect) {
|
|
184
|
+
if (defaultValue === null || defaultValue === undefined)
|
|
185
|
+
return null;
|
|
186
|
+
let def = String(defaultValue).trim();
|
|
187
|
+
if (!def)
|
|
188
|
+
return null;
|
|
189
|
+
if (targetDialect === 'sqlite') {
|
|
190
|
+
def = def.replace(/::[a-zA-Z0-9_ "]+(\[\])?/g, '').trim();
|
|
191
|
+
if (/nextval\(.*?\)/i.test(def)) {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
if (/^(now\(\)|current_timestamp|current_timestamp\(\)|clock_timestamp\(\)|statement_timestamp\(\)|transaction_timestamp\(\)|\('now'.*?\))$/i.test(def)) {
|
|
195
|
+
return 'CURRENT_TIMESTAMP';
|
|
196
|
+
}
|
|
197
|
+
if (/^(gen_random_uuid\(\)|uuid_generate_v\d\(\)|random\(\)|uuid\(\))$/i.test(def)) {
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
if (/^true$/i.test(def))
|
|
201
|
+
return '1';
|
|
202
|
+
if (/^false$/i.test(def))
|
|
203
|
+
return '0';
|
|
204
|
+
while (def.startsWith('(') && def.endsWith(')')) {
|
|
205
|
+
const inner = def.slice(1, -1).trim();
|
|
206
|
+
let depth = 0;
|
|
207
|
+
let okToStrip = true;
|
|
208
|
+
for (let i = 0; i < inner.length; i++) {
|
|
209
|
+
if (inner[i] === '(')
|
|
210
|
+
depth++;
|
|
211
|
+
else if (inner[i] === ')') {
|
|
212
|
+
depth--;
|
|
213
|
+
if (depth < 0) {
|
|
214
|
+
okToStrip = false;
|
|
215
|
+
break;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (okToStrip && depth === 0) {
|
|
220
|
+
def = inner;
|
|
221
|
+
}
|
|
222
|
+
else {
|
|
223
|
+
break;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (/^(now\(\)|current_timestamp|current_timestamp\(\)|clock_timestamp\(\)|\('now'.*?\))$/i.test(def)) {
|
|
227
|
+
return 'CURRENT_TIMESTAMP';
|
|
228
|
+
}
|
|
229
|
+
if (/^true$/i.test(def))
|
|
230
|
+
return '1';
|
|
231
|
+
if (/^false$/i.test(def))
|
|
232
|
+
return '0';
|
|
233
|
+
if (/nextval\(.*?\)/i.test(def))
|
|
234
|
+
return null;
|
|
235
|
+
if (/^(gen_random_uuid\(\)|uuid_generate_v\d\(\))$/i.test(def))
|
|
236
|
+
return null;
|
|
237
|
+
if (/^[a-zA-Z0-9_]+\(.*\)$/.test(def) && !/^(current_timestamp|current_date|current_time)$/i.test(def)) {
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
return def;
|
|
241
|
+
}
|
|
242
|
+
if (targetDialect === 'postgres') {
|
|
243
|
+
if (genericType === 'boolean') {
|
|
244
|
+
if (def === '1' || def === "'1'")
|
|
245
|
+
return 'TRUE';
|
|
246
|
+
if (def === '0' || def === "'0'")
|
|
247
|
+
return 'FALSE';
|
|
248
|
+
}
|
|
249
|
+
if (/nextval\(.*?\)/i.test(def)) {
|
|
250
|
+
return null;
|
|
251
|
+
}
|
|
252
|
+
return def;
|
|
253
|
+
}
|
|
254
|
+
return def;
|
|
255
|
+
}
|
|
256
|
+
function generateCreateTableForDialect(table, targetDialect) {
|
|
257
|
+
const lines = [];
|
|
258
|
+
const q = targetDialect === 'postgres' ? (s) => `"${s}"` : (s) => `\`${s}\``;
|
|
259
|
+
const singleAutoPk = targetDialect === 'sqlite' &&
|
|
260
|
+
table.columns.filter((c) => c.primaryKey).length === 1 &&
|
|
261
|
+
table.columns.find((c) => c.primaryKey)?.autoIncrement;
|
|
262
|
+
for (const col of table.columns) {
|
|
263
|
+
let colSql = `${q(col.name)} ${mapColumnToDialect(col, targetDialect)}`;
|
|
264
|
+
const isPgSerial = targetDialect === 'postgres' && col.autoIncrement;
|
|
265
|
+
if (singleAutoPk && col.primaryKey) {
|
|
266
|
+
}
|
|
267
|
+
else if (isPgSerial) {
|
|
268
|
+
if (col.unique && !col.primaryKey)
|
|
269
|
+
colSql += ' UNIQUE';
|
|
270
|
+
}
|
|
271
|
+
else {
|
|
272
|
+
if (col.notNull)
|
|
273
|
+
colSql += ' NOT NULL';
|
|
274
|
+
if (col.unique && !col.primaryKey)
|
|
275
|
+
colSql += ' UNIQUE';
|
|
276
|
+
const mappedDefault = mapDefaultValueForDialect(col.defaultValue, col.genericType, targetDialect);
|
|
277
|
+
if (mappedDefault !== null && mappedDefault !== undefined) {
|
|
278
|
+
colSql += ` DEFAULT ${mappedDefault}`;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
lines.push(colSql);
|
|
282
|
+
}
|
|
283
|
+
if (!singleAutoPk && table.primaryKeys.length > 0) {
|
|
284
|
+
const validPks = table.primaryKeys.filter((pk) => table.columns.some((c) => c.name === pk));
|
|
285
|
+
if (validPks.length > 0) {
|
|
286
|
+
lines.push(`PRIMARY KEY (${validPks.map(q).join(', ')})`);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
if (targetDialect === 'sqlite') {
|
|
290
|
+
for (const fk of table.foreignKeys) {
|
|
291
|
+
if (fk.from && fk.to && fk.table) {
|
|
292
|
+
const targetTable = fk.table.includes('.') ? fk.table.split('.').pop() : fk.table;
|
|
293
|
+
let fkSql = `FOREIGN KEY (${q(fk.from)}) REFERENCES ${q(targetTable)} (${q(fk.to)})`;
|
|
294
|
+
if (fk.on_delete && fk.on_delete !== 'NO ACTION')
|
|
295
|
+
fkSql += ` ON DELETE ${fk.on_delete}`;
|
|
296
|
+
if (fk.on_update && fk.on_update !== 'NO ACTION')
|
|
297
|
+
fkSql += ` ON UPDATE ${fk.on_update}`;
|
|
298
|
+
lines.push(fkSql);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
return `CREATE TABLE ${q(table.name)} (\n ${lines.join(',\n ')}\n);`;
|
|
303
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export interface MaskingRule{columnPattern:string;strategy:'email'|'password'|'phone'|'name'|'redact'|'hash';customValue?:string;}export declare const DEFAULT_MASKING_RULES:MaskingRule[];export declare function maskRowData(row:Record<string,unknown>,rules?:MaskingRule[],rowIndex?:number):Record<string,unknown>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_MASKING_RULES = void 0;
|
|
4
|
+
exports.maskRowData = maskRowData;
|
|
5
|
+
exports.DEFAULT_MASKING_RULES = [
|
|
6
|
+
{ columnPattern: 'email|e_mail|mail', strategy: 'email' },
|
|
7
|
+
{ columnPattern: 'pass|password|passwd|pwd|secret|token|api_key|auth', strategy: 'password' },
|
|
8
|
+
{ columnPattern: 'phone|mobile|cell|telephone', strategy: 'phone' },
|
|
9
|
+
{ columnPattern: 'first_name|last_name|full_name|username|user_name', strategy: 'name' },
|
|
10
|
+
];
|
|
11
|
+
function maskRowData(row, rules = exports.DEFAULT_MASKING_RULES, rowIndex = 1) {
|
|
12
|
+
const masked = { ...row };
|
|
13
|
+
for (const [colName, val] of Object.entries(row)) {
|
|
14
|
+
if (val === null || val === undefined)
|
|
15
|
+
continue;
|
|
16
|
+
for (const rule of rules) {
|
|
17
|
+
const regex = new RegExp(rule.columnPattern, 'i');
|
|
18
|
+
if (regex.test(colName)) {
|
|
19
|
+
switch (rule.strategy) {
|
|
20
|
+
case 'email':
|
|
21
|
+
masked[colName] = `user${rowIndex}@masked.test`;
|
|
22
|
+
break;
|
|
23
|
+
case 'password':
|
|
24
|
+
if (/token|secret|key|auth/i.test(colName) && !/pass|pwd/i.test(colName)) {
|
|
25
|
+
masked[colName] = `masked_${rowIndex}_${Math.random().toString(36).substring(2, 10)}${Math.random().toString(36).substring(2, 10)}`;
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
masked[colName] = `$2b$10$maskedpwdhash${String(rowIndex).padStart(6, '0')}${Math.random().toString(36).substring(2, 14)}`;
|
|
29
|
+
}
|
|
30
|
+
break;
|
|
31
|
+
case 'phone':
|
|
32
|
+
masked[colName] = `+1555000${String(rowIndex).padStart(4, '0')}`;
|
|
33
|
+
break;
|
|
34
|
+
case 'name':
|
|
35
|
+
masked[colName] = `Test User ${rowIndex}`;
|
|
36
|
+
break;
|
|
37
|
+
case 'redact':
|
|
38
|
+
masked[colName] = `[REDACTED_${rowIndex}]`;
|
|
39
|
+
break;
|
|
40
|
+
case 'hash':
|
|
41
|
+
masked[colName] = `hash_${rowIndex}_${Math.random().toString(36).substring(2, 8)}`;
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return masked;
|
|
49
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import type {IDatabase} from '../../db/types';import{type MaskingRule} from './masking';export type SyncMode='full'|'schema_only'|'data_only_replace'|'data_only_upsert';export interface SyncOptions{tables?:string[];mode?:SyncMode;maskData?:boolean;maskRules?:MaskingRule[];batchSize?:number;dryRun?:boolean;}export interface SyncTableProgress{table:string;rowsCopied:number;totalRows:number;status:'pending'|'in_progress'|'completed'|'skipped'|'failed';error?:string;}export interface SyncResult{success:boolean;dryRun:boolean;sourceDbId:string;targetDbId:string;mode:SyncMode;tablesProcessed:SyncTableProgress[];totalRowsTransferred:number;executionTimeMs:number;error?:string;summary:string;}export declare function executeSync(sourceDb:IDatabase,sourceDbId:string,targetDb:IDatabase,targetDbId:string,options?:SyncOptions):Promise<SyncResult>;
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.executeSync = executeSync;
|
|
4
|
+
const dialect_mapper_1 = require("./dialect-mapper");
|
|
5
|
+
const topological_sort_1 = require("./topological-sort");
|
|
6
|
+
const masking_1 = require("./masking");
|
|
7
|
+
async function executeSync(sourceDb, sourceDbId, targetDb, targetDbId, options = {}) {
|
|
8
|
+
const startTime = Date.now();
|
|
9
|
+
const mode = options.mode || 'full';
|
|
10
|
+
const batchSize = Math.max(50, options.batchSize || 500);
|
|
11
|
+
const dryRun = Boolean(options.dryRun);
|
|
12
|
+
const maskData = Boolean(options.maskData);
|
|
13
|
+
const sourceDialect = sourceDb.dialect === 'postgres' ? 'postgres' : 'sqlite';
|
|
14
|
+
const targetDialect = targetDb.dialect === 'postgres' ? 'postgres' : 'sqlite';
|
|
15
|
+
const srcTableListRes = await sourceDb.listTables();
|
|
16
|
+
if (!srcTableListRes.success)
|
|
17
|
+
throw new Error(srcTableListRes.error || 'Failed to list source tables');
|
|
18
|
+
let tablesToSync = (srcTableListRes.data || [])
|
|
19
|
+
.map((t) => t.name)
|
|
20
|
+
.filter((n) => !n.startsWith('sqlite_') && !n.startsWith('_admindb_'));
|
|
21
|
+
if (options.tables && options.tables.length > 0) {
|
|
22
|
+
const selected = new Set(options.tables);
|
|
23
|
+
tablesToSync = tablesToSync.filter((n) => selected.has(n));
|
|
24
|
+
}
|
|
25
|
+
const sourceTableInfos = [];
|
|
26
|
+
for (const name of tablesToSync) {
|
|
27
|
+
const infoRes = await sourceDb.getTableInfo(name);
|
|
28
|
+
if (infoRes.success && infoRes.data) {
|
|
29
|
+
sourceTableInfos.push(infoRes.data);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const deps = sourceTableInfos.map((i) => ({
|
|
33
|
+
name: i.table,
|
|
34
|
+
foreignKeys: i.foreignKeys || [],
|
|
35
|
+
}));
|
|
36
|
+
const insertionOrder = (0, topological_sort_1.sortTablesTopologically)(deps);
|
|
37
|
+
const deletionOrder = (0, topological_sort_1.sortTablesForDeletion)(deps);
|
|
38
|
+
const normalizedTables = new Map();
|
|
39
|
+
for (const info of sourceTableInfos) {
|
|
40
|
+
normalizedTables.set(info.table, (0, dialect_mapper_1.normalizeTableInfo)(info, sourceDialect));
|
|
41
|
+
}
|
|
42
|
+
const progressList = insertionOrder.map((name) => ({
|
|
43
|
+
table: name,
|
|
44
|
+
rowsCopied: 0,
|
|
45
|
+
totalRows: 0,
|
|
46
|
+
status: 'pending',
|
|
47
|
+
}));
|
|
48
|
+
if (dryRun) {
|
|
49
|
+
let totalEstimatedRows = 0;
|
|
50
|
+
for (const p of progressList) {
|
|
51
|
+
const countRes = await sourceDb.getRowCount(p.table);
|
|
52
|
+
p.totalRows = countRes.data || 0;
|
|
53
|
+
p.rowsCopied = mode === 'schema_only' ? 0 : p.totalRows;
|
|
54
|
+
p.status = 'completed';
|
|
55
|
+
totalEstimatedRows += p.rowsCopied;
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
success: true,
|
|
59
|
+
dryRun: true,
|
|
60
|
+
sourceDbId,
|
|
61
|
+
targetDbId,
|
|
62
|
+
mode,
|
|
63
|
+
tablesProcessed: progressList,
|
|
64
|
+
totalRowsTransferred: totalEstimatedRows,
|
|
65
|
+
executionTimeMs: Date.now() - startTime,
|
|
66
|
+
summary: `Dry Run: Ready to sync ${progressList.length} tables (${totalEstimatedRows} estimated rows) from "${sourceDbId}" to "${targetDbId}".`,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
let totalRowsTransferred = 0;
|
|
70
|
+
try {
|
|
71
|
+
if (targetDialect === 'sqlite') {
|
|
72
|
+
await targetDb.execResult('PRAGMA foreign_keys = OFF;');
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
try {
|
|
76
|
+
await targetDb.execResult('SET CONSTRAINTS ALL DEFERRED;');
|
|
77
|
+
}
|
|
78
|
+
catch { }
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
catch { }
|
|
82
|
+
try {
|
|
83
|
+
if (mode === 'full') {
|
|
84
|
+
for (const tableName of deletionOrder) {
|
|
85
|
+
try {
|
|
86
|
+
await targetDb.execResult(targetDialect === 'postgres'
|
|
87
|
+
? `DROP TABLE IF EXISTS "${tableName}" CASCADE;`
|
|
88
|
+
: `DROP TABLE IF EXISTS \`${tableName}\`;`);
|
|
89
|
+
}
|
|
90
|
+
catch { }
|
|
91
|
+
}
|
|
92
|
+
for (const tableName of insertionOrder) {
|
|
93
|
+
const norm = normalizedTables.get(tableName);
|
|
94
|
+
if (!norm)
|
|
95
|
+
continue;
|
|
96
|
+
const ddl = (0, dialect_mapper_1.generateCreateTableForDialect)(norm, targetDialect);
|
|
97
|
+
const createRes = await targetDb.execResult(ddl);
|
|
98
|
+
if (!createRes.success) {
|
|
99
|
+
throw new Error(`Failed to create table "${tableName}": ${createRes.error}`);
|
|
100
|
+
}
|
|
101
|
+
for (const idx of norm.indexes || []) {
|
|
102
|
+
if (idx.name?.startsWith('sqlite_'))
|
|
103
|
+
continue;
|
|
104
|
+
const colsArr = Array.isArray(idx?.columns) ? idx.columns.filter(Boolean) : [];
|
|
105
|
+
if (colsArr.length === 0)
|
|
106
|
+
continue;
|
|
107
|
+
const uq = idx.unique ? 'UNIQUE ' : '';
|
|
108
|
+
const q = targetDialect === 'postgres' ? (s) => `"${s}"` : (s) => `\`${s}\``;
|
|
109
|
+
const cols = colsArr.map(q).join(', ');
|
|
110
|
+
try {
|
|
111
|
+
await targetDb.execResult(`CREATE ${uq}INDEX IF NOT EXISTS ${q(idx.name)} ON ${q(tableName)} (${cols});`);
|
|
112
|
+
}
|
|
113
|
+
catch { }
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
else if (mode === 'schema_only') {
|
|
118
|
+
for (const tableName of insertionOrder) {
|
|
119
|
+
const norm = normalizedTables.get(tableName);
|
|
120
|
+
if (!norm)
|
|
121
|
+
continue;
|
|
122
|
+
const targetInfo = await targetDb.getTableInfo(tableName);
|
|
123
|
+
if (!targetInfo.success || !targetInfo.data || targetInfo.data.columns.length === 0) {
|
|
124
|
+
const ddl = (0, dialect_mapper_1.generateCreateTableForDialect)(norm, targetDialect);
|
|
125
|
+
const createRes = await targetDb.execResult(ddl);
|
|
126
|
+
if (!createRes.success) {
|
|
127
|
+
throw new Error(`Failed to create table "${tableName}": ${createRes.error}`);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
else if (mode === 'data_only_replace') {
|
|
133
|
+
for (const tableName of deletionOrder) {
|
|
134
|
+
try {
|
|
135
|
+
await targetDb.execResult(targetDialect === 'postgres' ? `TRUNCATE TABLE "${tableName}" CASCADE;` : `DELETE FROM \`${tableName}\`;`);
|
|
136
|
+
}
|
|
137
|
+
catch { }
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (mode !== 'schema_only') {
|
|
141
|
+
for (const p of progressList) {
|
|
142
|
+
const tableName = p.table;
|
|
143
|
+
p.status = 'in_progress';
|
|
144
|
+
const norm = normalizedTables.get(tableName);
|
|
145
|
+
if (!norm)
|
|
146
|
+
continue;
|
|
147
|
+
const countRes = await sourceDb.getRowCount(tableName);
|
|
148
|
+
p.totalRows = countRes.data || 0;
|
|
149
|
+
let offset = 0;
|
|
150
|
+
let tableRowsCopied = 0;
|
|
151
|
+
while (offset < p.totalRows || (offset === 0 && p.totalRows === 0)) {
|
|
152
|
+
if (p.totalRows === 0)
|
|
153
|
+
break;
|
|
154
|
+
const rowsRes = await sourceDb.getRows(tableName, { page: Math.floor(offset / batchSize) + 1, limit: batchSize });
|
|
155
|
+
const rows = rowsRes.data || [];
|
|
156
|
+
if (rows.length === 0)
|
|
157
|
+
break;
|
|
158
|
+
const targetRows = [];
|
|
159
|
+
for (let i = 0; i < rows.length; i++) {
|
|
160
|
+
let row = rows[i];
|
|
161
|
+
if (maskData) {
|
|
162
|
+
row = (0, masking_1.maskRowData)(row, options.maskRules, offset + i + 1);
|
|
163
|
+
}
|
|
164
|
+
const fields = [];
|
|
165
|
+
for (const col of norm.columns) {
|
|
166
|
+
const rawVal = row[col.name];
|
|
167
|
+
const convertedVal = (0, dialect_mapper_1.convertValueForDialect)(rawVal, col.genericType, targetDialect);
|
|
168
|
+
fields.push({ column: col.name, value: convertedVal });
|
|
169
|
+
}
|
|
170
|
+
targetRows.push(fields);
|
|
171
|
+
}
|
|
172
|
+
const insertRes = await targetDb.insertRows(tableName, targetRows);
|
|
173
|
+
if (!insertRes.success) {
|
|
174
|
+
throw new Error(`Failed to insert batch into "${tableName}": ${insertRes.error}`);
|
|
175
|
+
}
|
|
176
|
+
tableRowsCopied += targetRows.length;
|
|
177
|
+
offset += rows.length;
|
|
178
|
+
}
|
|
179
|
+
if (targetDialect === 'postgres') {
|
|
180
|
+
for (const col of norm.columns) {
|
|
181
|
+
if (col.primaryKey && col.autoIncrement) {
|
|
182
|
+
try {
|
|
183
|
+
await targetDb.all(`SELECT setval(pg_get_serial_sequence('"${tableName}"', '${col.name}'), COALESCE(MAX("${col.name}"), 1)) FROM "${tableName}";`);
|
|
184
|
+
}
|
|
185
|
+
catch { }
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
p.rowsCopied = tableRowsCopied;
|
|
190
|
+
totalRowsTransferred += tableRowsCopied;
|
|
191
|
+
p.status = 'completed';
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
catch (err) {
|
|
196
|
+
return {
|
|
197
|
+
success: false,
|
|
198
|
+
dryRun: false,
|
|
199
|
+
sourceDbId,
|
|
200
|
+
targetDbId,
|
|
201
|
+
mode,
|
|
202
|
+
tablesProcessed: progressList,
|
|
203
|
+
totalRowsTransferred,
|
|
204
|
+
executionTimeMs: Date.now() - startTime,
|
|
205
|
+
error: err.message || String(err),
|
|
206
|
+
summary: `Sync failed: ${err.message}`,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
finally {
|
|
210
|
+
try {
|
|
211
|
+
if (targetDialect === 'sqlite') {
|
|
212
|
+
await targetDb.execResult('PRAGMA foreign_keys = ON;');
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
try {
|
|
216
|
+
await targetDb.execResult('SET CONSTRAINTS ALL IMMEDIATE;');
|
|
217
|
+
}
|
|
218
|
+
catch { }
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
catch { }
|
|
222
|
+
}
|
|
223
|
+
return {
|
|
224
|
+
success: true,
|
|
225
|
+
dryRun: false,
|
|
226
|
+
sourceDbId,
|
|
227
|
+
targetDbId,
|
|
228
|
+
mode,
|
|
229
|
+
tablesProcessed: progressList,
|
|
230
|
+
totalRowsTransferred,
|
|
231
|
+
executionTimeMs: Date.now() - startTime,
|
|
232
|
+
summary: `Successfully synchronized ${progressList.length} tables (${totalRowsTransferred} rows) from "${sourceDbId}" to "${targetDbId}".`,
|
|
233
|
+
};
|
|
234
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import type {ForeignKeyInfo} from '../../db/types';export interface TableDependency{name:string;foreignKeys:ForeignKeyInfo[];}export declare function sortTablesTopologically(tables:TableDependency[]):string[];export declare function sortTablesForDeletion(tables:TableDependency[]):string[];
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.sortTablesTopologically = sortTablesTopologically;
|
|
4
|
+
exports.sortTablesForDeletion = sortTablesForDeletion;
|
|
5
|
+
function sortTablesTopologically(tables) {
|
|
6
|
+
const tableNames = new Set(tables.map((t) => t.name));
|
|
7
|
+
const adj = new Map();
|
|
8
|
+
const inDegree = new Map();
|
|
9
|
+
for (const t of tables) {
|
|
10
|
+
if (!adj.has(t.name))
|
|
11
|
+
adj.set(t.name, new Set());
|
|
12
|
+
if (!inDegree.has(t.name))
|
|
13
|
+
inDegree.set(t.name, 0);
|
|
14
|
+
}
|
|
15
|
+
for (const t of tables) {
|
|
16
|
+
for (const fk of t.foreignKeys) {
|
|
17
|
+
const parent = fk.table;
|
|
18
|
+
if (parent && tableNames.has(parent) && parent !== t.name) {
|
|
19
|
+
if (!adj.get(parent).has(t.name)) {
|
|
20
|
+
adj.get(parent).add(t.name);
|
|
21
|
+
inDegree.set(t.name, (inDegree.get(t.name) || 0) + 1);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
const queue = [];
|
|
27
|
+
for (const [name, deg] of inDegree.entries()) {
|
|
28
|
+
if (deg === 0)
|
|
29
|
+
queue.push(name);
|
|
30
|
+
}
|
|
31
|
+
const result = [];
|
|
32
|
+
while (queue.length > 0) {
|
|
33
|
+
const current = queue.shift();
|
|
34
|
+
result.push(current);
|
|
35
|
+
const children = adj.get(current) || new Set();
|
|
36
|
+
for (const child of children) {
|
|
37
|
+
const newDeg = (inDegree.get(child) || 0) - 1;
|
|
38
|
+
inDegree.set(child, newDeg);
|
|
39
|
+
if (newDeg === 0) {
|
|
40
|
+
queue.push(child);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (result.length < tables.length) {
|
|
45
|
+
for (const t of tables) {
|
|
46
|
+
if (!result.includes(t.name)) {
|
|
47
|
+
result.push(t.name);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return result;
|
|
52
|
+
}
|
|
53
|
+
function sortTablesForDeletion(tables) {
|
|
54
|
+
return [...sortTablesTopologically(tables)].reverse();
|
|
55
|
+
}
|