dbgate-api-premium 6.3.3 → 6.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -7
- package/src/controllers/archive.js +99 -6
- package/src/controllers/config.js +135 -22
- package/src/controllers/connections.js +35 -2
- package/src/controllers/databaseConnections.js +76 -1
- package/src/controllers/files.js +59 -0
- package/src/controllers/jsldata.js +9 -0
- package/src/controllers/runners.js +25 -5
- package/src/controllers/serverConnections.js +17 -2
- package/src/controllers/storage.js +51 -1
- package/src/controllers/uploads.js +0 -46
- package/src/currentVersion.js +2 -2
- package/src/proc/connectProcess.js +14 -2
- package/src/proc/databaseConnectionProcess.js +70 -5
- package/src/proc/serverConnectionProcess.js +7 -1
- package/src/proc/sessionProcess.js +15 -178
- package/src/shell/archiveReader.js +3 -1
- package/src/shell/collectorWriter.js +2 -2
- package/src/shell/copyStream.js +1 -0
- package/src/shell/dataReplicator.js +96 -0
- package/src/shell/download.js +22 -6
- package/src/shell/index.js +12 -2
- package/src/shell/jsonLinesWriter.js +4 -3
- package/src/shell/queryReader.js +10 -3
- package/src/shell/unzipDirectory.js +91 -0
- package/src/shell/unzipJsonLinesData.js +60 -0
- package/src/shell/unzipJsonLinesFile.js +59 -0
- package/src/shell/zipDirectory.js +49 -0
- package/src/shell/zipJsonLinesData.js +49 -0
- package/src/utility/cloudUpgrade.js +14 -1
- package/src/utility/crypting.js +56 -5
- package/src/utility/extractSingleFileFromZip.js +77 -0
- package/src/utility/handleQueryStream.js +186 -0
- package/src/utility/listZipEntries.js +41 -0
- package/src/utility/storageReplicatorItems.js +88 -0
- package/src/shell/dataDuplicator.js +0 -61
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dbgate-api-premium",
|
|
3
3
|
"main": "src/index.js",
|
|
4
|
-
"version": "6.
|
|
4
|
+
"version": "6.4.0",
|
|
5
5
|
"homepage": "https://dbgate.org/",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
"dependencies": {
|
|
23
23
|
"@aws-sdk/rds-signer": "^3.665.0",
|
|
24
24
|
"activedirectory2": "^2.1.0",
|
|
25
|
+
"archiver": "^7.0.1",
|
|
25
26
|
"async-lock": "^1.2.6",
|
|
26
27
|
"axios": "^0.21.1",
|
|
27
28
|
"body-parser": "^1.19.0",
|
|
@@ -29,10 +30,10 @@
|
|
|
29
30
|
"compare-versions": "^3.6.0",
|
|
30
31
|
"cors": "^2.8.5",
|
|
31
32
|
"cross-env": "^6.0.3",
|
|
32
|
-
"dbgate-datalib": "^6.
|
|
33
|
-
"dbgate-query-splitter": "^4.11.
|
|
34
|
-
"dbgate-sqltree": "^6.
|
|
35
|
-
"dbgate-tools": "^6.
|
|
33
|
+
"dbgate-datalib": "^6.4.0",
|
|
34
|
+
"dbgate-query-splitter": "^4.11.4",
|
|
35
|
+
"dbgate-sqltree": "^6.4.0",
|
|
36
|
+
"dbgate-tools": "^6.4.0",
|
|
36
37
|
"debug": "^4.3.4",
|
|
37
38
|
"diff": "^5.0.0",
|
|
38
39
|
"diff2html": "^3.4.13",
|
|
@@ -62,7 +63,8 @@
|
|
|
62
63
|
"simple-encryptor": "^4.0.0",
|
|
63
64
|
"ssh2": "^1.16.0",
|
|
64
65
|
"stream-json": "^1.8.0",
|
|
65
|
-
"tar": "^6.0.5"
|
|
66
|
+
"tar": "^6.0.5",
|
|
67
|
+
"yauzl": "^3.2.0"
|
|
66
68
|
},
|
|
67
69
|
"scripts": {
|
|
68
70
|
"start": "env-cmd -f .env node src/index.js --listen-api",
|
|
@@ -83,7 +85,7 @@
|
|
|
83
85
|
"devDependencies": {
|
|
84
86
|
"@types/fs-extra": "^9.0.11",
|
|
85
87
|
"@types/lodash": "^4.14.149",
|
|
86
|
-
"dbgate-types": "^6.
|
|
88
|
+
"dbgate-types": "^6.4.0",
|
|
87
89
|
"env-cmd": "^10.1.0",
|
|
88
90
|
"jsdoc-to-markdown": "^9.0.5",
|
|
89
91
|
"node-loader": "^1.0.2",
|
|
@@ -2,14 +2,20 @@ const fs = require('fs-extra');
|
|
|
2
2
|
const readline = require('readline');
|
|
3
3
|
const crypto = require('crypto');
|
|
4
4
|
const path = require('path');
|
|
5
|
-
const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories');
|
|
5
|
+
const { archivedir, clearArchiveLinksCache, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
|
|
6
6
|
const socket = require('../utility/socket');
|
|
7
7
|
const loadFilesRecursive = require('../utility/loadFilesRecursive');
|
|
8
8
|
const getJslFileName = require('../utility/getJslFileName');
|
|
9
|
-
const { getLogger, extractErrorLogData } = require('dbgate-tools');
|
|
9
|
+
const { getLogger, extractErrorLogData, jsonLinesParse } = require('dbgate-tools');
|
|
10
10
|
const dbgateApi = require('../shell');
|
|
11
11
|
const jsldata = require('./jsldata');
|
|
12
12
|
const platformInfo = require('../utility/platformInfo');
|
|
13
|
+
const { isProApp } = require('../utility/checkLicense');
|
|
14
|
+
const listZipEntries = require('../utility/listZipEntries');
|
|
15
|
+
const unzipJsonLinesFile = require('../shell/unzipJsonLinesFile');
|
|
16
|
+
const { zip } = require('lodash');
|
|
17
|
+
const zipDirectory = require('../shell/zipDirectory');
|
|
18
|
+
const unzipDirectory = require('../shell/unzipDirectory');
|
|
13
19
|
|
|
14
20
|
const logger = getLogger('archive');
|
|
15
21
|
|
|
@@ -47,9 +53,31 @@ module.exports = {
|
|
|
47
53
|
return folder;
|
|
48
54
|
},
|
|
49
55
|
|
|
56
|
+
async getZipFiles({ file }) {
|
|
57
|
+
const entries = await listZipEntries(path.join(archivedir(), file));
|
|
58
|
+
const files = entries.map(entry => {
|
|
59
|
+
let name = entry.fileName;
|
|
60
|
+
if (isProApp() && entry.fileName.endsWith('.jsonl')) {
|
|
61
|
+
name = entry.fileName.slice(0, -6);
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
name: name,
|
|
65
|
+
label: name,
|
|
66
|
+
type: isProApp() && entry.fileName.endsWith('.jsonl') ? 'jsonl' : 'other',
|
|
67
|
+
};
|
|
68
|
+
});
|
|
69
|
+
return files;
|
|
70
|
+
},
|
|
71
|
+
|
|
50
72
|
files_meta: true,
|
|
51
73
|
async files({ folder }) {
|
|
52
74
|
try {
|
|
75
|
+
if (folder.endsWith('.zip')) {
|
|
76
|
+
if (await fs.exists(path.join(archivedir(), folder))) {
|
|
77
|
+
return this.getZipFiles({ file: folder });
|
|
78
|
+
}
|
|
79
|
+
return [];
|
|
80
|
+
}
|
|
53
81
|
const dir = resolveArchiveFolder(folder);
|
|
54
82
|
if (!(await fs.exists(dir))) return [];
|
|
55
83
|
const files = await loadFilesRecursive(dir); // fs.readdir(dir);
|
|
@@ -91,6 +119,16 @@ module.exports = {
|
|
|
91
119
|
return true;
|
|
92
120
|
},
|
|
93
121
|
|
|
122
|
+
createFile_meta: true,
|
|
123
|
+
async createFile({ folder, file, fileType, tableInfo }) {
|
|
124
|
+
await fs.writeFile(
|
|
125
|
+
path.join(resolveArchiveFolder(folder), `${file}.${fileType}`),
|
|
126
|
+
tableInfo ? JSON.stringify({ __isStreamHeader: true, tableInfo }) : ''
|
|
127
|
+
);
|
|
128
|
+
socket.emitChanged(`archive-files-changed`, { folder });
|
|
129
|
+
return true;
|
|
130
|
+
},
|
|
131
|
+
|
|
94
132
|
deleteFile_meta: true,
|
|
95
133
|
async deleteFile({ folder, file, fileType }) {
|
|
96
134
|
await fs.unlink(path.join(resolveArchiveFolder(folder), `${file}.${fileType}`));
|
|
@@ -158,7 +196,7 @@ module.exports = {
|
|
|
158
196
|
deleteFolder_meta: true,
|
|
159
197
|
async deleteFolder({ folder }) {
|
|
160
198
|
if (!folder) throw new Error('Missing folder parameter');
|
|
161
|
-
if (folder.endsWith('.link')) {
|
|
199
|
+
if (folder.endsWith('.link') || folder.endsWith('.zip')) {
|
|
162
200
|
await fs.unlink(path.join(archivedir(), folder));
|
|
163
201
|
} else {
|
|
164
202
|
await fs.rmdir(path.join(archivedir(), folder), { recursive: true });
|
|
@@ -204,9 +242,10 @@ module.exports = {
|
|
|
204
242
|
},
|
|
205
243
|
|
|
206
244
|
async getNewArchiveFolder({ database }) {
|
|
207
|
-
const isLink = database.endsWith(
|
|
208
|
-
const
|
|
209
|
-
const
|
|
245
|
+
const isLink = database.endsWith('.link');
|
|
246
|
+
const isZip = database.endsWith('.zip');
|
|
247
|
+
const name = isLink ? database.slice(0, -5) : isZip ? database.slice(0, -4) : database;
|
|
248
|
+
const suffix = isLink ? '.link' : isZip ? '.zip' : '';
|
|
210
249
|
if (!(await fs.exists(path.join(archivedir(), database)))) return database;
|
|
211
250
|
let index = 2;
|
|
212
251
|
while (await fs.exists(path.join(archivedir(), `${name}${index}${suffix}`))) {
|
|
@@ -214,4 +253,58 @@ module.exports = {
|
|
|
214
253
|
}
|
|
215
254
|
return `${name}${index}${suffix}`;
|
|
216
255
|
},
|
|
256
|
+
|
|
257
|
+
getArchiveData_meta: true,
|
|
258
|
+
async getArchiveData({ folder, file }) {
|
|
259
|
+
let rows;
|
|
260
|
+
if (folder.endsWith('.zip')) {
|
|
261
|
+
rows = await unzipJsonLinesFile(path.join(archivedir(), folder), `${file}.jsonl`);
|
|
262
|
+
} else {
|
|
263
|
+
rows = jsonLinesParse(await fs.readFile(path.join(archivedir(), folder, `${file}.jsonl`), { encoding: 'utf8' }));
|
|
264
|
+
}
|
|
265
|
+
return rows.filter(x => !x.__isStreamHeader);
|
|
266
|
+
},
|
|
267
|
+
|
|
268
|
+
saveUploadedZip_meta: true,
|
|
269
|
+
async saveUploadedZip({ filePath, fileName }) {
|
|
270
|
+
if (!fileName?.endsWith('.zip')) {
|
|
271
|
+
throw new Error(`${fileName} is not a ZIP file`);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
const folder = await this.getNewArchiveFolder({ database: fileName });
|
|
275
|
+
await fs.copyFile(filePath, path.join(archivedir(), folder));
|
|
276
|
+
socket.emitChanged(`archive-folders-changed`);
|
|
277
|
+
|
|
278
|
+
return null;
|
|
279
|
+
},
|
|
280
|
+
|
|
281
|
+
zip_meta: true,
|
|
282
|
+
async zip({ folder }) {
|
|
283
|
+
const newFolder = await this.getNewArchiveFolder({ database: folder + '.zip' });
|
|
284
|
+
await zipDirectory(path.join(archivedir(), folder), path.join(archivedir(), newFolder));
|
|
285
|
+
socket.emitChanged(`archive-folders-changed`);
|
|
286
|
+
|
|
287
|
+
return null;
|
|
288
|
+
},
|
|
289
|
+
|
|
290
|
+
unzip_meta: true,
|
|
291
|
+
async unzip({ folder }) {
|
|
292
|
+
const newFolder = await this.getNewArchiveFolder({ database: folder.slice(0, -4) });
|
|
293
|
+
await unzipDirectory(path.join(archivedir(), folder), path.join(archivedir(), newFolder));
|
|
294
|
+
socket.emitChanged(`archive-folders-changed`);
|
|
295
|
+
|
|
296
|
+
return null;
|
|
297
|
+
},
|
|
298
|
+
|
|
299
|
+
getZippedPath_meta: true,
|
|
300
|
+
async getZippedPath({ folder }) {
|
|
301
|
+
if (folder.endsWith('.zip')) {
|
|
302
|
+
return { filePath: path.join(archivedir(), folder) };
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
const uploadName = crypto.randomUUID();
|
|
306
|
+
const filePath = path.join(uploadsdir(), uploadName);
|
|
307
|
+
await zipDirectory(path.join(archivedir(), folder), filePath);
|
|
308
|
+
return { filePath };
|
|
309
|
+
},
|
|
217
310
|
};
|
|
@@ -19,6 +19,14 @@ const storage = require('./storage');
|
|
|
19
19
|
const { getAuthProxyUrl } = require('../utility/authProxy');
|
|
20
20
|
const { getPublicHardwareFingerprint } = require('../utility/hardwareFingerprint');
|
|
21
21
|
const { extractErrorMessage } = require('dbgate-tools');
|
|
22
|
+
const {
|
|
23
|
+
generateTransportEncryptionKey,
|
|
24
|
+
createTransportEncryptor,
|
|
25
|
+
recryptConnection,
|
|
26
|
+
getInternalEncryptor,
|
|
27
|
+
recryptUser,
|
|
28
|
+
recryptObjectPasswordFieldInPlace,
|
|
29
|
+
} = require('../utility/crypting');
|
|
22
30
|
|
|
23
31
|
const lock = new AsyncLock();
|
|
24
32
|
|
|
@@ -107,6 +115,7 @@ module.exports = {
|
|
|
107
115
|
datadir(),
|
|
108
116
|
processArgs.runE2eTests ? 'connections-e2etests.jsonl' : 'connections.jsonl'
|
|
109
117
|
),
|
|
118
|
+
supportCloudAutoUpgrade: !!process.env.CLOUD_UPGRADE_FILE,
|
|
110
119
|
...currentVersion,
|
|
111
120
|
};
|
|
112
121
|
|
|
@@ -144,7 +153,7 @@ module.exports = {
|
|
|
144
153
|
const res = {
|
|
145
154
|
...value,
|
|
146
155
|
};
|
|
147
|
-
if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
|
|
156
|
+
if (platformInfo.isElectron && value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
|
|
148
157
|
// res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
|
|
149
158
|
res['app.useNativeMenu'] = false;
|
|
150
159
|
}
|
|
@@ -161,14 +170,19 @@ module.exports = {
|
|
|
161
170
|
|
|
162
171
|
async loadSettings() {
|
|
163
172
|
try {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
173
|
+
if (process.env.STORAGE_DATABASE) {
|
|
174
|
+
const settings = await storage.readConfig({ group: 'settings' });
|
|
175
|
+
return this.fillMissingSettings(settings);
|
|
176
|
+
} else {
|
|
177
|
+
const settingsText = await fs.readFile(
|
|
178
|
+
path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
|
|
179
|
+
{ encoding: 'utf-8' }
|
|
180
|
+
);
|
|
181
|
+
return {
|
|
182
|
+
...this.fillMissingSettings(JSON.parse(settingsText)),
|
|
183
|
+
'other.licenseKey': platformInfo.isElectron ? await this.loadLicenseKey() : undefined,
|
|
184
|
+
};
|
|
185
|
+
}
|
|
172
186
|
} catch (err) {
|
|
173
187
|
return this.fillMissingSettings({});
|
|
174
188
|
}
|
|
@@ -246,19 +260,31 @@ module.exports = {
|
|
|
246
260
|
const res = await lock.acquire('settings', async () => {
|
|
247
261
|
const currentValue = await this.loadSettings();
|
|
248
262
|
try {
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
263
|
+
let updated = currentValue;
|
|
264
|
+
if (process.env.STORAGE_DATABASE) {
|
|
265
|
+
updated = {
|
|
266
|
+
...currentValue,
|
|
267
|
+
...values,
|
|
268
|
+
};
|
|
269
|
+
await storage.writeConfig({
|
|
270
|
+
group: 'settings',
|
|
271
|
+
config: updated,
|
|
272
|
+
});
|
|
273
|
+
} else {
|
|
274
|
+
updated = {
|
|
275
|
+
...currentValue,
|
|
276
|
+
..._.omit(values, ['other.licenseKey']),
|
|
277
|
+
};
|
|
278
|
+
await fs.writeFile(
|
|
279
|
+
path.join(datadir(), processArgs.runE2eTests ? 'settings-e2etests.json' : 'settings.json'),
|
|
280
|
+
JSON.stringify(updated, undefined, 2)
|
|
281
|
+
);
|
|
282
|
+
// this.settingsValue = updated;
|
|
283
|
+
|
|
284
|
+
if (currentValue['other.licenseKey'] != values['other.licenseKey']) {
|
|
285
|
+
await this.saveLicenseKey({ licenseKey: values['other.licenseKey'] });
|
|
286
|
+
socket.emitChanged(`config-changed`);
|
|
287
|
+
}
|
|
262
288
|
}
|
|
263
289
|
|
|
264
290
|
socket.emitChanged(`settings-changed`);
|
|
@@ -281,4 +307,91 @@ module.exports = {
|
|
|
281
307
|
const resp = await checkLicenseKey(licenseKey);
|
|
282
308
|
return resp;
|
|
283
309
|
},
|
|
310
|
+
|
|
311
|
+
recryptDatabaseForExport(db) {
|
|
312
|
+
const encryptionKey = generateTransportEncryptionKey();
|
|
313
|
+
const transportEncryptor = createTransportEncryptor(encryptionKey);
|
|
314
|
+
|
|
315
|
+
const config = _.cloneDeep([
|
|
316
|
+
...(db.config?.filter(c => !(c.group == 'admin' && c.key == 'encryptionKey')) || []),
|
|
317
|
+
{ group: 'admin', key: 'encryptionKey', value: encryptionKey },
|
|
318
|
+
]);
|
|
319
|
+
const adminPassword = config.find(c => c.group == 'admin' && c.key == 'adminPassword');
|
|
320
|
+
recryptObjectPasswordFieldInPlace(adminPassword, 'value', getInternalEncryptor(), transportEncryptor);
|
|
321
|
+
|
|
322
|
+
return {
|
|
323
|
+
...db,
|
|
324
|
+
connections: db.connections?.map(conn => recryptConnection(conn, getInternalEncryptor(), transportEncryptor)),
|
|
325
|
+
users: db.users?.map(conn => recryptUser(conn, getInternalEncryptor(), transportEncryptor)),
|
|
326
|
+
config,
|
|
327
|
+
};
|
|
328
|
+
},
|
|
329
|
+
|
|
330
|
+
recryptDatabaseFromImport(db) {
|
|
331
|
+
const encryptionKey = db.config?.find(c => c.group == 'admin' && c.key == 'encryptionKey')?.value;
|
|
332
|
+
if (!encryptionKey) {
|
|
333
|
+
throw new Error('Missing encryption key in the database');
|
|
334
|
+
}
|
|
335
|
+
const config = _.cloneDeep(db.config || []).filter(c => !(c.group == 'admin' && c.key == 'encryptionKey'));
|
|
336
|
+
const transportEncryptor = createTransportEncryptor(encryptionKey);
|
|
337
|
+
|
|
338
|
+
const adminPassword = config.find(c => c.group == 'admin' && c.key == 'adminPassword');
|
|
339
|
+
recryptObjectPasswordFieldInPlace(adminPassword, 'value', transportEncryptor, getInternalEncryptor());
|
|
340
|
+
|
|
341
|
+
return {
|
|
342
|
+
...db,
|
|
343
|
+
connections: db.connections?.map(conn => recryptConnection(conn, transportEncryptor, getInternalEncryptor())),
|
|
344
|
+
users: db.users?.map(conn => recryptUser(conn, transportEncryptor, getInternalEncryptor())),
|
|
345
|
+
config,
|
|
346
|
+
};
|
|
347
|
+
},
|
|
348
|
+
|
|
349
|
+
exportConnectionsAndSettings_meta: true,
|
|
350
|
+
async exportConnectionsAndSettings(_params, req) {
|
|
351
|
+
if (!hasPermission(`admin/config`, req)) {
|
|
352
|
+
throw new Error('Permission denied: admin/config');
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
if (connections.portalConnections) {
|
|
356
|
+
throw new Error('Not allowed');
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
if (process.env.STORAGE_DATABASE) {
|
|
360
|
+
const db = await storage.getExportedDatabase();
|
|
361
|
+
return this.recryptDatabaseForExport(db);
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
return this.recryptDatabaseForExport({
|
|
365
|
+
connections: (await connections.list(null, req)).map((conn, index) => ({
|
|
366
|
+
..._.omit(conn, ['_id']),
|
|
367
|
+
id: index + 1,
|
|
368
|
+
conid: conn._id,
|
|
369
|
+
})),
|
|
370
|
+
});
|
|
371
|
+
},
|
|
372
|
+
|
|
373
|
+
importConnectionsAndSettings_meta: true,
|
|
374
|
+
async importConnectionsAndSettings({ db }, req) {
|
|
375
|
+
if (!hasPermission(`admin/config`, req)) {
|
|
376
|
+
throw new Error('Permission denied: admin/config');
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
if (connections.portalConnections) {
|
|
380
|
+
throw new Error('Not allowed');
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
const recryptedDb = this.recryptDatabaseFromImport(db);
|
|
384
|
+
if (process.env.STORAGE_DATABASE) {
|
|
385
|
+
await storage.replicateImportedDatabase(recryptedDb);
|
|
386
|
+
} else {
|
|
387
|
+
await connections.importFromArray(
|
|
388
|
+
recryptedDb.connections.map(conn => ({
|
|
389
|
+
..._.omit(conn, ['conid', 'id']),
|
|
390
|
+
_id: conn.conid,
|
|
391
|
+
}))
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
return true;
|
|
396
|
+
},
|
|
284
397
|
};
|
|
@@ -38,6 +38,11 @@ function getNamedArgs() {
|
|
|
38
38
|
res.databaseFile = name;
|
|
39
39
|
res.engine = 'sqlite@dbgate-plugin-sqlite';
|
|
40
40
|
}
|
|
41
|
+
|
|
42
|
+
if (name.endsWith('.duckdb')) {
|
|
43
|
+
res.databaseFile = name;
|
|
44
|
+
res.engine = 'duckdb@dbgate-plugin-duckdb';
|
|
45
|
+
}
|
|
41
46
|
}
|
|
42
47
|
}
|
|
43
48
|
return res;
|
|
@@ -102,8 +107,8 @@ function getPortalCollections() {
|
|
|
102
107
|
trustServerCertificate: process.env[`SSL_TRUST_CERTIFICATE_${id}`],
|
|
103
108
|
}));
|
|
104
109
|
|
|
105
|
-
for(const conn of connections) {
|
|
106
|
-
for(const prop in process.env) {
|
|
110
|
+
for (const conn of connections) {
|
|
111
|
+
for (const prop in process.env) {
|
|
107
112
|
if (prop.startsWith(`CONNECTION_${conn._id}_`)) {
|
|
108
113
|
const name = prop.substring(`CONNECTION_${conn._id}_`.length);
|
|
109
114
|
conn[name] = process.env[prop];
|
|
@@ -316,6 +321,18 @@ module.exports = {
|
|
|
316
321
|
return res;
|
|
317
322
|
},
|
|
318
323
|
|
|
324
|
+
importFromArray(list) {
|
|
325
|
+
this.datastore.transformAll(connections => {
|
|
326
|
+
const mapped = connections.map(x => {
|
|
327
|
+
const found = list.find(y => y._id == x._id);
|
|
328
|
+
if (found) return found;
|
|
329
|
+
return x;
|
|
330
|
+
});
|
|
331
|
+
return [...mapped, ...list.filter(x => !connections.find(y => y._id == x._id))];
|
|
332
|
+
});
|
|
333
|
+
socket.emitChanged('connection-list-changed');
|
|
334
|
+
},
|
|
335
|
+
|
|
319
336
|
async checkUnsavedConnectionsLimit() {
|
|
320
337
|
if (!this.datastore) {
|
|
321
338
|
return;
|
|
@@ -435,6 +452,22 @@ module.exports = {
|
|
|
435
452
|
return res;
|
|
436
453
|
},
|
|
437
454
|
|
|
455
|
+
newDuckdbDatabase_meta: true,
|
|
456
|
+
async newDuckdbDatabase({ file }) {
|
|
457
|
+
const duckdbDir = path.join(filesdir(), 'duckdb');
|
|
458
|
+
if (!(await fs.exists(duckdbDir))) {
|
|
459
|
+
await fs.mkdir(duckdbDir);
|
|
460
|
+
}
|
|
461
|
+
const databaseFile = path.join(duckdbDir, `${file}.duckdb`);
|
|
462
|
+
const res = await this.save({
|
|
463
|
+
engine: 'duckdb@dbgate-plugin-duckdb',
|
|
464
|
+
databaseFile,
|
|
465
|
+
singleDatabase: true,
|
|
466
|
+
defaultDatabase: `${file}.duckdb`,
|
|
467
|
+
});
|
|
468
|
+
return res;
|
|
469
|
+
},
|
|
470
|
+
|
|
438
471
|
dbloginWeb_meta: {
|
|
439
472
|
raw: true,
|
|
440
473
|
method: 'get',
|
|
@@ -39,6 +39,8 @@ const axios = require('axios');
|
|
|
39
39
|
const { callTextToSqlApi, callCompleteOnCursorApi, callRefactorSqlQueryApi } = require('../utility/authProxy');
|
|
40
40
|
const { decryptConnection } = require('../utility/crypting');
|
|
41
41
|
const { getSshTunnel } = require('../utility/sshTunnel');
|
|
42
|
+
const sessions = require('./sessions');
|
|
43
|
+
const jsldata = require('./jsldata');
|
|
42
44
|
|
|
43
45
|
const logger = getLogger('databaseConnections');
|
|
44
46
|
|
|
@@ -96,6 +98,52 @@ module.exports = {
|
|
|
96
98
|
|
|
97
99
|
handle_ping() {},
|
|
98
100
|
|
|
101
|
+
// session event handlers
|
|
102
|
+
|
|
103
|
+
handle_info(conid, database, props) {
|
|
104
|
+
const { sesid, info } = props;
|
|
105
|
+
sessions.dispatchMessage(sesid, info);
|
|
106
|
+
},
|
|
107
|
+
|
|
108
|
+
handle_done(conid, database, props) {
|
|
109
|
+
const { sesid } = props;
|
|
110
|
+
socket.emit(`session-done-${sesid}`);
|
|
111
|
+
sessions.dispatchMessage(sesid, 'Query execution finished');
|
|
112
|
+
},
|
|
113
|
+
|
|
114
|
+
handle_recordset(conid, database, props) {
|
|
115
|
+
const { jslid, resultIndex } = props;
|
|
116
|
+
socket.emit(`session-recordset-${props.sesid}`, { jslid, resultIndex });
|
|
117
|
+
},
|
|
118
|
+
|
|
119
|
+
handle_stats(conid, database, stats) {
|
|
120
|
+
jsldata.notifyChangedStats(stats);
|
|
121
|
+
},
|
|
122
|
+
|
|
123
|
+
handle_initializeFile(conid, database, props) {
|
|
124
|
+
const { jslid } = props;
|
|
125
|
+
socket.emit(`session-initialize-file-${jslid}`);
|
|
126
|
+
},
|
|
127
|
+
|
|
128
|
+
// eval event handler
|
|
129
|
+
handle_runnerDone(conid, database, props) {
|
|
130
|
+
const { runid } = props;
|
|
131
|
+
socket.emit(`runner-done-${runid}`);
|
|
132
|
+
},
|
|
133
|
+
|
|
134
|
+
handle_progress(conid, database, progressData) {
|
|
135
|
+
const { progressName } = progressData;
|
|
136
|
+
const { name, runid } = progressName;
|
|
137
|
+
socket.emit(`runner-progress-${runid}`, { ...progressData, progressName: name });
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
handle_copyStreamError(conid, database, { copyStreamError }) {
|
|
141
|
+
const { progressName } = copyStreamError;
|
|
142
|
+
const { runid } = progressName;
|
|
143
|
+
logger.error(`Error in database connection ${conid}, database ${database}: ${copyStreamError}`);
|
|
144
|
+
socket.emit(`runner-done-${runid}`);
|
|
145
|
+
},
|
|
146
|
+
|
|
99
147
|
async ensureOpened(conid, database) {
|
|
100
148
|
const existing = this.opened.find(x => x.conid == conid && x.database == database);
|
|
101
149
|
if (existing) return existing;
|
|
@@ -136,7 +184,13 @@ module.exports = {
|
|
|
136
184
|
const { msgtype } = message;
|
|
137
185
|
if (handleProcessCommunication(message, subprocess)) return;
|
|
138
186
|
if (newOpened.disconnected) return;
|
|
139
|
-
|
|
187
|
+
const funcName = `handle_${msgtype}`;
|
|
188
|
+
if (!this[funcName]) {
|
|
189
|
+
logger.error(`Unknown message type ${msgtype} from subprocess databaseConnectionProcess`);
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
this[funcName](conid, database, message);
|
|
140
194
|
});
|
|
141
195
|
subprocess.on('exit', () => {
|
|
142
196
|
if (newOpened.disconnected) return;
|
|
@@ -763,4 +817,25 @@ module.exports = {
|
|
|
763
817
|
commandLine: this.commandArgsToCommandLine(commandArgs),
|
|
764
818
|
};
|
|
765
819
|
},
|
|
820
|
+
|
|
821
|
+
executeSessionQuery_meta: true,
|
|
822
|
+
async executeSessionQuery({ sesid, conid, database, sql }, req) {
|
|
823
|
+
testConnectionPermission(conid, req);
|
|
824
|
+
logger.info({ sesid, sql }, 'Processing query');
|
|
825
|
+
sessions.dispatchMessage(sesid, 'Query execution started');
|
|
826
|
+
|
|
827
|
+
const opened = await this.ensureOpened(conid, database);
|
|
828
|
+
opened.subprocess.send({ msgtype: 'executeSessionQuery', sql, sesid });
|
|
829
|
+
|
|
830
|
+
return { state: 'ok' };
|
|
831
|
+
},
|
|
832
|
+
|
|
833
|
+
evalJsonScript_meta: true,
|
|
834
|
+
async evalJsonScript({ conid, database, script, runid }, req) {
|
|
835
|
+
testConnectionPermission(conid, req);
|
|
836
|
+
const opened = await this.ensureOpened(conid, database);
|
|
837
|
+
|
|
838
|
+
opened.subprocess.send({ msgtype: 'evalJsonScript', script, runid });
|
|
839
|
+
return { state: 'ok' };
|
|
840
|
+
},
|
|
766
841
|
};
|
package/src/controllers/files.js
CHANGED
|
@@ -9,6 +9,9 @@ const scheduler = require('./scheduler');
|
|
|
9
9
|
const getDiagramExport = require('../utility/getDiagramExport');
|
|
10
10
|
const apps = require('./apps');
|
|
11
11
|
const getMapExport = require('../utility/getMapExport');
|
|
12
|
+
const dbgateApi = require('../shell');
|
|
13
|
+
const { getLogger } = require('dbgate-tools');
|
|
14
|
+
const logger = getLogger('files');
|
|
12
15
|
|
|
13
16
|
function serialize(format, data) {
|
|
14
17
|
if (format == 'text') return data;
|
|
@@ -219,4 +222,60 @@ module.exports = {
|
|
|
219
222
|
return path.join(dir, file);
|
|
220
223
|
}
|
|
221
224
|
},
|
|
225
|
+
|
|
226
|
+
createZipFromJsons_meta: true,
|
|
227
|
+
async createZipFromJsons({ db, filePath }) {
|
|
228
|
+
logger.info(`Creating zip file from JSONS ${filePath}`);
|
|
229
|
+
await dbgateApi.zipJsonLinesData(db, filePath);
|
|
230
|
+
return true;
|
|
231
|
+
},
|
|
232
|
+
|
|
233
|
+
getJsonsFromZip_meta: true,
|
|
234
|
+
async getJsonsFromZip({ filePath }) {
|
|
235
|
+
const res = await dbgateApi.unzipJsonLinesData(filePath);
|
|
236
|
+
return res;
|
|
237
|
+
},
|
|
238
|
+
|
|
239
|
+
downloadText_meta: true,
|
|
240
|
+
async downloadText({ uri }, req) {
|
|
241
|
+
if (!uri) return null;
|
|
242
|
+
const filePath = await dbgateApi.download(uri);
|
|
243
|
+
const text = await fs.readFile(filePath, {
|
|
244
|
+
encoding: 'utf-8',
|
|
245
|
+
});
|
|
246
|
+
return text;
|
|
247
|
+
},
|
|
248
|
+
|
|
249
|
+
saveUploadedFile_meta: true,
|
|
250
|
+
async saveUploadedFile({ filePath, fileName }) {
|
|
251
|
+
const FOLDERS = ['sql', 'sqlite'];
|
|
252
|
+
for (const folder of FOLDERS) {
|
|
253
|
+
if (fileName.toLowerCase().endsWith('.' + folder)) {
|
|
254
|
+
logger.info(`Saving ${folder} file ${fileName}`);
|
|
255
|
+
await fs.copyFile(filePath, path.join(filesdir(), folder, fileName));
|
|
256
|
+
|
|
257
|
+
socket.emitChanged(`files-changed`, { folder: folder });
|
|
258
|
+
socket.emitChanged(`all-files-changed`);
|
|
259
|
+
return {
|
|
260
|
+
name: path.basename(filePath),
|
|
261
|
+
folder: folder,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
throw new Error(`${fileName} doesn't have one of supported extensions: ${FOLDERS.join(', ')}`);
|
|
267
|
+
},
|
|
268
|
+
|
|
269
|
+
exportFile_meta: true,
|
|
270
|
+
async exportFile({ folder, file, filePath }, req) {
|
|
271
|
+
if (!hasPermission(`files/${folder}/read`, req)) return false;
|
|
272
|
+
await fs.copyFile(path.join(filesdir(), folder, file), filePath);
|
|
273
|
+
return true;
|
|
274
|
+
},
|
|
275
|
+
|
|
276
|
+
simpleCopy_meta: true,
|
|
277
|
+
async simpleCopy({ sourceFilePath, targetFilePath }, req) {
|
|
278
|
+
await fs.copyFile(sourceFilePath, targetFilePath);
|
|
279
|
+
return true;
|
|
280
|
+
},
|
|
222
281
|
};
|
|
@@ -8,6 +8,8 @@ const getJslFileName = require('../utility/getJslFileName');
|
|
|
8
8
|
const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
|
|
9
9
|
const requirePluginFunction = require('../utility/requirePluginFunction');
|
|
10
10
|
const socket = require('../utility/socket');
|
|
11
|
+
const crypto = require('crypto');
|
|
12
|
+
const dbgateApi = require('../shell');
|
|
11
13
|
|
|
12
14
|
function readFirstLine(file) {
|
|
13
15
|
return new Promise((resolve, reject) => {
|
|
@@ -293,4 +295,11 @@ module.exports = {
|
|
|
293
295
|
})),
|
|
294
296
|
};
|
|
295
297
|
},
|
|
298
|
+
|
|
299
|
+
downloadJslData_meta: true,
|
|
300
|
+
async downloadJslData({ uri }) {
|
|
301
|
+
const jslid = crypto.randomUUID();
|
|
302
|
+
await dbgateApi.download(uri, { targetFile: getJslFileName(jslid) });
|
|
303
|
+
return { jslid };
|
|
304
|
+
},
|
|
296
305
|
};
|