dbgate-api-premium 5.5.7-alpha.53 → 5.5.7-alpha.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -7
- package/src/controllers/config.js +3 -1
- package/src/controllers/connections.js +24 -0
- package/src/controllers/runners.js +1 -1
- package/src/controllers/serverConnections.js +1 -1
- package/src/controllers/storage.js +34 -1
- package/src/controllers/storageDb.js +1 -0
- package/src/currentVersion.js +2 -2
- package/src/shell/copyStream.js +7 -0
- package/src/shell/deployDb.js +17 -0
- package/src/shell/dropAllDbObjects.js +9 -0
- package/src/shell/executeQuery.js +10 -0
- package/src/shell/generateDeploySql.js +40 -4
- package/src/shell/jsonLinesReader.js +8 -0
- package/src/shell/jsonLinesWriter.js +8 -0
- package/src/shell/jsonReader.js +11 -0
- package/src/shell/jsonWriter.js +10 -0
- package/src/shell/queryReader.js +9 -0
- package/src/shell/tableReader.js +9 -0
- package/src/shell/tableWriter.js +14 -0
- package/src/shell/types.js +27 -0
- package/src/utility/JsonLinesDatabase.js +9 -0
- package/src/utility/SSHConnection.js +1 -1
- package/src/utility/authProxy.js +39 -3
- package/src/utility/checkLicense.js +54 -6
- package/src/utility/directories.js +6 -1
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dbgate-api-premium",
|
|
3
3
|
"main": "src/index.js",
|
|
4
|
-
"version": "5.5.7-alpha.
|
|
4
|
+
"version": "5.5.7-alpha.68",
|
|
5
5
|
"homepage": "https://dbgate.org/",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -25,15 +25,14 @@
|
|
|
25
25
|
"async-lock": "^1.2.6",
|
|
26
26
|
"axios": "^0.21.1",
|
|
27
27
|
"body-parser": "^1.19.0",
|
|
28
|
-
"bufferutil": "^4.0.1",
|
|
29
28
|
"byline": "^5.0.0",
|
|
30
29
|
"compare-versions": "^3.6.0",
|
|
31
30
|
"cors": "^2.8.5",
|
|
32
31
|
"cross-env": "^6.0.3",
|
|
33
|
-
"dbgate-datalib": "^5.5.7-alpha.
|
|
32
|
+
"dbgate-datalib": "^5.5.7-alpha.68",
|
|
34
33
|
"dbgate-query-splitter": "^4.11.2",
|
|
35
|
-
"dbgate-sqltree": "^5.5.7-alpha.
|
|
36
|
-
"dbgate-tools": "^5.5.7-alpha.
|
|
34
|
+
"dbgate-sqltree": "^5.5.7-alpha.68",
|
|
35
|
+
"dbgate-tools": "^5.5.7-alpha.68",
|
|
37
36
|
"debug": "^4.3.4",
|
|
38
37
|
"diff": "^5.0.0",
|
|
39
38
|
"diff2html": "^3.4.13",
|
|
@@ -76,13 +75,15 @@
|
|
|
76
75
|
"start:storage:built": "env-cmd -f env/storage/.env cross-env DEVMODE= BUILTWEBMODE=1 node dist/bundle.js --listen-api",
|
|
77
76
|
"start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
|
|
78
77
|
"ts": "tsc",
|
|
79
|
-
"build": "webpack"
|
|
78
|
+
"build": "webpack",
|
|
79
|
+
"build:doc": "jsdoc2md --template doctpl.hbs ./src/shell/* > ../../../dbgate.github.io/_docs/apidoc.md"
|
|
80
80
|
},
|
|
81
81
|
"devDependencies": {
|
|
82
82
|
"@types/fs-extra": "^9.0.11",
|
|
83
83
|
"@types/lodash": "^4.14.149",
|
|
84
|
-
"dbgate-types": "^5.5.7-alpha.
|
|
84
|
+
"dbgate-types": "^5.5.7-alpha.68",
|
|
85
85
|
"env-cmd": "^10.1.0",
|
|
86
|
+
"jsdoc-to-markdown": "^9.0.5",
|
|
86
87
|
"node-loader": "^1.0.2",
|
|
87
88
|
"nodemon": "^2.0.2",
|
|
88
89
|
"typescript": "^4.4.3",
|
|
@@ -62,6 +62,8 @@ module.exports = {
|
|
|
62
62
|
const logoutUrl = storageConnectionError ? null : await authProvider.getLogoutUrl();
|
|
63
63
|
const adminConfig = storageConnectionError ? null : await storage.readConfig({ group: 'admin' });
|
|
64
64
|
|
|
65
|
+
storage.startRefreshLicense();
|
|
66
|
+
|
|
65
67
|
const isAdminPasswordMissing = !!(
|
|
66
68
|
process.env.STORAGE_DATABASE &&
|
|
67
69
|
!process.env.ADMIN_PASSWORD &&
|
|
@@ -81,7 +83,7 @@ module.exports = {
|
|
|
81
83
|
isElectron: platformInfo.isElectron,
|
|
82
84
|
isLicenseValid,
|
|
83
85
|
isLicenseExpired: checkedLicense?.isExpired,
|
|
84
|
-
trialDaysLeft: checkedLicense?.
|
|
86
|
+
trialDaysLeft: checkedLicense?.licenseTypeObj?.isTrial && !checkedLicense?.isExpired ? checkedLicense?.daysLeft : null,
|
|
85
87
|
checkedLicense,
|
|
86
88
|
configurationError,
|
|
87
89
|
logoutUrl,
|
|
@@ -201,6 +201,7 @@ module.exports = {
|
|
|
201
201
|
// @ts-ignore
|
|
202
202
|
this.datastore = new JsonLinesDatabase(path.join(dir, 'connections.jsonl'));
|
|
203
203
|
}
|
|
204
|
+
await this.checkUnsavedConnectionsLimit();
|
|
204
205
|
},
|
|
205
206
|
|
|
206
207
|
list_meta: true,
|
|
@@ -300,6 +301,29 @@ module.exports = {
|
|
|
300
301
|
return res;
|
|
301
302
|
},
|
|
302
303
|
|
|
304
|
+
async checkUnsavedConnectionsLimit() {
|
|
305
|
+
const MAX_UNSAVED_CONNECTIONS = 5;
|
|
306
|
+
await this.datastore.transformAll(connections => {
|
|
307
|
+
const count = connections.filter(x => x.unsaved).length;
|
|
308
|
+
if (count > MAX_UNSAVED_CONNECTIONS) {
|
|
309
|
+
const res = [];
|
|
310
|
+
let unsavedToSkip = count - MAX_UNSAVED_CONNECTIONS;
|
|
311
|
+
for (const item of connections) {
|
|
312
|
+
if (item.unsaved) {
|
|
313
|
+
if (unsavedToSkip > 0) {
|
|
314
|
+
unsavedToSkip--;
|
|
315
|
+
} else {
|
|
316
|
+
res.push(item);
|
|
317
|
+
}
|
|
318
|
+
} else {
|
|
319
|
+
res.push(item);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
return res;
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
},
|
|
326
|
+
|
|
303
327
|
update_meta: true,
|
|
304
328
|
async update({ _id, values }, req) {
|
|
305
329
|
if (portalConnections) return;
|
|
@@ -111,7 +111,7 @@ module.exports = {
|
|
|
111
111
|
const scriptFile = path.join(uploadsdir(), runid + '.js');
|
|
112
112
|
fs.writeFileSync(`${scriptFile}`, scriptText);
|
|
113
113
|
fs.mkdirSync(directory);
|
|
114
|
-
const pluginNames =
|
|
114
|
+
const pluginNames = extractPlugins(scriptText);
|
|
115
115
|
logger.info({ scriptFile }, 'Running script');
|
|
116
116
|
// const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
|
|
117
117
|
const subprocess = fork(
|
|
@@ -52,7 +52,7 @@ module.exports = {
|
|
|
52
52
|
if (existing) return existing;
|
|
53
53
|
const connection = await connections.getCore({ conid });
|
|
54
54
|
if (!connection) {
|
|
55
|
-
throw new Error(`Connection with conid="${conid}" not
|
|
55
|
+
throw new Error(`Connection with conid="${conid}" not found`);
|
|
56
56
|
}
|
|
57
57
|
if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
|
|
58
58
|
throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const fs = require('fs-extra');
|
|
2
2
|
const _ = require('lodash');
|
|
3
|
+
const path = require('path');
|
|
3
4
|
const { setAuthProviders, getAuthProviderById } = require('../auth/authProvider');
|
|
4
5
|
const { createStorageAuthProvider } = require('../auth/storageAuthProvider');
|
|
5
6
|
const {
|
|
@@ -16,8 +17,12 @@ const { hasPermission } = require('../utility/hasPermission');
|
|
|
16
17
|
const { changeSetToSql, removeSchemaFromChangeSet } = require('dbgate-datalib');
|
|
17
18
|
const storageModel = require('../storageModel');
|
|
18
19
|
const { dumpSqlCommand } = require('dbgate-sqltree');
|
|
19
|
-
const { runCommandOnDriver } = require('dbgate-tools');
|
|
20
|
+
const { runCommandOnDriver, getLogger } = require('dbgate-tools');
|
|
20
21
|
const socket = require('../utility/socket');
|
|
22
|
+
const { obtainRefreshedLicense } = require('../utility/authProxy');
|
|
23
|
+
const { datadir } = require('../utility/directories');
|
|
24
|
+
|
|
25
|
+
const logger = getLogger('storage');
|
|
21
26
|
|
|
22
27
|
function mapConnection(connnection) {
|
|
23
28
|
return {
|
|
@@ -32,6 +37,8 @@ async function runQueryFmt(driver, conn, query, ...args) {
|
|
|
32
37
|
await driver.query(conn, dmp.s);
|
|
33
38
|
}
|
|
34
39
|
|
|
40
|
+
let refreshLicenseStarted = false;
|
|
41
|
+
|
|
35
42
|
module.exports = {
|
|
36
43
|
async _init() {
|
|
37
44
|
if (!process.env.STORAGE_DATABASE) {
|
|
@@ -48,6 +55,32 @@ module.exports = {
|
|
|
48
55
|
setAuthProviders(providers, providers[defIndex]);
|
|
49
56
|
},
|
|
50
57
|
|
|
58
|
+
async startRefreshLicense() {
|
|
59
|
+
if (refreshLicenseStarted) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
refreshLicenseStarted = true;
|
|
63
|
+
const resp = await obtainRefreshedLicense();
|
|
64
|
+
if (!resp) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
if (resp.status == 'error') {
|
|
68
|
+
logger.error(`Error refreshing license: ${resp.message}`);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
if (resp.status != 'ok') {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
const { token } = resp;
|
|
75
|
+
logger.info('License succesfully refreshed');
|
|
76
|
+
if (process.env.STORAGE_DATABASE) {
|
|
77
|
+
await this.writeConfig({ group: 'license', config: { licenseKey: token } });
|
|
78
|
+
} else {
|
|
79
|
+
await fs.writeFile(path.join(datadir(), 'license.key'), token);
|
|
80
|
+
}
|
|
81
|
+
socket.emitChanged(`config-changed`);
|
|
82
|
+
},
|
|
83
|
+
|
|
51
84
|
connections_meta: true,
|
|
52
85
|
async connections(req) {
|
|
53
86
|
if (!process.env.STORAGE_DATABASE) {
|
package/src/currentVersion.js
CHANGED
package/src/shell/copyStream.js
CHANGED
|
@@ -2,6 +2,13 @@ const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
|
|
|
2
2
|
const Stream = require('stream');
|
|
3
3
|
const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream');
|
|
4
4
|
|
|
5
|
+
/**
|
|
6
|
+
* Copies reader to writer. Used for import, export tables and transfer data between tables
|
|
7
|
+
* @param {readerType} input - reader object
|
|
8
|
+
* @param {writerType} output - writer object
|
|
9
|
+
* @param {object} options - options
|
|
10
|
+
* @returns {Promise}
|
|
11
|
+
*/
|
|
5
12
|
function copyStream(input, output, options) {
|
|
6
13
|
const { columns } = options || {};
|
|
7
14
|
|
package/src/shell/deployDb.js
CHANGED
|
@@ -6,6 +6,21 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
|
|
|
6
6
|
const loadModelFolder = require('../utility/loadModelFolder');
|
|
7
7
|
const crypto = require('crypto');
|
|
8
8
|
|
|
9
|
+
/**
|
|
10
|
+
* Deploys database model stored in modelFolder (table as yamls) to database
|
|
11
|
+
* @param {object} options
|
|
12
|
+
* @param {connectionType} options.connection - connection object
|
|
13
|
+
* @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
|
|
14
|
+
* @param {object} options.driver - driver object. If not provided, it will be loaded from connection
|
|
15
|
+
* @param {object} options.analysedStructure - analysed structure of the database. If not provided, it will be loaded
|
|
16
|
+
* @param {string} options.modelFolder - folder with model files (YAML files for tables, SQL files for views, procedures, ...)
|
|
17
|
+
* @param {import('dbgate-tools').DatabaseModelFile[]} options.loadedDbModel - loaded database model - collection of yaml and SQL files loaded into array
|
|
18
|
+
* @param {function[]} options.modelTransforms - array of functions for transforming model
|
|
19
|
+
* @param {object} options.dbdiffOptionsExtra - extra options for dbdiff
|
|
20
|
+
* @param {string} options.ignoreNameRegex - regex for ignoring objects by name
|
|
21
|
+
* @param {string} options.targetSchema - target schema for deployment
|
|
22
|
+
* @param {number} options.maxMissingTablesRatio - maximum ratio of missing tables in database. Safety check, if missing ratio is highe, deploy is stopped (preventing accidental drop of all tables)
|
|
23
|
+
*/
|
|
9
24
|
async function deployDb({
|
|
10
25
|
connection,
|
|
11
26
|
systemConnection,
|
|
@@ -17,6 +32,7 @@ async function deployDb({
|
|
|
17
32
|
dbdiffOptionsExtra,
|
|
18
33
|
ignoreNameRegex = '',
|
|
19
34
|
targetSchema = null,
|
|
35
|
+
maxMissingTablesRatio = undefined,
|
|
20
36
|
}) {
|
|
21
37
|
if (!driver) driver = requireEngineDriver(connection);
|
|
22
38
|
const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
|
|
@@ -41,6 +57,7 @@ async function deployDb({
|
|
|
41
57
|
dbdiffOptionsExtra,
|
|
42
58
|
ignoreNameRegex,
|
|
43
59
|
targetSchema,
|
|
60
|
+
maxMissingTablesRatio,
|
|
44
61
|
});
|
|
45
62
|
// console.log('RUNNING DEPLOY SCRIPT:', sql);
|
|
46
63
|
await executeQuery({ connection, systemConnection: dbhan, driver, sql, logScriptItems: true });
|
|
@@ -5,6 +5,15 @@ const { getLogger, extendDatabaseInfo } = require('dbgate-tools');
|
|
|
5
5
|
|
|
6
6
|
const logger = getLogger('dropAllDbObjects');
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
* Drops all database objects
|
|
10
|
+
* @param {object} options
|
|
11
|
+
* @param {connectionType} options.connection - connection object
|
|
12
|
+
* @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
|
|
13
|
+
* @param {object} options.driver - driver object. If not provided, it will be loaded from connection
|
|
14
|
+
* @param {object} options.analysedStructure - analysed structure of the database. If not provided, it will be loaded
|
|
15
|
+
* @returns {Promise}
|
|
16
|
+
*/
|
|
8
17
|
async function dropAllDbObjects({ connection, systemConnection, driver, analysedStructure }) {
|
|
9
18
|
if (!driver) driver = requireEngineDriver(connection);
|
|
10
19
|
|
|
@@ -5,6 +5,16 @@ const { getLogger, getLimitedQuery } = require('dbgate-tools');
|
|
|
5
5
|
|
|
6
6
|
const logger = getLogger('execQuery');
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
* Executes SQL query
|
|
10
|
+
* @param {object} options
|
|
11
|
+
* @param {connectionType} [options.connection] - connection object
|
|
12
|
+
* @param {object} [options.systemConnection] - system connection (result of driver.connect). If not provided, new connection will be created
|
|
13
|
+
* @param {object} [options.driver] - driver object. If not provided, it will be loaded from connection
|
|
14
|
+
* @param {string} [options.sql] - SQL query
|
|
15
|
+
* @param {string} [options.sqlFile] - SQL file
|
|
16
|
+
* @param {boolean} [options.logScriptItems] - whether to log script items instead of whole script
|
|
17
|
+
*/
|
|
8
18
|
async function executeQuery({
|
|
9
19
|
connection = undefined,
|
|
10
20
|
systemConnection = undefined,
|
|
@@ -15,6 +15,21 @@ const importDbModel = require('../utility/importDbModel');
|
|
|
15
15
|
const requireEngineDriver = require('../utility/requireEngineDriver');
|
|
16
16
|
const connectUtility = require('../utility/connectUtility');
|
|
17
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Generates query for deploying model into database
|
|
20
|
+
* @param {object} options
|
|
21
|
+
* @param {connectionType} options.connection - connection object
|
|
22
|
+
* @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
|
|
23
|
+
* @param {object} options.driver - driver object. If not provided, it will be loaded from connection
|
|
24
|
+
* @param {object} options.analysedStructure - analysed structure of the database. If not provided, it will be loaded
|
|
25
|
+
* @param {string} options.modelFolder - folder with model files (YAML files for tables, SQL files for views, procedures, ...)
|
|
26
|
+
* @param {import('dbgate-tools').DatabaseModelFile[]} options.loadedDbModel - loaded database model - collection of yaml and SQL files loaded into array
|
|
27
|
+
* @param {function[]} options.modelTransforms - array of functions for transforming model
|
|
28
|
+
* @param {object} options.dbdiffOptionsExtra - extra options for dbdiff
|
|
29
|
+
* @param {string} options.ignoreNameRegex - regex for ignoring objects by name
|
|
30
|
+
* @param {string} options.targetSchema - target schema for deployment
|
|
31
|
+
* @param {number} options.maxMissingTablesRatio - maximum ratio of missing tables in database. Safety check, if missing ratio is highe, deploy is stopped (preventing accidental drop of all tables)
|
|
32
|
+
*/
|
|
18
33
|
async function generateDeploySql({
|
|
19
34
|
connection,
|
|
20
35
|
systemConnection = undefined,
|
|
@@ -26,25 +41,35 @@ async function generateDeploySql({
|
|
|
26
41
|
dbdiffOptionsExtra = {},
|
|
27
42
|
ignoreNameRegex = '',
|
|
28
43
|
targetSchema = null,
|
|
44
|
+
maxMissingTablesRatio = undefined,
|
|
29
45
|
}) {
|
|
30
46
|
if (!driver) driver = requireEngineDriver(connection);
|
|
31
47
|
|
|
32
48
|
const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
|
|
49
|
+
if (
|
|
50
|
+
driver?.dialect?.multipleSchema &&
|
|
51
|
+
!targetSchema &&
|
|
52
|
+
dbdiffOptionsExtra?.['schemaMode'] !== 'ignore' &&
|
|
53
|
+
dbdiffOptionsExtra?.['schemaMode'] !== 'ignoreImplicit'
|
|
54
|
+
) {
|
|
55
|
+
throw new Error('targetSchema is required for databases with multiple schemas');
|
|
56
|
+
}
|
|
33
57
|
|
|
34
58
|
try {
|
|
35
59
|
if (!analysedStructure) {
|
|
36
60
|
analysedStructure = await driver.analyseFull(dbhan);
|
|
37
61
|
}
|
|
38
62
|
|
|
63
|
+
let deployedModelSource = loadedDbModel
|
|
64
|
+
? databaseInfoFromYamlModel(loadedDbModel)
|
|
65
|
+
: await importDbModel(modelFolder);
|
|
66
|
+
|
|
39
67
|
if (ignoreNameRegex) {
|
|
40
68
|
analysedStructure = skipNamesInStructureByRegex(analysedStructure, new RegExp(ignoreNameRegex, 'i'));
|
|
69
|
+
deployedModelSource = skipNamesInStructureByRegex(deployedModelSource, new RegExp(ignoreNameRegex, 'i'));
|
|
41
70
|
}
|
|
42
71
|
analysedStructure = skipDbGateInternalObjects(analysedStructure);
|
|
43
72
|
|
|
44
|
-
let deployedModelSource = loadedDbModel
|
|
45
|
-
? databaseInfoFromYamlModel(loadedDbModel)
|
|
46
|
-
: await importDbModel(modelFolder);
|
|
47
|
-
|
|
48
73
|
for (const transform of modelTransforms || []) {
|
|
49
74
|
deployedModelSource = transform(deployedModelSource);
|
|
50
75
|
}
|
|
@@ -71,6 +96,17 @@ async function generateDeploySql({
|
|
|
71
96
|
const currentModelPaired = matchPairedObjects(deployedModel, currentModel, opts);
|
|
72
97
|
const currentModelPairedPreloaded = await enrichWithPreloadedRows(deployedModel, currentModelPaired, dbhan, driver);
|
|
73
98
|
|
|
99
|
+
if (maxMissingTablesRatio != null) {
|
|
100
|
+
const missingTables = currentModelPaired.tables.filter(
|
|
101
|
+
x => !deployedModel.tables.find(y => y.pairingId == x.pairingId)
|
|
102
|
+
);
|
|
103
|
+
const missingTableCount = missingTables.length;
|
|
104
|
+
const missingTablesRatio = missingTableCount / (currentModelPaired.tables.length || 1);
|
|
105
|
+
if (missingTablesRatio > maxMissingTablesRatio) {
|
|
106
|
+
throw new Error(`Too many missing tables (${missingTablesRatio * 100}%), aborting deploy`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
74
110
|
// console.log('currentModelPairedPreloaded', currentModelPairedPreloaded.tables[0]);
|
|
75
111
|
// console.log('deployedModel', deployedModel.tables[0]);
|
|
76
112
|
// console.log('currentModel', currentModel.tables[0]);
|
|
@@ -33,6 +33,14 @@ class ParseStream extends stream.Transform {
|
|
|
33
33
|
}
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Reader function, which reads JSNOL file or URL. JSONL format - text file, every line is JSON encoded row.
|
|
38
|
+
* @param {Object} options
|
|
39
|
+
* @param {string} options.fileName - file name or URL
|
|
40
|
+
* @param {string} options.encoding - encoding of the file
|
|
41
|
+
* @param {number} options.limitRows - maximum number of rows to read
|
|
42
|
+
* @returns {Promise<readerType>} - reader object
|
|
43
|
+
*/
|
|
36
44
|
async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
|
|
37
45
|
logger.info(`Reading file ${fileName}`);
|
|
38
46
|
|
|
@@ -24,6 +24,14 @@ class StringifyStream extends stream.Transform {
|
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Returns writer object for {@link copyStream} function. This writer object writes data to JSONL file. JSONL format - text file, every line is JSON encoded row, used eg. by MongoDB.
|
|
29
|
+
* @param {object} options
|
|
30
|
+
* @param {string} options.fileName - file name
|
|
31
|
+
* @param {string} [options.encoding] - encoding of the file
|
|
32
|
+
* @param {boolean} [options.header] - whether to write header. Header is JSON describing source table structure. Header is specific to DbGate, if you want eg. to import data to MongoDB, you should not write header.
|
|
33
|
+
* @returns {Promise<writerType>} - writer object
|
|
34
|
+
*/
|
|
27
35
|
async function jsonLinesWriter({ fileName, encoding = 'utf-8', header = true }) {
|
|
28
36
|
logger.info(`Writing file ${fileName}`);
|
|
29
37
|
const stringify = new StringifyStream({ header });
|
package/src/shell/jsonReader.js
CHANGED
|
@@ -45,6 +45,17 @@ class ParseStream extends stream.Transform {
|
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
/**
|
|
49
|
+
* Creates reader object for JSON file for {@link copyStream} function.
|
|
50
|
+
* @param {object} options
|
|
51
|
+
* @param {string} options.fileName - file name or URL
|
|
52
|
+
* @param {string} options.jsonStyle - 'object' or 'array'
|
|
53
|
+
* @param {string} [options.keyField] - key field for object style
|
|
54
|
+
* @param {string} [options.rootField] - root field for object style
|
|
55
|
+
* @param {string} [options.encoding] - encoding of the file
|
|
56
|
+
* @param {number} [options.limitRows] - maximum number of rows to read
|
|
57
|
+
* @returns {Promise<readerType>} - reader object
|
|
58
|
+
*/
|
|
48
59
|
async function jsonReader({
|
|
49
60
|
fileName,
|
|
50
61
|
jsonStyle,
|
package/src/shell/jsonWriter.js
CHANGED
|
@@ -85,6 +85,16 @@ class StringifyStream extends stream.Transform {
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
+
/**
|
|
89
|
+
* Returns writer object for {@link copyStream} function. This writer object writes data to JSON file.
|
|
90
|
+
* @param {object} options
|
|
91
|
+
* @param {string} options.fileName - file name
|
|
92
|
+
* @param {string} [options.jsonStyle] - 'object' or 'array'
|
|
93
|
+
* @param {string} [options.keyField] - key field for object style
|
|
94
|
+
* @param {string} [options.rootField] - root field for object style
|
|
95
|
+
* @param {string} [options.encoding] - encoding of the file
|
|
96
|
+
* @returns {Promise<writerType>} - writer object
|
|
97
|
+
*/
|
|
88
98
|
async function jsonWriter({ fileName, jsonStyle, keyField = '_key', rootField, encoding = 'utf-8' }) {
|
|
89
99
|
logger.info(`Writing file ${fileName}`);
|
|
90
100
|
const stringify = new StringifyStream({ jsonStyle, keyField, rootField });
|
package/src/shell/queryReader.js
CHANGED
|
@@ -3,6 +3,15 @@ const connectUtility = require('../utility/connectUtility');
|
|
|
3
3
|
const { getLogger } = require('dbgate-tools');
|
|
4
4
|
const logger = getLogger('queryReader');
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Returns reader object for {@link copyStream} function. This reader object reads data from query.
|
|
8
|
+
* @param {object} options
|
|
9
|
+
* @param {connectionType} options.connection - connection object
|
|
10
|
+
* @param {string} options.query - SQL query
|
|
11
|
+
* @param {string} [options.queryType] - query type
|
|
12
|
+
* @param {string} [options.sql] - SQL query. obsolete; use query instead
|
|
13
|
+
* @returns {Promise<readerType>} - reader object
|
|
14
|
+
*/
|
|
6
15
|
async function queryReader({
|
|
7
16
|
connection,
|
|
8
17
|
query,
|
package/src/shell/tableReader.js
CHANGED
|
@@ -3,6 +3,15 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
|
|
|
3
3
|
const connectUtility = require('../utility/connectUtility');
|
|
4
4
|
const logger = getLogger('tableReader');
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Creates reader object for {@link copyStream} function. This reader object reads data from table or view.
|
|
8
|
+
* @param {object} options
|
|
9
|
+
* @param {connectionType} options.connection - connection object
|
|
10
|
+
* @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
|
|
11
|
+
* @param {string} options.pureName - table name
|
|
12
|
+
* @param {string} options.schemaName - schema name
|
|
13
|
+
* @returns {Promise<readerType>} - reader object
|
|
14
|
+
*/
|
|
6
15
|
async function tableReader({ connection, systemConnection, pureName, schemaName }) {
|
|
7
16
|
const driver = requireEngineDriver(connection);
|
|
8
17
|
const dbhan = systemConnection || (await connectUtility(driver, connection, 'read'));
|
package/src/shell/tableWriter.js
CHANGED
|
@@ -3,6 +3,20 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
|
|
|
3
3
|
const connectUtility = require('../utility/connectUtility');
|
|
4
4
|
const logger = getLogger('tableWriter');
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Creates writer object for {@link copyStream} function. This writer object writes data to table. Table could be created if not exists.
|
|
8
|
+
* @param {object} options
|
|
9
|
+
* @param {connectionType} options.connection - connection object
|
|
10
|
+
* @param {object} options.systemConnection - system connection (result of driver.connect). If not provided, new connection will be created
|
|
11
|
+
* @param {object} options.driver - driver object. If not provided, it will be loaded from connection
|
|
12
|
+
* @param {string} options.pureName - table name
|
|
13
|
+
* @param {string} options.schemaName - schema name
|
|
14
|
+
* @param {boolean} options.dropIfExists - drop table if exists
|
|
15
|
+
* @param {boolean} options.truncate - truncate table before insert
|
|
16
|
+
* @param {boolean} options.createIfNotExists - create table if not exists
|
|
17
|
+
* @param {boolean} options.commitAfterInsert - commit transaction after insert
|
|
18
|
+
* @returns {Promise<writerType>} - writer object
|
|
19
|
+
*/
|
|
6
20
|
async function tableWriter({ connection, schemaName, pureName, driver, systemConnection, ...options }) {
|
|
7
21
|
logger.info(`Writing table ${fullNameToString({ schemaName, pureName })}`);
|
|
8
22
|
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reader (input) object for {@link copyStream} function
|
|
3
|
+
* @typedef {Object} readerType
|
|
4
|
+
*
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Writer (output) object for {@link copyStream} function
|
|
9
|
+
* @typedef {Object} writerType
|
|
10
|
+
*
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Typ uživatelské role.
|
|
15
|
+
* @typedef {('mysql@dbgate-plugin-mysql' | 'mariadb@dbgate-plugin-mysql' | 'postgres@dbgate-plugin-postgres'
|
|
16
|
+
* |'sqlite@dbgate-plugin-sqlite' | 'oracle@dbgate-plugin-oracle' | 'cockroach@dbgate-plugin-postgres' | 'redshift@dbgate-plugin-postgres')} engineType
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @typedef {Object} connectionType
|
|
21
|
+
* @property {engineType} engine
|
|
22
|
+
* @property {string} server
|
|
23
|
+
* @property {string} user
|
|
24
|
+
* @property {string} password
|
|
25
|
+
* @property {string} database
|
|
26
|
+
* @property {string} port
|
|
27
|
+
*/
|
|
@@ -111,6 +111,15 @@ class JsonLinesDatabase {
|
|
|
111
111
|
return removed;
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
+
async transformAll(transformFunction) {
|
|
115
|
+
await this._ensureLoaded();
|
|
116
|
+
const newData = transformFunction(this.data);
|
|
117
|
+
if (newData) {
|
|
118
|
+
this.data = newData;
|
|
119
|
+
await this._save();
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
114
123
|
// async _openReader() {
|
|
115
124
|
// return new Promise((resolve, reject) =>
|
|
116
125
|
// lineReader.open(this.filename, (err, reader) => {
|
|
@@ -14,7 +14,6 @@
|
|
|
14
14
|
* limitations under the License.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
const { Client } = require('ssh2');
|
|
18
17
|
const net = require('net');
|
|
19
18
|
const fs = require('fs');
|
|
20
19
|
const os = require('os');
|
|
@@ -147,6 +146,7 @@ class SSHConnection {
|
|
|
147
146
|
}
|
|
148
147
|
|
|
149
148
|
async connect(host, stream) {
|
|
149
|
+
const { Client } = require('ssh2');
|
|
150
150
|
this.debug('Connecting to "%s"', host);
|
|
151
151
|
const connection = new Client();
|
|
152
152
|
return new Promise(async (resolve, reject) => {
|
package/src/utility/authProxy.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
const axios = require('axios');
|
|
2
2
|
const { Signer } = require('@aws-sdk/rds-signer');
|
|
3
|
+
const jwt = require('jsonwebtoken');
|
|
4
|
+
const { getLogger, extractErrorLogData } = require('dbgate-tools');
|
|
3
5
|
|
|
4
|
-
const
|
|
5
|
-
|
|
6
|
+
const logger = getLogger('authProxy');
|
|
7
|
+
|
|
8
|
+
const AUTH_PROXY_URL = process.env.DEVWEB ? 'https://auth-proxy.dbgate.udolni.net' : 'https://auth.dbgate.eu';
|
|
9
|
+
// const AUTH_PROXY_URL = 'https://auth-proxy.dbgate.udolni.net';
|
|
6
10
|
|
|
7
11
|
let licenseKey = null;
|
|
8
12
|
|
|
@@ -91,7 +95,7 @@ function startTokenChecking(sid, callback) {
|
|
|
91
95
|
callback(resp.data.token);
|
|
92
96
|
}
|
|
93
97
|
} catch (err) {
|
|
94
|
-
|
|
98
|
+
logger.error(extractErrorLogData(err), 'Error checking token');
|
|
95
99
|
}
|
|
96
100
|
}, 500);
|
|
97
101
|
}
|
|
@@ -121,6 +125,37 @@ async function getAwsIamToken(props) {
|
|
|
121
125
|
return token;
|
|
122
126
|
}
|
|
123
127
|
|
|
128
|
+
async function obtainRefreshedLicense() {
|
|
129
|
+
if (!licenseKey) {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const decoded = jwt.decode(licenseKey);
|
|
134
|
+
|
|
135
|
+
if (Date.now() > decoded.end * 1000) {
|
|
136
|
+
logger.info('License expired, trying to obtain fresh license');
|
|
137
|
+
|
|
138
|
+
try {
|
|
139
|
+
const respToken = await axios.default.post(
|
|
140
|
+
`${AUTH_PROXY_URL}/refresh-license`,
|
|
141
|
+
{},
|
|
142
|
+
{
|
|
143
|
+
headers: {
|
|
144
|
+
'Content-Type': 'application/json',
|
|
145
|
+
Authorization: `Bearer ${licenseKey}`,
|
|
146
|
+
},
|
|
147
|
+
}
|
|
148
|
+
);
|
|
149
|
+
return respToken.data;
|
|
150
|
+
} catch (err) {
|
|
151
|
+
return {
|
|
152
|
+
status: 'error',
|
|
153
|
+
message: err.message,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
124
159
|
module.exports = {
|
|
125
160
|
isAuthProxySupported,
|
|
126
161
|
authProxyGetRedirectUrl,
|
|
@@ -130,4 +165,5 @@ module.exports = {
|
|
|
130
165
|
getAuthProxyUrl,
|
|
131
166
|
supportsAwsIam,
|
|
132
167
|
getAwsIamToken,
|
|
168
|
+
obtainRefreshedLicense,
|
|
133
169
|
};
|
|
@@ -23,6 +23,46 @@ mQIDAQAB
|
|
|
23
23
|
-----END PUBLIC KEY-----
|
|
24
24
|
`;
|
|
25
25
|
|
|
26
|
+
const licenseTypeById = {
|
|
27
|
+
'1414ede2-dfb3-4539-a93d-24db7e5b59d9': {
|
|
28
|
+
// premium generated by stripe
|
|
29
|
+
name: 'Premium',
|
|
30
|
+
isPremium: true,
|
|
31
|
+
isForWeb: false,
|
|
32
|
+
isForApp: true,
|
|
33
|
+
},
|
|
34
|
+
'9682a88b-909f-48b1-adbf-c03622884421': {
|
|
35
|
+
name: 'Team Premium',
|
|
36
|
+
isPremium: true,
|
|
37
|
+
isForWeb: true,
|
|
38
|
+
isForApp: true,
|
|
39
|
+
},
|
|
40
|
+
'81456363-f167-41e3-9496-b540f4b0c150': {
|
|
41
|
+
name: 'Premium Trial',
|
|
42
|
+
isPremium: true,
|
|
43
|
+
isForWeb: true,
|
|
44
|
+
isForApp: true,
|
|
45
|
+
isTrial: true,
|
|
46
|
+
},
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
function getLicenseByDecoded(decoded) {
|
|
50
|
+
if (!decoded) {
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
if (decoded.licenseId) {
|
|
54
|
+
return licenseTypeById[decoded.licenseId];
|
|
55
|
+
}
|
|
56
|
+
if (decoded.licenseType == 'premium') {
|
|
57
|
+
if (decoded.isGeneratedTrial) {
|
|
58
|
+
return licenseTypeById['81456363-f167-41e3-9496-b540f4b0c150'];
|
|
59
|
+
} else {
|
|
60
|
+
return licenseTypeById['9682a88b-909f-48b1-adbf-c03622884421'];
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
|
|
26
66
|
let awsMetadataLoaded = false;
|
|
27
67
|
let awsMetadata = null;
|
|
28
68
|
async function getAwsMetadata() {
|
|
@@ -71,20 +111,27 @@ function checkLicenseKey(licenseKey) {
|
|
|
71
111
|
const decoded = jwt.verify(licenseKey, publicKey, {
|
|
72
112
|
algorithms: ['RS256'],
|
|
73
113
|
});
|
|
74
|
-
|
|
75
|
-
|
|
114
|
+
|
|
115
|
+
const licenseTypeObj = getLicenseByDecoded(decoded);
|
|
116
|
+
|
|
117
|
+
if (
|
|
118
|
+
!licenseTypeObj ||
|
|
119
|
+
(platformInfo.isElectron && !licenseTypeObj.isForApp) ||
|
|
120
|
+
(!platformInfo.isElectron && !licenseTypeObj.isForWeb)
|
|
121
|
+
) {
|
|
122
|
+
logger.error(`Incorrect license type, found ${decoded.licenseType}`);
|
|
76
123
|
return {
|
|
77
124
|
status: 'error',
|
|
78
|
-
error: `Incorrect license type,
|
|
125
|
+
error: `Incorrect license type, found ${decoded.licenseType}`,
|
|
79
126
|
};
|
|
80
127
|
}
|
|
128
|
+
|
|
81
129
|
return {
|
|
82
130
|
status: 'ok',
|
|
83
|
-
type: 'premium',
|
|
84
131
|
validTo: decoded.validTo,
|
|
85
132
|
expiration: new Date(decoded.exp * 1000).toISOString(),
|
|
86
133
|
daysLeft: Math.round((decoded.exp * 1000 - Date.now()) / (24 * 60 * 60 * 1000)),
|
|
87
|
-
|
|
134
|
+
licenseTypeObj,
|
|
88
135
|
};
|
|
89
136
|
} catch (err) {
|
|
90
137
|
try {
|
|
@@ -187,5 +234,6 @@ function isProApp() {
|
|
|
187
234
|
module.exports = {
|
|
188
235
|
checkLicense,
|
|
189
236
|
checkLicenseKey,
|
|
190
|
-
isProApp
|
|
237
|
+
isProApp,
|
|
238
|
+
licenseTypeById,
|
|
191
239
|
};
|
|
@@ -110,7 +110,12 @@ function getPluginBackendPath(packageName) {
|
|
|
110
110
|
return path.join(packagedPluginsDir(), packageName, 'dist', 'backend.js');
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
-
|
|
113
|
+
const res = path.join(pluginsdir(), packageName, 'dist', 'backend.js')
|
|
114
|
+
if (fs.existsSync(res)) {
|
|
115
|
+
return res;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
return require.resolve(packageName);
|
|
114
119
|
}
|
|
115
120
|
|
|
116
121
|
let archiveLinksCache = {};
|