dbgate-api 5.1.5 → 5.1.7-alpha.13

package/src/controllers/jsldata.js

@@ -7,6 +7,7 @@ const DatastoreProxy = require('../utility/DatastoreProxy');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
+const requirePluginFunction = require('../utility/requirePluginFunction');
 const socket = require('../utility/socket');
 
 function readFirstLine(file) {
@@ -99,10 +100,13 @@ module.exports = {
   //   return readerInfo;
   // },
 
-  async ensureDatastore(jslid) {
+  async ensureDatastore(jslid, formatterFunction) {
     let datastore = this.datastores[jslid];
-    if (!datastore) {
-      datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    if (!datastore || datastore.formatterFunction != formatterFunction) {
+      if (datastore) {
+        datastore._closeReader();
+      }
+      datastore = new JsonLinesDatastore(getJslFileName(jslid), formatterFunction);
       // datastore = new DatastoreProxy(getJslFileName(jslid));
       this.datastores[jslid] = datastore;
     }
@@ -131,8 +135,8 @@ module.exports = {
   },
 
   getRows_meta: true,
-  async getRows({ jslid, offset, limit, filters }) {
-    const datastore = await this.ensureDatastore(jslid);
+  async getRows({ jslid, offset, limit, filters, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
     return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
   },
 
@@ -150,8 +154,8 @@ module.exports = {
   },
 
   loadFieldValues_meta: true,
-  async loadFieldValues({ jslid, field, search }) {
-    const datastore = await this.ensureDatastore(jslid);
+  async loadFieldValues({ jslid, field, search, formatterFunction }) {
+    const datastore = await this.ensureDatastore(jslid, formatterFunction);
     const res = new Set();
     await datastore.enumRows(row => {
       if (!filterName(search, row[field])) return true;
@@ -188,4 +192,85 @@ module.exports = {
     await fs.promises.writeFile(getJslFileName(jslid), text);
     return true;
   },
+
+  extractTimelineChart_meta: true,
+  async extractTimelineChart({ jslid, timestampFunction, aggregateFunction, measures }) {
+    const timestamp = requirePluginFunction(timestampFunction);
+    const aggregate = requirePluginFunction(aggregateFunction);
+    const datastore = new JsonLinesDatastore(getJslFileName(jslid));
+    let mints = null;
+    let maxts = null;
+    // pass 1 - counts stats, time range
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      if (!mints || ts < mints) mints = ts;
+      if (!maxts || ts > maxts) maxts = ts;
+      return true;
+    });
+    const minTime = new Date(mints).getTime();
+    const maxTime = new Date(maxts).getTime();
+    const duration = maxTime - minTime;
+    const STEPS = 100;
+    let stepCount = duration > 100 * 1000 ? STEPS : Math.round((maxTime - minTime) / 1000);
+    if (stepCount < 2) {
+      stepCount = 2;
+    }
+    const stepDuration = duration / stepCount;
+    const labels = _.range(stepCount).map(i => new Date(minTime + stepDuration / 2 + stepDuration * i));
+
+    // const datasets = measures.map(m => ({
+    //   label: m.label,
+    //   data: Array(stepCount).fill(0),
+    // }));
+
+    const mproc = measures.map(m => ({
+      ...m,
+    }));
+
+    const data = Array(stepCount)
+      .fill(0)
+      .map(() => ({}));
+
+    // pass 2 - count measures
+    await datastore.enumRows(row => {
+      const ts = timestamp(row);
+      let part = Math.round((new Date(ts).getTime() - minTime) / stepDuration);
+      if (part < 0) part = 0;
+      if (part >= stepCount) part - stepCount - 1;
+      if (data[part]) {
+        data[part] = aggregate(data[part], row, stepDuration);
+      }
+      return true;
+    });
+
+    datastore._closeReader();
+
+    // const measureByField = _.fromPairs(measures.map((m, i) => [m.field, i]));
+
+    // for (let mindex = 0; mindex < measures.length; mindex++) {
+    //   for (let stepIndex = 0; stepIndex < stepCount; stepIndex++) {
+    //     const measure = measures[mindex];
+    //     if (measure.perSecond) {
+    //       datasets[mindex].data[stepIndex] /= stepDuration / 1000;
+    //     }
+    //     if (measure.perField) {
+    //       datasets[mindex].data[stepIndex] /= datasets[measureByField[measure.perField]].data[stepIndex];
+    //     }
+    //   }
+    // }
+
+    // for (let i = 0; i < measures.length; i++) {
+    //   if (measures[i].hidden) {
+    //     datasets[i] = null;
+    //   }
+    // }
+
+    return {
+      labels,
+      datasets: mproc.map(m => ({
+        label: m.label,
+        data: data.map(d => d[m.field] || 0),
+      })),
+    };
+  },
 };

package/src/controllers/serverConnections.js

@@ -1,6 +1,7 @@
 const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
+const uuidv1 = require('uuid/v1');
 const _ = require('lodash');
 const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -8,23 +9,25 @@ const lock = new AsyncLock();
 const config = require('./config');
 const processArgs = require('../utility/processArgs');
 const { testConnectionPermission } = require('../utility/hasPermission');
+const { MissingCredentialsError } = require('../utility/exceptions');
 
 module.exports = {
   opened: [],
   closed: {},
   lastPinged: {},
+  requests: {},
 
   handle_databases(conid, { databases }) {
     const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
     existing.databases = databases;
-    socket.emitChanged(`database-list-changed-${conid}`);
+    socket.emitChanged(`database-list-changed`, { conid });
   },
   handle_version(conid, { version }) {
     const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
     existing.version = version;
-    socket.emitChanged(`server-version-changed-${conid}`);
+    socket.emitChanged(`server-version-changed`, { conid });
   },
   handle_status(conid, { status }) {
     const existing = this.opened.find(x => x.conid == conid);
@@ -33,12 +36,20 @@ module.exports = {
     socket.emitChanged(`server-status-changed`);
   },
   handle_ping() {},
+  handle_response(conid, { msgid, ...response }) {
+    const [resolve, reject] = this.requests[msgid];
+    resolve(response);
+    delete this.requests[msgid];
+  },
 
   async ensureOpened(conid) {
     const res = await lock.acquire(conid, async () => {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
       const connection = await connections.getCore({ conid });
+      if (connection.passwordMode == 'askPassword' || connection.passwordMode == 'askUser') {
+        throw new MissingCredentialsError({ conid, passwordMode: connection.passwordMode });
+      }
       const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
         '--is-forked-api',
         '--start-process',
@@ -120,9 +131,9 @@ module.exports = {
   },
 
   ping_meta: true,
-  async ping({ connections }) {
+  async ping({ conidArray }) {
     await Promise.all(
-      _.uniq(connections).map(async conid => {
+      _.uniq(conidArray).map(async conid => {
         const last = this.lastPinged[conid];
         if (last && new Date().getTime() - last < 30 * 1000) {
           return Promise.resolve();
@@ -161,4 +172,41 @@ module.exports = {
     opened.subprocess.send({ msgtype: 'dropDatabase', name });
     return { status: 'ok' };
   },
+
+  sendRequest(conn, message) {
+    const msgid = uuidv1();
+    const promise = new Promise((resolve, reject) => {
+      this.requests[msgid] = [resolve, reject];
+      conn.subprocess.send({ msgid, ...message });
+    });
+    return promise;
+  },
+
+  async loadDataCore(msgtype, { conid, ...args }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    const res = await this.sendRequest(opened, { msgtype, ...args });
+    if (res.errorMessage) {
+      console.error(res.errorMessage);
+
+      return {
+        errorMessage: res.errorMessage,
+      };
+    }
+    return res.result || null;
+  },
+
+  serverSummary_meta: true,
+  async serverSummary({ conid }, req) {
+    testConnectionPermission(conid, req);
+    return this.loadDataCore('serverSummary', { conid });
+  },
+
+  summaryCommand_meta: true,
+  async summaryCommand({ conid, command, row }, req) {
+    testConnectionPermission(conid, req);
+    const opened = await this.ensureOpened(conid);
+    if (opened.connection.isReadOnly) return false;
+    return this.loadDataCore('summaryCommand', { conid, command, row });
+  },
 };

package/src/controllers/sessions.js

@@ -103,6 +103,12 @@ module.exports = {
       if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](sesid, message);
     });
+    subprocess.on('exit', () => {
+      this.opened = this.opened.filter(x => x.sesid != sesid);
+      this.dispatchMessage(sesid, 'Query session closed');
+      socket.emit(`session-closed-${sesid}`);
+    });
+
     subprocess.send({ msgtype: 'connect', ...connection, database });
     return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
@@ -144,6 +150,31 @@ module.exports = {
     return true;
   },
 
+  startProfiler_meta: true,
+  async startProfiler({ sesid }) {
+    const jslid = uuidv1();
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+
+    console.log(`Starting profiler, sesid=${sesid}`);
+    session.loadingReader_jslid = jslid;
+    session.subprocess.send({ msgtype: 'startProfiler', jslid });
+
+    return { state: 'ok', jslid };
+  },
+
+  stopProfiler_meta: true,
+  async stopProfiler({ sesid }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+    session.subprocess.send({ msgtype: 'stopProfiler' });
+    return { state: 'ok' };
+  },
+
   // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);
@@ -165,6 +196,17 @@ module.exports = {
     return { state: 'ok' };
   },
 
+  ping_meta: true,
+  async ping({ sesid }) {
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (!session) {
+      throw new Error('Invalid session');
+    }
+    session.subprocess.send({ msgtype: 'ping' });
+
+    return { state: 'ok' };
+  },
+
   // runCommand_meta: true,
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '5.1.5',
-  buildTime: '2022-10-15T20:12:15.112Z'
+  version: '5.1.7-alpha.13',
+  buildTime: '2023-01-02T17:41:24.485Z'
 };

package/src/main.js

@@ -20,6 +20,7 @@ const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const archive = require('./controllers/archive');
 const apps = require('./controllers/apps');
+const auth = require('./controllers/auth');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
@@ -41,7 +42,7 @@ function start() {
   const server = http.createServer(app);
 
   const logins = getLogins();
-  if (logins) {
+  if (logins && process.env.BASIC_AUTH) {
     app.use(
       basicAuth({
         users: _.fromPairs(logins.map(x => [x.login, x.password])),
@@ -53,6 +54,25 @@ function start() {
 
   app.use(cors());
 
+  if (platformInfo.isDocker) {
+    // server static files inside docker container
+    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
+  } else if (platformInfo.isNpmDist) {
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
+  } else if (process.env.DEVWEB) {
+    console.log('__dirname', __dirname);
+    console.log(path.join(__dirname, '../../web/public/build'));
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
+  } else {
+    app.get(getExpressPath('/'), (req, res) => {
+      res.send('DbGate API');
+    });
+  }
+
+  if (auth.shouldAuthorizeApi()) {
+    app.use(auth.authMiddleware);
+  }
+
   app.get(getExpressPath('/stream'), async function (req, res) {
     res.set({
       'Cache-Control': 'no-cache',
@@ -88,14 +108,10 @@ function start() {
   app.use(getExpressPath('/runners/data'), express.static(rundir()));
 
   if (platformInfo.isDocker) {
-    // server static files inside docker container
-    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
-
     const port = process.env.PORT || 3000;
     console.log('DbGate API listening on port (docker build)', port);
     server.listen(port);
   } else if (platformInfo.isNpmDist) {
-    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
     getPort({
       port: parseInt(
         // @ts-ignore
@@ -107,18 +123,10 @@ function start() {
       });
     });
   } else if (process.env.DEVWEB) {
-    console.log('__dirname', __dirname);
-    console.log(path.join(__dirname, '../../web/public/build'));
-    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../web/public')));
-
     const port = process.env.PORT || 3000;
     console.log('DbGate API & web listening on port (dev web build)', port);
     server.listen(port);
   } else {
-    app.get(getExpressPath('/'), (req, res) => {
-      res.send('DbGate API');
-    });
-
     const port = process.env.PORT || 3000;
     console.log('DbGate API listening on port (dev API build)', port);
     server.listen(port);
@@ -157,6 +165,7 @@ function useAllControllers(app, electron) {
   useController(app, electron, '/scheduler', scheduler);
   useController(app, electron, '/query-history', queryHistory);
   useController(app, electron, '/apps', apps);
+  useController(app, electron, '/auth', auth);
 }
 
 function setElectronSender(electronSender) {

package/src/nativeModulesContent.js

@@ -2,6 +2,6 @@
 // this file is generated automatically by script fillNativeModules.js, do not edit it manually
 const content = {};
 
-content['better-sqlite3'] = () => require('better-sqlite3');
+content.oracledb = () => require('oracledb');content['better-sqlite3'] = () => require('better-sqlite3');
 
 module.exports = content;

package/src/proc/databaseConnectionProcess.js

@@ -335,11 +335,11 @@ function start() {
 
   setInterval(() => {
     const time = new Date().getTime();
-    if (time - lastPing > 120 * 1000) {
+    if (time - lastPing > 40 * 1000) {
       console.log('Database connection not alive, exiting');
       process.exit(0);
     }
-  }, 60 * 1000);
+  }, 10 * 1000);
 
   process.on('message', async message => {
     if (handleProcessCommunication(message)) return;

package/src/proc/serverConnectionProcess.js

@@ -10,6 +10,7 @@ let storedConnection;
 let lastDatabases = null;
 let lastStatus = null;
 let lastPing = null;
+let afterConnectCallbacks = [];
 
 async function handleRefresh() {
   const driver = requireEngineDriver(storedConnection);
@@ -74,6 +75,18 @@ async function handleConnect(connection) {
     // console.error(err);
     setTimeout(() => process.exit(1), 1000);
   }
+
+  for (const [resolve] of afterConnectCallbacks) {
+    resolve();
+  }
+  afterConnectCallbacks = [];
+}
+
+function waitConnected() {
+  if (systemConnection) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    afterConnectCallbacks.push([resolve, reject]);
+  });
 }
 
 function handlePing() {
@@ -94,9 +107,30 @@ async function handleDatabaseOp(op, { name }) {
   await handleRefresh();
 }
 
+async function handleDriverDataCore(msgid, callMethod) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await callMethod(driver);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
+async function handleServerSummary({ msgid }) {
+  return handleDriverDataCore(msgid, driver => driver.serverSummary(systemConnection));
+}
+
+async function handleSummaryCommand({ msgid, command, row }) {
+  return handleDriverDataCore(msgid, driver => driver.summaryCommand(systemConnection, command, row));
+}
+
 const messageHandlers = {
   connect: handleConnect,
   ping: handlePing,
+  serverSummary: handleServerSummary,
+  summaryCommand: handleSummaryCommand,
   createDatabase: props => handleDatabaseOp('createDatabase', props),
   dropDatabase: props => handleDatabaseOp('dropDatabase', props),
 };
@@ -111,11 +145,11 @@ function start() {
 
   setInterval(() => {
     const time = new Date().getTime();
-    if (time - lastPing > 120 * 1000) {
+    if (time - lastPing > 40 * 1000) {
       console.log('Server connection not alive, exiting');
       process.exit(0);
     }
-  }, 60 * 1000);
+  }, 10 * 1000);
 
   process.on('message', async message => {
     if (handleProcessCommunication(message)) return;

package/src/proc/sessionProcess.js

@@ -15,6 +15,8 @@ let systemConnection;
 let storedConnection;
 let afterConnectCallbacks = [];
 // let currentHandlers = [];
+let lastPing = null;
+let currentProfiler = null;
 
 class TableWriter {
   constructor() {
@@ -209,6 +211,31 @@ function waitConnected() {
   });
 }
 
+async function handleStartProfiler({ jslid }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({ msgtype: 'done' });
+    return;
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  currentProfiler = await driver.startProfiler(systemConnection, {
+    row: data => writer.rowFromReader(data),
+  });
+  currentProfiler.writer = writer;
+}
+
+async function handleStopProfiler({ jslid }) {
+  const driver = requireEngineDriver(storedConnection);
+  currentProfiler.writer.close();
+  driver.stopProfiler(systemConnection, currentProfiler);
+  currentProfiler = null;
+}
+
 async function handleExecuteQuery({ sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
@@ -271,10 +298,17 @@ async function handleExecuteReader({ jslid, sql, fileName }) {
   });
 }
 
+function handlePing() {
+  lastPing = new Date().getTime();
+}
+
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
   executeReader: handleExecuteReader,
+  startProfiler: handleStartProfiler,
+  stopProfiler: handleStopProfiler,
+  ping: handlePing,
   // cancel: handleCancel,
 };
 
@@ -285,6 +319,17 @@ async function handleMessage({ msgtype, ...other }) {
 
 function start() {
   childProcessChecker();
+
+  lastPing = new Date().getTime();
+
+  setInterval(() => {
+    const time = new Date().getTime();
+    if (time - lastPing > 25 * 1000) {
+      console.log('Session not alive, exiting');
+      process.exit(0);
+    }
+  }, 10 * 1000);
+
   process.on('message', async message => {
     if (handleProcessCommunication(message)) return;
     try {

package/src/utility/JsonLinesDatabase.js

@@ -90,6 +90,12 @@ class JsonLinesDatabase {
     return obj;
   }
 
+  async updateAll(mapFunction) {
+    await this._ensureLoaded();
+    this.data = this.data.map(mapFunction);
+    await this._save();
+  }
+
   async patch(id, values) {
     await this._ensureLoaded();
     this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));

package/src/utility/JsonLinesDatastore.js

@@ -3,6 +3,7 @@ const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
+const requirePluginFunction = require('./requirePluginFunction');
 
 function fetchNextLineFromReader(reader) {
   return new Promise((resolve, reject) => {
@@ -22,14 +23,16 @@ function fetchNextLineFromReader(reader) {
 }
 
 class JsonLinesDatastore {
-  constructor(file) {
+  constructor(file, formatterFunction) {
     this.file = file;
+    this.formatterFunction = formatterFunction;
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
     // this.firstRowToBeReturned = null;
     this.notifyChangedCallback = null;
     this.currentFilter = null;
+    this.rowFormatter = requirePluginFunction(formatterFunction);
   }
 
   _closeReader() {
@@ -62,6 +65,11 @@ class JsonLinesDatastore {
     );
   }
 
+  parseLine(line) {
+    const res = JSON.parse(line);
+    return this.rowFormatter ? this.rowFormatter(res) : res;
+  }
+
   async _readLine(parse) {
     // if (this.firstRowToBeReturned) {
     //   const res = this.firstRowToBeReturned;
@@ -84,14 +92,14 @@ class JsonLinesDatastore {
         }
       }
       if (this.currentFilter) {
-        const parsedLine = JSON.parse(line);
+        const parsedLine = this.parseLine(line);
         if (evaluateCondition(this.currentFilter, parsedLine)) {
           this.readedDataRowCount += 1;
           return parse ? parsedLine : true;
         }
       } else {
         this.readedDataRowCount += 1;
-        return parse ? JSON.parse(line) : true;
+        return parse ? this.parseLine(line) : true;
       }
     }
 

package/src/utility/exceptions.js

@@ -0,0 +1,9 @@
+class MissingCredentialsError {
+  constructor(detail) {
+    this.detail = detail;
+  }
+}
+
+module.exports = {
+  MissingCredentialsError,
+};

package/src/utility/requirePluginFunction.js

@@ -0,0 +1,16 @@
+const _ = require('lodash');
+const requirePlugin = require('../shell/requirePlugin');
+
+function requirePluginFunction(functionName) {
+  if (!functionName) return null;
+  if (functionName.includes('@')) {
+    const [shortName, packageName] = functionName.split('@');
+    const plugin = requirePlugin(packageName);
+    if (plugin.functions) {
+      return plugin.functions[shortName];
+    }
+  }
+  return null;
+}
+
+module.exports = requirePluginFunction;

package/src/utility/socket.js

@@ -1,4 +1,5 @@
 const _ = require('lodash');
+const stableStringify = require('json-stable-stringify');
 
 const sseResponses = [];
 let electronSender = null;
@@ -27,12 +28,12 @@ module.exports = {
       electronSender.send(message, data == null ? null : data);
     }
     for (const res of sseResponses) {
-      res.write(`event: ${message}\ndata: ${JSON.stringify(data == null ? null : data)}\n\n`);
+      res.write(`event: ${message}\ndata: ${stableStringify(data == null ? null : data)}\n\n`);
     }
   },
-  emitChanged(key) {
+  emitChanged(key, params = undefined) {
     // console.log('EMIT CHANGED', key);
-    this.emit('changed-cache', key);
+    this.emit('changed-cache', { key, ...params });
     // this.emit(key);
   },
 };