dbgate-api-premium 5.5.7-alpha.45

package/src/utility/ColumnMapTransformStream.js

@@ -0,0 +1,21 @@
+const stream = require('stream');
+const { transformRowUsingColumnMap } = require('dbgate-tools');
+
+class ColumnMapTransformStream extends stream.Transform {
+  constructor(columns) {
+    super({ objectMode: true });
+    this.columns = columns;
+  }
+  _transform(chunk, encoding, done) {
+    if (chunk.__isStreamHeader) {
+      // skip stream header
+      done();
+      return;
+    }
+
+    this.push(transformRowUsingColumnMap(chunk, this.columns));
+    done();
+  }
+}
+
+module.exports = ColumnMapTransformStream;

package/src/utility/DatastoreProxy.js

@@ -0,0 +1,106 @@
+const crypto = require('crypto');
+const { fork } = require('child_process');
+const { handleProcessCommunication } = require('./processComm');
+const processArgs = require('../utility/processArgs');
+const pipeForkLogs = require('./pipeForkLogs');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+const logger = getLogger('DatastoreProxy');
+
+class DatastoreProxy {
+  constructor(file) {
+    this.subprocess = null;
+    this.disconnected = false;
+    this.file = file;
+    this.requests = {};
+    this.handle_response = this.handle_response.bind(this);
+    this.handle_ping = this.handle_ping.bind(this);
+    this.notifyChangedCallback = null;
+  }
+
+  handle_response({ msgid, rows }) {
+    const [resolve, reject] = this.requests[msgid];
+    resolve(rows);
+    delete this.requests[msgid];
+  }
+
+  handle_ping() {}
+
+  handle_notify({ msgid }) {
+    const [resolve, reject] = this.requests[msgid];
+    resolve();
+    delete this.requests[msgid];
+  }
+
+  async ensureSubprocess() {
+    if (!this.subprocess) {
+      this.subprocess = fork(
+        global['API_PACKAGE'] || process.argv[1],
+        [
+          '--is-forked-api',
+          '--start-process',
+          'jslDatastoreProcess',
+          ...processArgs.getPassArgs(),
+          // ...process.argv.slice(3),
+        ],
+        {
+          stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        }
+      );
+      pipeForkLogs(this.subprocess);
+
+      this.subprocess.on('message', message => {
+        // @ts-ignore
+        const { msgtype } = message;
+        if (handleProcessCommunication(message, this.subprocess)) return;
+
+        // if (this.disconnected) return;
+        this[`handle_${msgtype}`](message);
+      });
+      this.subprocess.on('exit', () => {
+        // if (this.disconnected) return;
+        this.subprocess = null;
+      });
+      this.subprocess.send({ msgtype: 'open', file: this.file });
+    }
+    return this.subprocess;
+  }
+
+  async getRows(offset, limit) {
+    await this.ensureSubprocess();
+    const msgid = crypto.randomUUID();
+    const promise = new Promise((resolve, reject) => {
+      this.requests[msgid] = [resolve, reject];
+      try {
+        this.subprocess.send({ msgtype: 'read', msgid, offset, limit });
+      } catch (err) {
+        logger.error(extractErrorLogData(err), 'Error getting rows');
+        this.subprocess = null;
+      }
+    });
+    return promise;
+  }
+
+  async notifyChangedCore() {
+    const msgid = crypto.randomUUID();
+    const promise = new Promise((resolve, reject) => {
+      this.requests[msgid] = [resolve, reject];
+      try {
+        this.subprocess.send({ msgtype: 'notify', msgid });
+      } catch (err) {
+        logger.error(extractErrorLogData(err), 'Error notifying subprocess');
+        this.subprocess = null;
+      }
+    });
+    return promise;
+  }
+
+  async notifyChanged(callback) {
+    this.notifyChangedCallback = callback;
+    await this.notifyChangedCore();
+    const call = this.notifyChangedCallback;
+    this.notifyChangedCallback = null;
+    if (call) call();
+  }
+}
+
+module.exports = DatastoreProxy;

package/src/utility/EnsureStreamHeaderStream.js

@@ -0,0 +1,31 @@
+const stream = require('stream');
+
+class EnsureStreamHeaderStream extends stream.Transform {
+  constructor() {
+    super({ objectMode: true });
+    this.wasHeader = false;
+  }
+  _transform(chunk, encoding, done) {
+    if (!this.wasHeader) {
+      if (chunk.__isDynamicStructure) {
+        // ignore dynamic structure header
+        done();
+        return;
+      }
+
+      if (!chunk.__isStreamHeader) {
+        this.push({
+          __isStreamHeader: true,
+          __isComputedStructure: true,
+          columns: Object.keys(chunk).map(columnName => ({ columnName })),
+        });
+      }
+
+      this.wasHeader = true;
+    }
+    this.push(chunk);
+    done();
+  }
+}
+
+module.exports = EnsureStreamHeaderStream;

package/src/utility/JsonLinesDatabase.js

@@ -0,0 +1,148 @@
+const crypto = require('crypto');
+const AsyncLock = require('async-lock');
+const fs = require('fs-extra');
+
+const lock = new AsyncLock();
+
+// const lineReader = require('line-reader');
+// const { fetchNextLineFromReader } = require('./JsonLinesDatastore');
+
+class JsonLinesDatabase {
+  constructor(filename) {
+    this.filename = filename;
+    this.data = [];
+    this.loadedOk = false;
+    this.loadPerformed = false;
+  }
+
+  async _save() {
+    if (!this.loadedOk) {
+      // don't override data
+      return;
+    }
+    await fs.writeFile(this.filename, this.data.map(x => JSON.stringify(x)).join('\n'));
+  }
+
+  async _ensureLoaded() {
+    if (!this.loadPerformed) {
+      await lock.acquire('reader', async () => {
+        if (!this.loadPerformed) {
+          if (!(await fs.exists(this.filename))) {
+            this.loadedOk = true;
+            this.loadPerformed = true;
+            return;
+          }
+          try {
+            const text = await fs.readFile(this.filename, { encoding: 'utf-8' });
+            this.data = text
+              .split('\n')
+              .filter(x => x.trim())
+              .map(x => JSON.parse(x));
+            this.loadedOk = true;
+          } catch (err) {
+            console.error(`Error loading file ${this.filename}`, err);
+          }
+          this.loadPerformed = true;
+        }
+      });
+    }
+  }
+
+  async insert(obj) {
+    await this._ensureLoaded();
+    if (obj._id && (await this.get(obj._id))) {
+      throw new Error(`Cannot insert duplicate ID ${obj._id} into ${this.filename}`);
+    }
+    const elem = obj._id
+      ? obj
+      : {
+          ...obj,
+          _id: crypto.randomUUID(),
+        };
+    this.data.push(elem);
+    await this._save();
+    return elem;
+  }
+
+  async get(id) {
+    await this._ensureLoaded();
+    return this.data.find(x => x._id == id);
+  }
+
+  async find(cond) {
+    await this._ensureLoaded();
+    if (cond) {
+      return this.data.filter(x => {
+        for (const key of Object.keys(cond)) {
+          if (x[key] != cond[key]) return false;
+        }
+        return true;
+      });
+    } else {
+      return this.data;
+    }
+  }
+
+  async update(obj) {
+    await this._ensureLoaded();
+    this.data = this.data.map(x => (x._id == obj._id ? obj : x));
+    await this._save();
+    return obj;
+  }
+
+  async updateAll(mapFunction) {
+    await this._ensureLoaded();
+    this.data = this.data.map(mapFunction);
+    await this._save();
+  }
+
+  async patch(id, values) {
+    await this._ensureLoaded();
+    this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));
+    await this._save();
+    return this.data.find(x => x._id == id);
+  }
+
+  async remove(id) {
+    await this._ensureLoaded();
+    const removed = this.data.find(x => x._id == id);
+    this.data = this.data.filter(x => x._id != id);
+    await this._save();
+    return removed;
+  }
+
+  //   async _openReader() {
+  //     return new Promise((resolve, reject) =>
+  //       lineReader.open(this.filename, (err, reader) => {
+  //         if (err) reject(err);
+  //         resolve(reader);
+  //       })
+  //     );
+  //   }
+
+  //   async _read() {
+  //     this.data = [];
+  //     if (!(await fs.exists(this.filename))) return;
+  //     try {
+  //       const reader = await this._openReader();
+  //       for (;;) {
+  //         const line = await fetchNextLineFromReader(reader);
+  //         if (!line) break;
+  //         this.data.push(JSON.parse(line));
+  //       }
+  //     } catch (err) {
+  //       console.error(`Error loading file ${this.filename}`, err);
+  //     }
+  //   }
+
+  //   async _write() {
+  //     const fw = fs.createWriteStream(this.filename);
+  //     for (const obj of this.data) {
+  //       await fw.write(JSON.stringify(obj));
+  //       await fw.write('\n');
+  //     }
+  //     await fw.end();
+  //   }
+}
+
+module.exports = JsonLinesDatabase;

package/src/utility/JsonLinesDatastore.js

@@ -0,0 +1,232 @@
+const crypto = require('crypto');
+const fs = require('fs');
+const os = require('os');
+const rimraf = require('rimraf');
+const path = require('path');
+const AsyncLock = require('async-lock');
+const lock = new AsyncLock();
+const stableStringify = require('json-stable-stringify');
+const { evaluateCondition } = require('dbgate-sqltree');
+const requirePluginFunction = require('./requirePluginFunction');
+const esort = require('external-sorting');
+const { jsldir } = require('./directories');
+const LineReader = require('./LineReader');
+
+class JsonLinesDatastore {
+  constructor(file, formatterFunction) {
+    this.file = file;
+    this.formatterFunction = formatterFunction;
+    this.reader = null;
+    this.readedDataRowCount = 0;
+    this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
+    this.notifyChangedCallback = null;
+    this.currentFilter = null;
+    this.currentSort = null;
+    this.rowFormatter = requirePluginFunction(formatterFunction);
+    this.sortedFiles = {};
+  }
+
+  static async sortFile(infile, outfile, sort) {
+    const tempDir = path.join(os.tmpdir(), crypto.randomUUID());
+    fs.mkdirSync(tempDir);
+
+    await esort
+      .default({
+        input: fs.createReadStream(infile),
+        output: fs.createWriteStream(outfile),
+        deserializer: JSON.parse,
+        serializer: JSON.stringify,
+        tempDir,
+        maxHeap: 100,
+        comparer: (a, b) => {
+          for (const item of sort) {
+            const { uniqueName, order } = item;
+            if (a[uniqueName] < b[uniqueName]) {
+              return order == 'ASC' ? -1 : 1;
+            }
+            if (a[uniqueName] > b[uniqueName]) {
+              return order == 'ASC' ? 1 : -1;
+            }
+          }
+          return 0;
+        },
+      })
+      .asc();
+
+    await new Promise(resolve => rimraf(tempDir, resolve));
+  }
+
+  async _closeReader() {
+    // console.log('CLOSING READER', this.reader);
+    if (!this.reader) return;
+    const reader = this.reader;
+    this.reader = null;
+    this.readedDataRowCount = 0;
+    this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
+    this.currentFilter = null;
+    this.currentSort = null;
+    await reader.close();
+  }
+
+  async notifyChanged(callback) {
+    this.notifyChangedCallback = callback;
+    await lock.acquire('reader', async () => {
+      this._closeReader();
+    });
+    const call = this.notifyChangedCallback;
+    this.notifyChangedCallback = null;
+    if (call) call();
+  }
+
+  async _openReader(fileName) {
+    // console.log('OPENING READER', fileName);
+    // console.log(fs.readFileSync(fileName, 'utf-8'));
+
+    const fileStream = fs.createReadStream(fileName);
+    return new LineReader(fileStream);
+  }
+
+  parseLine(line) {
+    const res = JSON.parse(line);
+    return this.rowFormatter ? this.rowFormatter(res) : res;
+  }
+
+  async _readLine(parse) {
+    // if (this.firstRowToBeReturned) {
+    //   const res = this.firstRowToBeReturned;
+    //   this.firstRowToBeReturned = null;
+    //   return res;
+    // }
+    for (;;) {
+      const line = await this.reader.readLine();
+      if (!line) {
+        // EOF
+        return null;
+      }
+
+      if (!this.readedSchemaRow) {
+        this.readedSchemaRow = true;
+        const parsedLine = JSON.parse(line);
+        if (parsedLine.__isStreamHeader) {
+          // skip to next line
+          continue;
+        }
+      }
+      if (this.currentFilter) {
+        const parsedLine = this.parseLine(line);
+        if (evaluateCondition(this.currentFilter, parsedLine)) {
+          this.readedDataRowCount += 1;
+          return parse ? parsedLine : true;
+        }
+      } else {
+        this.readedDataRowCount += 1;
+        return parse ? this.parseLine(line) : true;
+      }
+    }
+
+    // return new Promise((resolve, reject) => {
+    //   const reader = this.reader;
+    //   if (!reader.hasNextLine()) {
+    //     resolve(null);
+    //     return;
+    //   }
+
+    //   reader.nextLine((err, line) => {
+    //     if (err) {
+    //       reject(err);
+    //       return;
+    //     }
+    //     if (!this.readedSchemaRow) {
+    //       this.readedSchemaRow = true;
+    //       resolve(true);
+    //       return;
+    //     }
+    //     if (this.currentFilter) {
+    //       const parsedLine = JSON.parse(line);
+    //       if (evaluateCondition(this.currentFilter, parsedLine)) {
+    //         console.log('TRUE');
+    //         resolve(parse ? parsedLine : true);
+    //         this.readedDataRowCount += 1;
+    //         return;
+    //       } else {
+    //         console.log('FALSE');
+    //         // skip row
+    //         return;
+    //       }
+    //     }
+
+    //     this.readedDataRowCount += 1;
+    //     resolve(parse ? JSON.parse(line) : true);
+    //   });
+    // });
+  }
+
+  async _ensureReader(offset, filter, sort) {
+    if (
+      this.readedDataRowCount > offset ||
+      stableStringify(filter) != stableStringify(this.currentFilter) ||
+      stableStringify(sort) != stableStringify(this.currentSort)
+    ) {
+      this._closeReader();
+    }
+    if (!this.reader) {
+      const reader = await this._openReader(sort ? this.sortedFiles[stableStringify(sort)] : this.file);
+      this.reader = reader;
+      this.currentFilter = filter;
+      this.currentSort = sort;
+    }
+    // if (!this.readedSchemaRow) {
+    //   const line = await this._readLine(true); // skip structure
+    //   if (!line.__isStreamHeader) {
+    //     // line contains data
+    //     this.firstRowToBeReturned = line;
+    //   }
+    // }
+    while (this.readedDataRowCount < offset) {
+      const line = await this._readLine(false);
+      if (line == null) break;
+      // if (this.firstRowToBeReturned) {
+      //   this.firstRowToBeReturned = null;
+      // } else {
+      //   await this._readLine(false);
+      // }
+    }
+  }
+
+  async enumRows(eachRow) {
+    await lock.acquire('reader', async () => {
+      await this._ensureReader(0, null);
+      for (;;) {
+        const line = await this._readLine(true);
+        if (line == null) break;
+        const shouldContinue = eachRow(line);
+        if (!shouldContinue) break;
+      }
+    });
+  }
+
+  async getRows(offset, limit, filter, sort) {
+    const res = [];
+    if (sort && !this.sortedFiles[stableStringify(sort)]) {
+      const jslid = crypto.randomUUID();
+      const sortedFile = path.join(jsldir(), `${jslid}.jsonl`);
+      await JsonLinesDatastore.sortFile(this.file, sortedFile, sort);
+      this.sortedFiles[stableStringify(sort)] = sortedFile;
+    }
+    await lock.acquire('reader', async () => {
+      await this._ensureReader(offset, filter, sort);
+      // console.log(JSON.stringify(this.currentFilter, undefined, 2));
+      for (let i = 0; i < limit; i += 1) {
+        const line = await this._readLine(true);
+        // console.log('READED LINE', i);
+        if (line == null) break;
+        res.push(line);
+      }
+    });
+    return res;
+  }
+}
+
+module.exports = JsonLinesDatastore;

package/src/utility/LineReader.js

@@ -0,0 +1,88 @@
+const readline = require('readline');
+
+class Queue {
+  constructor() {
+    this.elements = {};
+    this.head = 0;
+    this.tail = 0;
+  }
+  enqueue(element) {
+    this.elements[this.tail] = element;
+    this.tail++;
+  }
+  dequeue() {
+    const item = this.elements[this.head];
+    delete this.elements[this.head];
+    this.head++;
+    return item;
+  }
+  peek() {
+    return this.elements[this.head];
+  }
+  getLength() {
+    return this.tail - this.head;
+  }
+  isEmpty() {
+    return this.getLength() === 0;
+  }
+}
+
+class LineReader {
+  constructor(input) {
+    this.input = input;
+    this.queue = new Queue();
+    this.resolve = null;
+    this.isEnded = false;
+    this.rl = readline.createInterface({
+      input,
+    });
+    this.input.pause();
+
+    this.rl.on('line', line => {
+      this.input.pause();
+      if (this.resolve) {
+        const resolve = this.resolve;
+        this.resolve = null;
+        resolve(line);
+        return;
+      }
+      this.queue.enqueue(line);
+    });
+
+    this.rl.on('close', () => {
+      if (this.resolve) {
+        const resolve = this.resolve;
+        this.resolve = null;
+        this.isEnded = true;
+        resolve(null);
+        return;
+      }
+      this.queue.enqueue(null);
+    });
+  }
+
+  readLine() {
+    if (this.isEnded) {
+      return Promise.resolve(null);
+    }
+
+    if (!this.queue.isEmpty()) {
+      const res = this.queue.dequeue();
+      if (res == null) this.isEnded = true;
+      return Promise.resolve(res);
+    }
+
+    this.input.resume();
+
+    return new Promise(resolve => {
+      this.resolve = resolve;
+    });
+  }
+
+  close() {
+    this.isEnded = true;
+    return new Promise(resolve => this.input.close(resolve));
+  }
+}
+
+module.exports = LineReader;