agora-foundation 1.0.0

package/lib/worker/handler/fs-log.js

@@ -0,0 +1,156 @@
+"use strict";
+
+require("core-js/modules/es.symbol.js");
+require("core-js/modules/es.symbol.description.js");
+require("core-js/modules/es.symbol.iterator.js");
+require("core-js/modules/es.error.cause.js");
+require("core-js/modules/es.error.to-string.js");
+require("core-js/modules/es.array.from.js");
+require("core-js/modules/es.array.is-array.js");
+require("core-js/modules/es.array.iterator.js");
+require("core-js/modules/es.array.slice.js");
+require("core-js/modules/es.date.to-string.js");
+require("core-js/modules/es.function.name.js");
+require("core-js/modules/es.object.define-property.js");
+require("core-js/modules/es.object.to-string.js");
+require("core-js/modules/es.regexp.exec.js");
+require("core-js/modules/es.regexp.test.js");
+require("core-js/modules/es.regexp.to-string.js");
+require("core-js/modules/es.string.iterator.js");
+require("core-js/modules/web.dom-collections.iterator.js");
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.collectLogsOnFs = exports.clearTempLogs = void 0;
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+require("core-js/modules/es.array.concat.js");
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+var _jszip = _interopRequireDefault(require("jszip"));
+var _constants = require("../constants");
+var _db = require("./db");
+var _reply = require("./reply");
+function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it["return"] != null) it["return"](); } finally { if (didErr) throw err; } } }; }
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) arr2[i] = arr[i]; return arr2; }
+var collectLogsOnFs = exports.collectLogsOnFs = /*#__PURE__*/function () {
+  var _ref = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(self, callId, logDirPath) {
+    var fs;
+    return _regenerator["default"].wrap(function _callee2$(_context2) {
+      while (1) switch (_context2.prev = _context2.next) {
+        case 0:
+          try {
+            fs = window.require("fs");
+            fs.readdir(logDirPath, /*#__PURE__*/function () {
+              var _ref2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee(err, files) {
+                var zip, _iterator, _step, file, filePath, logsStr, blob, buffer;
+                return _regenerator["default"].wrap(function _callee$(_context) {
+                  while (1) switch (_context.prev = _context.next) {
+                    case 0:
+                      if (!err) {
+                        _context.next = 5;
+                        break;
+                      }
+                      console.error("[logger] failed to read log dir.", err);
+                      (0, _reply.reply)(self, callId, _constants.WorkerDirectives.COLLECT_LOGS_ON_FS, {
+                        isSuccess: false
+                      });
+                      _context.next = 19;
+                      break;
+                    case 5:
+                      if (!(files.length === 0)) {
+                        _context.next = 9;
+                        break;
+                      }
+                      (0, _reply.reply)(self, callId, _constants.WorkerDirectives.COLLECT_LOGS_ON_FS, {
+                        isSuccess: true,
+                        buffer: null
+                      });
+                      _context.next = 19;
+                      break;
+                    case 9:
+                      zip = new _jszip["default"]();
+                      _iterator = _createForOfIteratorHelper(files);
+                      try {
+                        for (_iterator.s(); !(_step = _iterator.n()).done;) {
+                          file = _step.value;
+                          filePath = "".concat(logDirPath, "/").concat(file);
+                          logsStr = fs.readFileSync(filePath, "utf8");
+                          zip.file(file, logsStr);
+                        }
+                      } catch (err) {
+                        _iterator.e(err);
+                      } finally {
+                        _iterator.f();
+                      }
+                      _context.next = 14;
+                      return zip.generateAsync({
+                        type: "blob"
+                      });
+                    case 14:
+                      blob = _context.sent;
+                      _context.next = 17;
+                      return blob.arrayBuffer();
+                    case 17:
+                      buffer = _context.sent;
+                      (0, _reply.reply)(self, callId, _constants.WorkerDirectives.COLLECT_LOGS_ON_FS, {
+                        isSuccess: true,
+                        buffer: buffer
+                      });
+                    case 19:
+                    case "end":
+                      return _context.stop();
+                  }
+                }, _callee);
+              }));
+              return function (_x4, _x5) {
+                return _ref2.apply(this, arguments);
+              };
+            }());
+          } catch (e) {
+            console.error("[logger] failed to get buffer.", e);
+            (0, _reply.reply)(self, callId, _constants.WorkerDirectives.GET_BUFFER, {
+              isSuccess: false
+            });
+          }
+        case 1:
+        case "end":
+          return _context2.stop();
+      }
+    }, _callee2);
+  }));
+  return function collectLogsOnFs(_x, _x2, _x3) {
+    return _ref.apply(this, arguments);
+  };
+}();
+var clearTempLogs = exports.clearTempLogs = /*#__PURE__*/function () {
+  var _ref3 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee3(self, callId) {
+    return _regenerator["default"].wrap(function _callee3$(_context3) {
+      while (1) switch (_context3.prev = _context3.next) {
+        case 0:
+          _context3.prev = 0;
+          _context3.next = 3;
+          return _db.db.tempLogs.clear();
+        case 3:
+          (0, _reply.reply)(self, callId, _constants.WorkerDirectives.CLEAR_TEMP_LOGS, {
+            isSuccess: true
+          });
+          _context3.next = 10;
+          break;
+        case 6:
+          _context3.prev = 6;
+          _context3.t0 = _context3["catch"](0);
+          console.error("[logger] failed to clear temp logs.", _context3.t0);
+          (0, _reply.reply)(self, callId, _constants.WorkerDirectives.CLEAR_TEMP_LOGS, {
+            isSuccess: false
+          });
+        case 10:
+        case "end":
+          return _context3.stop();
+      }
+    }, _callee3, null, [[0, 6]]);
+  }));
+  return function clearTempLogs(_x6, _x7) {
+    return _ref3.apply(this, arguments);
+  };
+}();

package/lib/worker/handler/log.d.ts

@@ -0,0 +1,12 @@
+import { WorkerContext } from '../type';
+type Log = {
+    content: string;
+    label: string;
+    created_at: number;
+    timestamp: string;
+};
+export declare const writeLog: (self: WorkerContext, callId: number, log: Log) => void;
+export declare const collectLogs: (self: WorkerContext, callId: number, labels: string[]) => Promise<void>;
+export declare const deleteLogs: (self: WorkerContext, callId: number, labels: string[], lastId: number) => Promise<void>;
+export declare const setMaxRecords: (self: WorkerContext, callId: number, max: number) => void;
+export {};

package/lib/worker/handler/log.js

@@ -0,0 +1,246 @@
+"use strict";
+
+require("core-js/modules/es.symbol.js");
+require("core-js/modules/es.symbol.description.js");
+require("core-js/modules/es.symbol.iterator.js");
+require("core-js/modules/es.error.cause.js");
+require("core-js/modules/es.error.to-string.js");
+require("core-js/modules/es.array.from.js");
+require("core-js/modules/es.array.is-array.js");
+require("core-js/modules/es.array.slice.js");
+require("core-js/modules/es.function.name.js");
+require("core-js/modules/es.object.define-property.js");
+require("core-js/modules/es.object.to-string.js");
+require("core-js/modules/es.regexp.exec.js");
+require("core-js/modules/es.regexp.test.js");
+require("core-js/modules/es.regexp.to-string.js");
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.writeLog = exports.setMaxRecords = exports.deleteLogs = exports.collectLogs = void 0;
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+require("core-js/modules/es.array.concat.js");
+require("core-js/modules/es.array.iterator.js");
+require("core-js/modules/es.array.push.js");
+require("core-js/modules/es.array.reverse.js");
+require("core-js/modules/es.date.to-string.js");
+require("core-js/modules/es.string.iterator.js");
+require("core-js/modules/web.dom-collections.iterator.js");
+var _debounce = _interopRequireDefault(require("lodash/debounce"));
+var _db = require("./db");
+var _mutex = require("../mutex");
+var _constants = require("../constants");
+var _reply = require("./reply");
+var _raceCondition = require("../../utilities/race-condition");
+var _jszip = _interopRequireDefault(require("jszip"));
+function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it["return"] != null) it["return"](); } finally { if (didErr) throw err; } } }; }
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) arr2[i] = arr[i]; return arr2; }
+var MAX_WAIT_TIME = 3000;
+var mutex = new _mutex.Mutex();
+var maxRecords = 50000;
+var logsCount = 0;
+var logs = [];
+var writeLog = exports.writeLog = function writeLog(self, callId, log) {
+  logsCount++;
+  log.content = "[".concat(logsCount, "]").concat(log.content);
+  logs.push(log);
+  debouncedWriting();
+  // reply(self, callId, WorkerDirectives.WRITE_LOG, { isSuccess: true });
+};
+var debouncedWriting = (0, _debounce["default"])(function () {
+  mutex.dispatch(function () {
+    return (0, _raceCondition.timeout)((0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee() {
+      var time1, logscopy, count, exceedNo, total, pk, time2;
+      return _regenerator["default"].wrap(function _callee$(_context) {
+        while (1) switch (_context.prev = _context.next) {
+          case 0:
+            time1 = new Date().getTime();
+            logscopy = logs;
+            logs = [];
+            _context.prev = 3;
+            _context.next = 6;
+            return _db.db.logs.count();
+          case 6:
+            count = _context.sent;
+            exceedNo = logscopy.length + count - maxRecords;
+            if (!(exceedNo > 0)) {
+              _context.next = 15;
+              break;
+            }
+            console.log("[worker] the count of stored logs exceeded limit of ".concat(maxRecords, " for ").concat(exceedNo, "."));
+            console.log("[worker] start deleting records. exceeded: ".concat(exceedNo));
+            _context.next = 13;
+            return _db.db.logs.orderBy(':id').reverse().limit(exceedNo)["delete"]();
+          case 13:
+            total = _context.sent;
+            console.log("[worker] done deleting records. total: ".concat(total));
+          case 15:
+            _context.next = 20;
+            break;
+          case 17:
+            _context.prev = 17;
+            _context.t0 = _context["catch"](3);
+            console.error("[worker] clear db failed.", _context.t0);
+          case 20:
+            _context.prev = 20;
+            console.log("[worker] start writing logs. total: ".concat(logscopy.length));
+            _context.next = 24;
+            return _db.db.logs.bulkPut(logscopy);
+          case 24:
+            pk = _context.sent;
+            time2 = new Date().getTime();
+            console.log("[worker] done writing logs in ".concat(time2 - time1, "ms. total: ").concat(logscopy.length, ", lastId: ").concat(pk));
+            _context.next = 32;
+            break;
+          case 29:
+            _context.prev = 29;
+            _context.t1 = _context["catch"](20);
+            console.error("[worker] bulkPut ".concat(logscopy.length, " failed."), _context.t1);
+          case 32:
+          case "end":
+            return _context.stop();
+        }
+      }, _callee, null, [[3, 17], [20, 29]]);
+    }))(), MAX_WAIT_TIME);
+  })["catch"](function (e) {
+    console.log('[worker] an error during dispatching writeLog', e);
+  });
+}, 2 * 1000, {
+  maxWait: 30 * 1000
+});
+var collectLogs = exports.collectLogs = /*#__PURE__*/function () {
+  var _ref2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(self, callId, labels) {
+    var blob, lastId, _iterator, _step, _loop, buffer;
+    return _regenerator["default"].wrap(function _callee2$(_context3) {
+      while (1) switch (_context3.prev = _context3.next) {
+        case 0:
+          blob = new Blob();
+          lastId = 0;
+          _context3.prev = 2;
+          _iterator = _createForOfIteratorHelper(labels);
+          _context3.prev = 4;
+          _loop = /*#__PURE__*/_regenerator["default"].mark(function _loop() {
+            var label, logsStr, count, zip;
+            return _regenerator["default"].wrap(function _loop$(_context2) {
+              while (1) switch (_context2.prev = _context2.next) {
+                case 0:
+                  label = _step.value;
+                  logsStr = '';
+                  count = 0;
+                  _context2.next = 5;
+                  return _db.db.logs.where('label').equals(label).each(function (e) {
+                    count++;
+                    logsStr += e.content + '\n';
+                    if (e.id > lastId) lastId = e.id;
+                  });
+                case 5:
+                  zip = new _jszip["default"]();
+                  zip.file("".concat(label, "-logs.txt"), logsStr);
+                  _context2.next = 9;
+                  return zip.generateAsync({
+                    type: 'blob'
+                  });
+                case 9:
+                  blob = _context2.sent;
+                  console.log("[worker] collected ".concat(count, " logs by labels [").concat(label, "]."));
+                case 11:
+                case "end":
+                  return _context2.stop();
+              }
+            }, _loop);
+          });
+          _iterator.s();
+        case 7:
+          if ((_step = _iterator.n()).done) {
+            _context3.next = 11;
+            break;
+          }
+          return _context3.delegateYield(_loop(), "t0", 9);
+        case 9:
+          _context3.next = 7;
+          break;
+        case 11:
+          _context3.next = 16;
+          break;
+        case 13:
+          _context3.prev = 13;
+          _context3.t1 = _context3["catch"](4);
+          _iterator.e(_context3.t1);
+        case 16:
+          _context3.prev = 16;
+          _iterator.f();
+          return _context3.finish(16);
+        case 19:
+          _context3.next = 25;
+          break;
+        case 21:
+          _context3.prev = 21;
+          _context3.t2 = _context3["catch"](2);
+          console.error("[worker] failed to collect logs.", _context3.t2);
+          (0, _reply.reply)(self, callId, _constants.WorkerDirectives.COLLECT_LOGS, {
+            isSuccess: false
+          });
+        case 25:
+          console.log("[worker] completed collecting logs, compressed size: ".concat(blob.size, " bytes."));
+          _context3.next = 28;
+          return blob.arrayBuffer();
+        case 28:
+          buffer = _context3.sent;
+          (0, _reply.reply)(self, callId, _constants.WorkerDirectives.COLLECT_LOGS, {
+            isSuccess: true,
+            buffer: buffer,
+            lastId: lastId
+          }, [buffer]);
+        case 30:
+        case "end":
+          return _context3.stop();
+      }
+    }, _callee2, null, [[2, 21], [4, 13, 16, 19]]);
+  }));
+  return function collectLogs(_x, _x2, _x3) {
+    return _ref2.apply(this, arguments);
+  };
+}();
+var deleteLogs = exports.deleteLogs = /*#__PURE__*/function () {
+  var _ref3 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee3(self, callId, labels, lastId) {
+    var count;
+    return _regenerator["default"].wrap(function _callee3$(_context4) {
+      while (1) switch (_context4.prev = _context4.next) {
+        case 0:
+          _context4.prev = 0;
+          _context4.next = 3;
+          return _db.db.logs.where('id').belowOrEqual(lastId)["delete"]();
+        case 3:
+          count = _context4.sent;
+          console.log("[logger] deleted ".concat(count, " logs from ").concat(lastId, "."));
+          (0, _reply.reply)(self, callId, _constants.WorkerDirectives.DELETE_LOGS, {
+            isSuccess: true
+          });
+          _context4.next = 12;
+          break;
+        case 8:
+          _context4.prev = 8;
+          _context4.t0 = _context4["catch"](0);
+          console.error("[logger] failed to delete logs.", _context4.t0);
+          (0, _reply.reply)(self, callId, _constants.WorkerDirectives.DELETE_LOGS, {
+            isSuccess: false
+          });
+        case 12:
+        case "end":
+          return _context4.stop();
+      }
+    }, _callee3, null, [[0, 8]]);
+  }));
+  return function deleteLogs(_x4, _x5, _x6, _x7) {
+    return _ref3.apply(this, arguments);
+  };
+}();
+var setMaxRecords = exports.setMaxRecords = function setMaxRecords(self, callId, max) {
+  maxRecords = max;
+  (0, _reply.reply)(self, callId, _constants.WorkerDirectives.SET_MAX_RECORDS, {
+    isSuccess: true
+  });
+};

package/lib/worker/handler/reply.d.ts

@@ -0,0 +1,3 @@
+import { WorkerDirectives } from '../constants';
+import { WorkerContext } from '../type';
+export declare const reply: (self: WorkerContext, callId: number, type: WorkerDirectives, data: unknown, transfer?: Transferable[]) => void;

package/lib/worker/handler/reply.js

@@ -0,0 +1,22 @@
+"use strict";
+
+require("core-js/modules/es.object.define-property.js");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.reply = void 0;
+var reply = exports.reply = function reply(self, callId, type, data, transfer) {
+  if (transfer) {
+    self.postMessage({
+      type: type,
+      callId: callId,
+      data: data
+    }, transfer);
+  } else {
+    self.postMessage({
+      type: type,
+      callId: callId,
+      data: data
+    });
+  }
+};

package/lib/worker/index.d.ts

@@ -0,0 +1,2 @@
+import { WorkerInteractor } from './interactor';
+export declare const getWorkerSingleton: () => WorkerInteractor;

package/lib/worker/index.js

@@ -0,0 +1,15 @@
+"use strict";
+
+require("core-js/modules/es.object.define-property.js");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getWorkerSingleton = void 0;
+var _interactor = require("./interactor");
+var singleton;
+var getWorkerSingleton = exports.getWorkerSingleton = function getWorkerSingleton() {
+  if (!singleton) {
+    singleton = new _interactor.WorkerInteractor();
+  }
+  return singleton;
+};

package/lib/worker/interactor.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+import { LogWorkerInteractor } from '../logger/type';
+export declare class WorkerInteractor implements LogWorkerInteractor {
+    private _eventEmitter;
+    private _logWorker;
+    constructor();
+    clearTempLogs(): Promise<void>;
+    setMaxRecords(maxRecords: number): Promise<void>;
+    sendLog(message: string, label: string): void;
+    collectLogs(labels: string[]): Promise<{
+        file: File;
+        lastId: number;
+    }>;
+    deleteLogs(labels: string[], lastId: number): Promise<void>;
+    persistBinary(id: number, buffer: Buffer): Promise<void>;
+    unpersistBinary(id: number): Promise<void>;
+    getBinary(id: number): Promise<Buffer | null>;
+    collectLogsOnFs(logDirPath: string): Promise<File>;
+    private _handleMessageCallback;
+    private _handleErrorCallback;
+    private _handleMessageErrorCallback;
+    private _attachEventHandlers;
+    private _spawnNewWorker;
+    private _promisify;
+}