@miso.ai/server-commons 0.5.4-beta.3 → 0.5.4-beta.5

package/package.json

@@ -16,5 +16,5 @@
     "js-yaml": "^4.1.0",
     "toml": "^3.0.0"
   },
-  "version": "0.5.4-beta.3"
+  "version": "0.5.4-beta.5"
 }

package/src/async.js

@@ -0,0 +1,4 @@
+export async function delay(duration) {
+  // TODO: can accept abort singal
+  return new Promise((resolve) => setTimeout(resolve, duration));
+}

package/src/index.js

@@ -1,6 +1,12 @@
 export * from './object.js';
-export * as stream from './stream.js';
+export * from './string.js';
 export * from './date.js';
 export * from './file.js';
 export * from './config.js';
+export * from './async.js';
+
+export * as stream from './stream/index.js';
+export * as log from './log.js';
+
 export { default as TaskQueue } from './task-queue.js';
+export { default as Resolution } from './resolution.js';

package/src/log.js

@@ -0,0 +1,96 @@
+import { padLeft, padRight } from './string.js';
+
+export const FATAL = 'fatal';
+export const ERROR = 'error';
+export const WARNING = 'warning';
+export const INFO = 'info';
+export const DEBUG = 'debug';
+
+const LEVEL_VALUES = {
+  [FATAL]: 2,
+  [ERROR]: 4,
+  [WARNING]: 6,
+  [INFO]: 8,
+  [DEBUG]: 50,
+};
+const FALLBACK_LEVEL_VALUE = 40;
+
+export const LEVELS = Object.keys(LEVEL_VALUES);
+
+LEVELS.sort(compareLevel);
+
+export function reachesThreshold(level, threshold) {
+  if (!LEVEL_VALUES[threshold]) {
+    throw new Error(`Unrecognized log level: ${threshold}`);
+  }
+  if (!level) {
+    throw new Error(`Level is absent: ${level}`);
+  }
+  return compareLevel(level, threshold) <= 0;
+}
+
+export function isError(level) {
+  return reachesThreshold(level, ERROR);
+}
+
+function compareLevel(a, b) {
+  return (LEVEL_VALUES[a] || FALLBACK_LEVEL_VALUE) - (LEVEL_VALUES[b] || FALLBACK_LEVEL_VALUE);
+}
+
+export function formatDuration(value) {
+  if (typeof value !== 'number') {
+    throw new Error(`Value must be a number: ${value}`);
+  }
+  value = Math.floor(value);
+  const ms = padLeft(`${value % 60000}`, 5, '0');
+  let min = '00';
+  let hour = '00';
+  if (value >= 60000) {
+    value = Math.floor(value / 60000);
+    min = padLeft(`${value % 60}`, 2, '0');
+    if (value >= 60) {
+      hour = padLeft(Math.floor(value / 60), 2, '0');
+    }
+  }
+  return `${hour}:${min}:${ms.substring(0, 2)}.${ms.substring(2, 5)}`;
+}
+
+export function formatBytes(value) {
+  let i = 0;
+  for (; i < 4; i++) {
+    if (value < 1024) {
+      break;
+    }
+    value /= 1024;
+  }
+  return `${Math.floor(value * 100) / 100} ${UNITS[i]}`;
+}
+
+const UNITS = ['B', 'KB', 'MB', 'GB', 'TB'];
+
+export function formatTable(rows, { columnPadding: columnSpacing = 3 } = {}) {
+  const maxLens = [];
+  for (const cells of rows) {
+    for (let j = 0, len = cells.length; j < len; j++) {
+      const len = `${cells[j]}`.length;
+      if (!maxLens[j] || maxLens[j] < len) {
+        maxLens[j] = len;
+      }
+    }
+  }
+  const colSpc = ' '.repeat(columnSpacing);
+  let str = '';
+  for (let i = 0, len = rows.length; i < len; i++) {
+    if (i > 0) {
+      str += '\n';
+    }
+    const cells = rows[i];
+    for (let j = 0, len = cells.length; j < len; j++) {
+      if (j > 0) {
+        str += colSpc;
+      }
+      str += padRight(`${cells[j]}`, maxLens[j], ' ');
+    }
+  }
+  return str;
+}

package/src/resolution.js

@@ -0,0 +1,12 @@
+export default class Resolution {
+
+  constructor() {
+    const self = this;
+    self.promise = new Promise((resolve, reject) => {
+      self.resolve = resolve;
+      self.reject = reject;
+    });
+    Object.freeze(this);
+  }
+
+}

package/src/stream/buffered-read.js

@@ -0,0 +1,299 @@
+import { Readable } from 'stream';
+import Denque from 'denque';
+import { trimObj } from '../object.js';
+import TaskQueue from '../task-queue.js';
+import Resolution from '../resolution.js';
+
+export default class BufferedReadStream extends Readable {
+
+  constructor(source, { strategy, filter, transform, onLoad, debug } = {}) {
+    super({ objectMode: true });
+    this._debug = debug || (() => {});
+    this._source = source;
+    this._strategy = new Strategy(strategy);
+    this._state = new State();
+    this._loads = new TaskQueue();
+    this._buckets = new Denque();
+
+    this._filter = filter || (() => true);
+    this._transform = transform || (v => v);
+    this._onLoad = onLoad || (() => {});
+    this._index = 0;
+
+    this._debug(`[BufferedReadStream] strategy: ${this._strategy}`);
+    this._strategy.initialize(this, source);
+  }
+
+  async _construct() {
+    if (this._source.init) {
+      this._debug(`[BufferedReadStream] init source start`);
+      await this._source.init();
+      this._debug(`[BufferedReadStream] init source done`);
+    }
+  }
+
+  async _read() {
+    const record = await this._next();
+    this.push(record != null ? record : null);
+  }
+
+  async _next() {
+   // TODO: put in action queue to support parallel call
+   const bucket = await this._peekBucket();
+    if (!bucket) {
+      return undefined;
+    }
+    const record = bucket[this._index++];
+    this._state.serve();
+    if (this._index >= bucket.length) {
+      this._buckets.shift();
+      this._index = 0;
+    }
+    return record;
+  }
+
+  async peek() {
+   // TODO: put in action queue to support parallel call
+   const bucket = await this._peekBucket();
+    return bucket && bucket[this._index];
+  }
+
+  async _peekBucket() {
+    if (this._buckets.isEmpty()) {
+      if (this._state.terminated) {
+        return undefined;
+      }
+      this._loadIfNecessary();
+      await this._waitForData();
+    }
+    return this._buckets.isEmpty() ? undefined : this._buckets.peekFront();
+  }
+
+  _loadIfNecessaryNextTick() {
+    process.nextTick(() => this._loadIfNecessary());
+  }
+
+  _loadIfNecessary() {
+    if (this._shallLoad()) {
+      this._load();
+      this._loadIfNecessaryNextTick(true);
+    }
+  }
+
+  async _load() {
+    const request = this._state.request(this._source.request());
+
+    this._debug(`[BufferedReadStream] Load request: ${request}`);
+    const { data, ...info } = await this._source.get(request);
+    const response = new Response(request, info);
+    this._debug(`[BufferedReadStream] Load response: ${JSON.stringify(response)} => data = ${data && data.length}`);
+
+    // TODO: support strategy option: keepOrder = false
+    this._loads.push(request.index, () => this._resolveLoad(response, data));
+  }
+
+  _resolveLoad(response, records) {
+    const state = this._state;
+    const strategy = this._strategy;
+
+    state.resolve(response);
+    this._debug(`[BufferedReadStream] Load resolved: ${response}`);
+
+    // apply terminate and filter function
+    let terminate = false;
+    const accepted = [];
+    for (const record of records) {
+      terminate = terminate || strategy.terminate(record, state);
+      if (!terminate && this._filter(record)) {
+        state.accept();
+        accepted.push(this._transform(record));
+      }
+      if (terminate) {
+        break;
+      }
+    }
+    if (terminate || (this._state.pendingLoads === 0 && this._state.exhausted)) {
+      state.terminate();
+    }
+
+    if (accepted.length > 0) {
+      this._buckets.push(accepted);
+      this._onLoad(accepted);
+    } else {
+      this._loadIfNecessaryNextTick(); // just in case
+    }
+
+    if (accepted.length > 0 || state.terminated) {
+      this._resolveDataPromise();
+    }
+  }
+
+  _waitForData() {
+    if (this._state.pendingLoads === 0) {
+      throw new Error(`No pending loads.`);
+    }
+    if (this._dataRes) {
+      throw new Error(`Parallel bucket peek.`);
+    }
+    return (this._dataRes = new Resolution()).promise;
+  }
+
+  _resolveDataPromise() {
+    if (this._dataRes) {
+      this._dataRes.resolve();
+      this._dataRes = undefined;
+    }
+  }
+
+  _shallLoad() {
+    const state = this._state;
+    if (state.exhausted) {
+      return false;
+    }
+    return !state.exhausted && this._strategy.shallLoad(state);
+  }
+
+}
+
+class Request {
+
+  constructor(info) {
+    this.timestamp = Date.now();
+    Object.assign(this, info);
+    Object.freeze(this);
+  }
+
+  toString() {
+    return JSON.stringify(this);
+  }
+
+}
+
+class Response {
+
+  constructor({ timestamp, ...request }, info) {
+    const now = this.timestamp = Date.now();
+    this.took = now - timestamp;
+    Object.assign(this, request);
+    Object.assign(this, info);
+    Object.freeze(this);
+  }
+
+  toString() {
+    return JSON.stringify(this);
+  }
+
+}
+
+class Strategy {
+
+  constructor({
+    highWatermark = 1000,
+    eagerLoad = false,
+    initialize,
+    shallLoad,
+    terminate,
+  } = {}) {
+    this.options = Object.freeze({ highWatermark, eagerLoad });
+    // overwrite methods
+    Object.assign(this, trimObj({ initialize, shallLoad, terminate }));
+  }
+
+  initialize(stream) {
+    if (this.options.eagerLoad) {
+      stream._loadIfNecessaryNextTick();
+    }
+  }
+
+  shallLoad(state) {
+    // TODO: we can have a slower start
+    return state.watermark < this.options.highWatermark;
+  }
+
+  terminate(record, state) {
+    return false;
+  }
+
+  toString() {
+    return JSON.stringify(this);
+  }
+
+}
+
+class State {
+
+  constructor() {
+    this.records = {
+      requested: 0,
+      resolved: 0,
+      accepted: 0,
+      served: 0,
+    };
+    this.loads = {
+      requested: 0,
+      resolved: 0,
+    };
+    this.took = 0;
+    this.exhausted = false;
+    this.terminated = false;
+  }
+
+  request({ records, exhaust, ...data }) {
+    // note that we may not know how many records will come form this request, so record sum may become NaN
+    this.records.requested += records;
+    const index = this.loads.requested++;
+    if (exhaust) {
+      this.exhaust();
+    }
+    return new Request({ ...data, index, records, exhaust });
+  }
+
+  resolve({ records, took, terminate }) {
+    // note that we may not know how many records will come form this request, so record sum may become NaN
+    this.records.resolved += records;
+    this.loads.resolved++;
+    this.took += took;
+    if (terminate) {
+      this.terminate();
+    }
+  }
+
+  accept() {
+    this.records.accepted++;;
+  }
+
+  serve() {
+    this.records.served++;
+  }
+
+  exhaust() {
+    this.exhausted = true;
+  }
+
+  terminate() {
+    this.exhaust();
+    this.terminated = true;
+  }
+
+  get pendingLoads() {
+    const { loads: loads } = this;
+    return loads.requested - loads.resolved;
+  }
+
+  get pendingRecords() {
+    const { records } = this;
+    return records.requested - records.resolved;
+  }
+
+  get unservedRecords() {
+    const { records } = this;
+    return records.accepted - records.served;
+  }
+
+  get watermark() {
+    let { pendingRecords } = this;
+    // TODO: better estimated pendingRecords when NaN
+    return (isNaN(pendingRecords) ? 0 : pendingRecords) + this.unservedRecords;
+  }
+
+}

package/src/stream/index.js

@@ -0,0 +1,3 @@
+export * from './misc.js';
+export { default as BufferedReadStream } from './buffered-read.js';
+export { default as OutputStream } from './output.js';

package/src/{stream.js → stream/misc.js}

@@ -12,12 +12,22 @@ export function parse() {
 export function stringify() {
   return new Transform({
     transform(chunk, _, callback) {
-      callback(null, JSON.stringify(chunk) + '\n');
+      try {
+        callback(null, JSON.stringify(chunk) + '\n');
+      } catch(e) {
+        callback(null, `${chunk}` + '\n');
+      }
     },
     writableObjectMode: true,
   });
 }
 
+export async function pipeline(...streams) {
+  return new Promise((resolve, reject) =>
+    _pipeline(...streams, err => err ? reject(err) : resolve())
+  );
+}
+
 export async function pipelineToStdout(...streams) {
   return new Promise((resolve, reject) => 
     _pipeline(...streams, err => err ? reject(err) : resolve())

package/src/stream/output.js

@@ -0,0 +1,24 @@
+import { Writable } from 'stream';
+
+export default class OutputStream extends Writable {
+
+  constructor({
+    out = process.stdout,
+    err = process.stderr,
+    format,
+    objectMode = true,
+  } = {}) {
+    super({
+      objectMode,
+    });
+    this._format = format || (v => `${v}`);
+    this._out = out;
+    this._err = err;
+  }
+
+  _write(record, _, next) {
+    this._out.write(this._format(record) + '\n');
+    next();
+  }
+
+}

package/src/string.js

@@ -0,0 +1,32 @@
+// From:
+// https://github.com/jonschlinkert/pad-right
+
+const DEFAULT_PAD = ' ';
+const DEFAULT_PAD_5 = DEFAULT_PAD.repeat(5);
+const DEFAULT_PAD_25 = DEFAULT_PAD.repeat(25);
+
+function pad(left, val, num, str) {
+  const diff = num - val.length;
+  if (diff <= 0) {
+    return val;
+  }
+  const padding = buildPadding(diff, str);
+  return left ? padding + val : val + padding;
+}
+
+function buildPadding(count, str) {
+  // Breakpoints based on benchmarks to use the fastest approach
+  // for the given number of zeros
+  return str ? str.repeat(count) :
+    count <= 5 ? DEFAULT_PAD_5.slice(0, count) :
+    count <= 25 ? DEFAULT_PAD_25.slice(0, count) :
+    DEFAULT_PAD.repeat(count);
+}
+
+export function padLeft(val, num, str) {
+  return pad(true, val, num, str);
+}
+
+export function padRight(val, num, str) {
+  return pad(false, val, num, str);
+}