@miso.ai/server-commons 0.5.4-beta.5 → 0.6.0-beta.0

package/package.json

@@ -13,8 +13,11 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
+    "denque": "^2.1.0",
     "js-yaml": "^4.1.0",
-    "toml": "^3.0.0"
+    "log-update": "^5.0.1",
+    "toml": "^3.0.0",
+    "uuid": "^9.0.0"
   },
-  "version": "0.5.4-beta.5"
+  "version": "0.6.0-beta.0"
 }

package/src/buffer/base.js

@@ -0,0 +1,67 @@
+export default class BaseBuffer {
+
+  constructor(type, { recordsLimit, bytesLimit }) {
+    this._options = {
+      type,
+      recordsLimit,
+      bytesLimit,
+    };
+  }
+
+  _reset() {
+    this._records = this._bytes = 0;
+  }
+
+  get config() {
+    return Object.freeze({ ...this._options });
+  }
+
+  get limit() {
+    return this._recordsPerRequest;
+  }
+
+  get empty() {
+    return this.records === 0;
+  }
+
+  get full() {
+    const { recordsLimit, bytesLimit } = this._options;
+    return (recordsLimit !== undefined && this.records >= recordsLimit) || 
+      (bytesLimit !== undefined && this.bytes >= bytesLimit);
+  }
+
+  get records() {
+    return this._records;
+  }
+
+  get bytes() {
+    return this._bytes;
+  }
+
+  get payload() {
+    return this._payload;
+  }
+
+  get state() {
+    const { records, bytes, payload } = this;
+    return Object.freeze({ records, bytes, payload });
+  }
+
+  push(record) {
+    const dispatches = this._push(record);
+    return this.full ? [...dispatches, ...this.flush()] : dispatches;
+  }
+
+  _push(record) {
+    throw new Error(`Unimplemented.`);
+  }
+
+  flush() {
+    const { state } = this;
+    this._reset();
+    return state.records > 0 ? [state] : [];
+  }
+
+  destroy() {}
+
+}

package/src/buffer/index.js

@@ -0,0 +1,2 @@
+export { default as BaseBuffer } from './base.js';
+export { default as JsonBuffer } from './json.js';

package/src/buffer/json.js

@@ -0,0 +1,66 @@
+import BaseBuffer from './base.js';
+
+function buildTransformFunction(transform, objectMode) {
+  // TODO: review this: v could be a buffer when objectMode = false
+  return objectMode ? (
+    transform ? v => JSON.stringify(transform(v)) : v => JSON.stringify(v)
+  ) : (
+    transform ? transform : v => v
+  );
+}
+
+export default class JsonBuffer extends BaseBuffer {
+
+  constructor({
+    prefix = '[',
+    suffix = ']',
+    separator = ',',
+    transform,
+    objectMode,
+    recordsLimit,
+    bytesLimit,
+  }) {
+    super('json', { recordsLimit, bytesLimit });
+    this._format = { prefix, suffix, separator };
+    this._transform = buildTransformFunction(transform, objectMode);
+    this._affixSize = (prefix.length + suffix.length) * 2;
+    this._separatorSize = separator.length * 2;
+    this._reset();
+  }
+
+  _reset() {
+    this._records = 0;
+    this._bytes = this._affixSize;
+    this._payload = this._format.prefix;
+  }
+
+  get payload() {
+    return this._payload + this._format.suffix;
+  }
+
+  _push(record) {
+    let dispatches = [];
+    const payload = this._transform(record);
+    const { empty } = this;
+    const cententSize = payload.length * 2;
+
+    if (!empty) {
+      const separatorSize = this._separatorSize;
+      if (this._bytes + separatorSize + cententSize > this._options.bytesLimit) {
+        // if buffer is not empty and exceeds bytes limit after pushing, then flush right now
+        dispatches = this.flush();
+      } else {
+        // non-empty, not flushed
+        this._bytes += separatorSize;
+        this._payload += this._format.separator;
+      }
+    }
+
+    this._records++;
+    this._bytes += cententSize;
+    this._payload += payload;
+
+    return dispatches;
+  }
+
+}

package/src/index.js

@@ -5,8 +5,11 @@ export * from './file.js';
 export * from './config.js';
 export * from './async.js';
 
+export * as sink from './sink/index.js';
+export * as buffer from './buffer/index.js';
 export * as stream from './stream/index.js';
-export * as log from './log.js';
+export * as log from './log/index.js';
 
-export { default as TaskQueue } from './task-queue.js';
 export { default as Resolution } from './resolution.js';
+export { default as TaskQueue } from './task-queue.js';
+export { default as RateLimitingQueue } from './rate-limiting-queue.js';

package/src/log/index.js

@@ -0,0 +1 @@
+export * from './utils.js';

package/src/{log.js → log/utils.js}

@@ -1,4 +1,4 @@
-import { padLeft, padRight } from './string.js';
+import { padLeft, padRight } from '../string.js';
 
 export const FATAL = 'fatal';
 export const ERROR = 'error';
@@ -38,7 +38,7 @@ function compareLevel(a, b) {
 }
 
 export function formatDuration(value) {
-  if (typeof value !== 'number') {
+  if (isNaN(value)) {
     throw new Error(`Value must be a number: ${value}`);
   }
   value = Math.floor(value);

package/src/object.js

@@ -62,3 +62,7 @@ export function copyValues(target, source, propNames) {
   }
   return target;
 }
+
+export function unimplemented() {
+  throw new Error(`Unimplemented!`);
+}

package/src/rate-limiting-queue.js

@@ -0,0 +1,72 @@
+import Denque from 'denque';
+import Resolution from './resolution.js';
+
+export default class RateLimitingQueue {
+
+  constructor({
+    actionsPerSecond,
+  } = {}) {
+    if (actionsPerSecond !== undefined && (isNaN(actionsPerSecond) || actionsPerSecond <= 0)) {
+      throw new Error(`Invalid value for actionsPerSecond: ${actionsPerSecond}`);
+    }
+    this._options = {
+      actionsPerSecond,
+    };
+    this._interval = actionsPerSecond ? 1000 / actionsPerSecond : 0;
+    this._queue = new Denque();
+    this._running = this._busy = false;
+  }
+
+  push(fn) {
+    const res = new Resolution();
+    this._queue.push([fn, res]);
+    this.start();
+    return res.promise;
+  }
+
+  get running() {
+    return this._running;
+  }
+
+  get busy() {
+    return this._busy;
+  }
+
+  start() {
+    this._running = true;
+    if (!this._busy) {
+      this._next();
+    }
+  }
+
+  pause() {
+    this._running = false;
+  }
+
+  _next() {
+    const queue = this._queue;
+    if (queue.isEmpty()) {
+      this._running = false;
+    }
+    if (!this._running || this._busy) {
+      return;
+    }
+    this._busy = true;
+    const [fn, res] = queue.shift();
+    this._exec(fn, res);
+
+    setTimeout(() => {
+      this._busy = false;
+      this._next();
+    }, this._interval);
+  }
+
+  async _exec(fn, res) {
+    try {
+      res.resolve(await fn());
+    } catch(error) {
+      res.reject(error);
+    }
+  }
+
+}

package/src/resolution.js

@@ -9,4 +9,12 @@ export default class Resolution {
     Object.freeze(this);
   }
 
+  async execute(fn) {
+    try {
+      this.resolve(await fn());
+    } catch(error) {
+      this.reject(error);
+    }
+  }
+
 }

package/src/sink/bps.js

@@ -0,0 +1,89 @@
+import { trimObj } from '../object.js';
+
+export default class BpsSink {
+
+  constructor(options) {
+    this._options = this._normalizeOptions(options);
+    this._stats = {
+      started: { count: 0, records: 0, bytes: 0 },
+      finished: { count: 0, records: 0, bytes: 0 },
+    };
+  }
+
+  _normalizeOptions({
+    recordsPerSecord = 100000,
+    bytesPerSecond = 4 * 1024 * 1024,
+    ...options
+  } = {}) {
+    return {
+      recordsPerSecord,
+      bytesPerSecond,
+      ...options,
+    };
+  }
+
+  get config() {
+    return Object.freeze({
+      ...this._options,
+    });
+  }
+  
+  get stats() {
+    return Object.freeze({ ...this._stats });
+  }
+
+  async write(payload, { records, bytes } = {}) {
+    const { started, finished } = this._stats;
+    if (started.count === 0) {
+      this._firstWriteAt = Date.now();
+    }
+    started.count++;
+    started.records += records;
+    started.bytes += bytes;
+
+    let response;
+    try {
+      response = await this._write(payload, { records, bytes });
+    } catch(error) {
+      response = trimObj({ errors: true, cause: error.message });
+    }
+    finished.count++;
+    finished.records += records;
+    finished.bytes += bytes;
+
+    return trimObj({
+      ...response,
+      timestamp: Date.now(),
+    });
+  }
+
+  blockedTime() {
+    const now = Date.now();
+    const elapsed = now - this._firstWriteAt;
+    const targetBps = this._targetBps(now);
+    const targetRps = this._targetRps(now);
+
+    const { started } = this._stats;
+    const shallElapsed = Math.max(started.records / targetRps, started.bytes / targetBps) * 1000;
+
+    const blockedTime = shallElapsed - elapsed;
+    if (blockedTime <= 1000) {
+      return 0;
+    }
+    // TODO: review this
+    return Math.ceil(blockedTime);
+  }
+
+  async _write(payload) {
+    throw new Error(`Unimplemented.`);
+  }
+
+  _targetBps(timestamp) {
+    return this._options.bytesPerSecond;
+  }
+
+  _targetRps(timestamp) {
+    return this._options.recordsPerSecord;
+  }
+
+}

package/src/sink/index.js

@@ -0,0 +1 @@
+export { default as BpsSink } from './bps.js';

package/src/stream/buffered-read.js

@@ -187,6 +187,8 @@ class Response {
 
 class Strategy {
 
+  // TODO: introduce RPS
+
   constructor({
     highWatermark = 1000,
     eagerLoad = false,

package/src/stream/buffered-write-state.js

@@ -0,0 +1,257 @@
+import { trimObj } from '../object.js';
+import Resolution from '../resolution.js';
+
+const STATUS = Object.freeze({
+  WAITING: 'waiting',
+  RUNNING: 'running',
+  PAUSED: 'paused',
+  FINISHED: 'finished',
+});
+
+export default class State {
+
+  static get STATUS() {
+    return STATUS;
+  }
+
+  constructor() {
+    this._status = 'waiting';
+    this._resolutions = new WeakMap();
+    this._time = new Time(Date.now());
+
+    this._next = Object.freeze({
+      request: 0,
+      record: 0,
+    });
+    this._pending = [];
+    this._successful = {
+      requests: 0,
+      records: 0,
+      bytes: 0,
+    };
+    this._failed = {
+      requests: 0,
+      records: 0,
+      bytes: 0,
+    };
+  }
+
+  // status //
+  get status() {
+    return this._status;
+  }
+
+  pause(pausedAt, pauseTime) {
+    if (this._status === STATUS.PAUSED) {
+      return;
+    }
+    this._time.pause(pausedAt, pauseTime);
+    this._status = STATUS.PAUSED;
+  }
+
+  resume(resumeAt) {
+    if (this._status !== STATUS.PAUSED) {
+      return;
+    }
+    this._time.resume(resumeAt);
+    this._status = this._next.request === 0 ? STATUS.WAITING : STATUS.RUNNING;
+  }
+
+  async finish() {
+    await Promise.all(this._pending.map(r => this._resolutions.get(r).promise));
+    this._status = STATUS.FINISHED;
+  }
+
+  // data stats //
+  get next() {
+    return this._next;
+  }
+
+  get pending() {
+    return Object.freeze(this._pending.slice());
+  }
+
+  get successful() {
+    return Object.freeze({ ...this._successful });
+  }
+
+  get failed() {
+    return Object.freeze({ ...this._failed });
+  }
+
+  get time() {
+    return this._time;
+  }
+
+  // speed management //
+  bps(timestamp) {
+    const elapsed = this._time.elapsed(timestamp || Date.now());
+    return elapsed > 0 ? this.sent.bytes / elapsed * 1000 : NaN;
+  }
+
+  // operation //
+  request({ records, bytes }) {
+    const timestamp = Date.now();
+    const { next } = this;
+    if (next.request === 0) {
+      this._time.setFirstWrite(timestamp);
+    }
+    const request = Object.freeze({
+      index: next.request,
+      recordOffset: next.record,
+      records,
+      bytes,
+      timestamp,
+    });
+    this._resolutions.set(request, new Resolution());
+    this._next = Object.freeze({
+      request: next.request + 1,
+      record: next.record + records,
+    });
+    this._pending.push(request);
+    return request;
+  }
+
+  resolve(request, response) {
+    this._pending = this._pending.filter(r => r.index !== request.index);
+
+    const category = response.errors ? this._failed : this._successful;
+    category.requests++;
+    category.records += request.records;
+    category.bytes += request.bytes;
+
+    this._time.addWrite(response.timestamp - request.timestamp);
+
+    this._resolutions.get(request).resolve();
+  }
+
+  // summary //
+  get sent() {
+    const { _pending, completed } = this;
+    return Object.freeze(_pending.reduce((acc, request) => {
+      acc.requests++;
+      acc.records += request.records;
+      acc.bytes += request.bytes;
+      return acc;
+    }, {
+      requests: completed.requests,
+      records: completed.records,
+      bytes: completed.bytes,
+    }));
+  }
+
+  get completed() {
+    const { _successful, _failed } = this;
+    return Object.freeze({
+      requests: _successful.requests + _failed.requests,
+      records: _successful.records + _failed.records,
+      bytes: _successful.bytes + _failed.bytes,
+    });
+  }
+
+  snapshot(timestamp) {
+    timestamp = timestamp || Date.now();
+    const { status, next, pending, successful, failed, sent, completed } = this;
+    return Object.freeze({
+      status, next, pending, sent, successful, failed, completed,
+      time: this._time.snapshot(timestamp),
+      bps: this.bps(timestamp),
+    });
+  }
+
+}
+
+class Time {
+
+  constructor(timestamp) {
+    Object.defineProperty(this, 'constructedAt', { value: timestamp });
+    this._firstWriteAt = undefined;
+    this._write = 0;
+    this._pausedAt = undefined;
+    this._willResumeAt = undefined;
+    this._paused = 0;
+  }
+
+  // operation //
+  setFirstWrite(timestamp) {
+    if (this._firstWriteAt !== undefined) {
+      throw new Error(`First write time already set: ${this._firstWriteAt}`);
+    }
+    this._firstWriteAt = timestamp;
+  }
+
+  addWrite(duration) {
+    this._write += duration;
+  }
+
+  pause(pausedAt, pauseTime) {
+    if (this._firstWriteAt === undefined || this._pausedAt !== undefined) {
+      throw new Error(`Cannot pause() before first write or when paused.`);
+    }
+    this._pausedAt = pausedAt;
+    this._willResumeAt = pausedAt + pauseTime;
+  }
+
+  resume(resumeAt) {
+    if (this._firstWriteAt === undefined || this._pausedAt === undefined) {
+      throw new Error(`Cannot resume() before first write or when not paused.`);
+    }
+    this._paused += resumeAt - this._pausedAt;
+    this._pausedAt = this._willResumeAt = undefined;
+  }
+
+  // stats //
+  get firstWriteAt() {
+    return this._firstWriteAt;
+  }
+
+  get pausedAt() {
+    return this._pausedAt;
+  }
+
+  get willResumeAt() {
+    return this._willResumeAt;
+  }
+
+  get write() {
+    return this._write;
+  }
+
+  total(timestamp) {
+    return timestamp - this.constructedAt;
+  }
+
+  waiting(timestamp) {
+    return (this._firstWriteAt !== undefined ? this._firstWriteAt : timestamp) - this.constructedAt;
+  }
+
+  elapsed(timestamp) {
+    return this._firstWriteAt === undefined ? 0 : timestamp - this._firstWriteAt;
+  }
+
+  paused(timestamp) {
+    return this._paused + (this._pausedAt !== undefined ? timestamp - this._pausedAt : 0);
+  }
+
+  running(timestamp) {
+    return this.elapsed(timestamp) - this.paused(timestamp);
+  }
+
+  snapshot(timestamp) {
+    const { constructedAt, firstWriteAt, pausedAt, willResumeAt, write } = this;
+    return Object.freeze(trimObj({
+      currentTime: timestamp,
+      constructedAt,
+      firstWriteAt,
+      pausedAt,
+      willResumeAt,
+      write,
+      total: this.total(timestamp),
+      waiting: this.waiting(timestamp),
+      elapsed: this.elapsed(timestamp),
+      paused: this.paused(timestamp),
+      running: this.running(timestamp),
+    }));
+  }
+
+}

package/src/stream/buffered-write.js

@@ -0,0 +1,194 @@
+import { Transform } from 'stream';
+import { v4 as uuid } from 'uuid';
+import { trimObj } from '../object.js';
+import { delay } from '../async.js';
+import * as log from '../log/index.js';
+import State from './buffered-write-state.js';
+
+const MIN_HREATBEAT_INTERVAL = 100;
+
+export default class BufferedWriteStream extends Transform {
+
+  static DEFAULTS = {
+    // merge everything except for payload
+    output: (message, { payload: _, ...args } = {}) => ({ ...message, ...args }),
+  };
+
+  constructor({
+    objectMode,
+    sink,
+    buffer,
+    output,
+    heartbeatInterval,
+    ...options
+  } = {}) {
+    super({
+      readableObjectMode: true,
+      writableObjectMode: objectMode,
+    });
+    Object.defineProperty(this, 'id', { value: uuid() });
+
+    this._sink = sink;
+    this._buffer = buffer;
+
+    this._fns = {
+      output,
+    };
+
+    this._options = this._normalizeOptions({
+      objectMode,
+      heartbeatInterval,
+      ...options,
+    });
+
+    this._state = new State();
+
+    // log functions
+    for (const level of log.LEVELS) {
+      this[`_${level}`] = this._log.bind(this, level);
+    }
+  }
+
+  _normalizeOptions({
+    objectMode,
+    heartbeatInterval = false,
+    ...options
+  } = {}) {
+    if (heartbeatInterval !== false && (isNaN(heartbeatInterval) || heartbeatInterval < MIN_HREATBEAT_INTERVAL)) {
+      throw new Error(`Heartbeat interval must be a number in milliseconds at least ${MIN_HREATBEAT_INTERVAL}: ${heartbeatInterval}`);
+    }
+    return trimObj({
+      objectMode: !!objectMode,
+      heartbeatInterval,
+      ...options,
+    });
+  }
+
+  _construct(done) {
+    const heartbeatInterval = this._options.heartbeatInterval;
+    if (heartbeatInterval) {
+      this._heartbeatIntervalId = setInterval(this._heartbeat.bind(this), heartbeatInterval);
+    }
+
+    const { config } = this;
+    this._info('construct', { config });
+
+    done();
+  }
+
+  async _transform(record, _) {
+    this._pushStartEventIfNecessary();
+    this._dispatchAll(this._buffer.push(record));
+    await this._pauseIfNecessary();
+  }
+
+  async _flush(done) {
+    this._dispatchAll(this._buffer.flush());
+    await this._state.finish();
+    this._buffer.destroy();
+
+    // stop heartbeat before end event
+    if (this._heartbeatIntervalId) {
+      clearInterval(this._heartbeatIntervalId);
+      delete this._heartbeatIntervalId;
+    }
+
+    // in case of empty stream, we still want a start event
+    this._pushStartEventIfNecessary();
+
+    const { successful, failed } = this._state;
+    this._info('end', { successful, failed });
+
+    done();
+  }
+
+  get state() {
+    return this._state.snapshot();
+  }
+
+  get config() {
+    return Object.freeze(this._exportConfig());
+  }
+
+  // can be overwritten //
+  _exportConfig() {
+    return {
+      id: this.id.substring(0, 8),
+      sink: this._sink.config,
+      buffer: this._buffer.config,
+      ...this._options,
+    };
+  }
+
+  _output(message, args) {
+    return (this._fns.output || BufferedWriteStream.DEFAULTS.output)(message, args);
+  }
+
+  // helper //
+  _heartbeat() {
+    this._log(log.DEBUG, 'heartbeat');
+  }
+
+  _log(level, event, args) {
+    const message = trimObj({
+      level,
+      event,
+      timestamp: Date.now(),
+      state: this.state,
+    });
+    this.push(this._output(message, args));
+  }
+
+  _pushStartEventIfNecessary() {
+    if (this._state.next.request === 0 && this._buffer.records === 0) {
+      this._info('start');
+    }
+  }
+
+  async _pauseIfNecessary() {
+    const pauseTime = this._sink.blockedTime();
+    if (pauseTime > 0) {
+      const pausedAt = Date.now();
+      this._state.pause(pausedAt, pauseTime);
+      this._debug('pause', { pausedAt, pauseTime });
+      await delay(pauseTime);
+      const resumeAt = Date.now();
+      this._state.resume(resumeAt);
+      this._debug('resume', { resumeAt });
+    } else if (this._state._pending.length > 15) {
+      // TODO: figure out best strategy on this
+      // release event loop for downstream
+      await delay();
+    }
+  }
+
+  _dispatchAll(dispatches) {
+    for (const dispatch of dispatches) {
+      this._dispatch(dispatch); // don't wait
+    }
+  }
+
+  async _dispatch({ records, bytes, payload }) {
+    const request = this._state.request({ records, bytes });
+    this._debug('request', { request, payload });
+
+    const response = await this._writeToSink(payload, request);
+
+    this._state.resolve(request, response);
+    const failed = response.errors;
+    (failed ? this._error : this._debug)('response', { request, response, payload });
+
+    this._info('write', {
+      result: failed ? 'failed' : 'successful',
+      index: request.index,
+      records: request.records,
+      bytes: request.bytes,
+      time: response.timestamp - request.timestamp,
+    });
+  }
+
+  async _writeToSink(payload, request) {
+    return this._sink.write(payload, request);
+  }
+
+}

package/src/stream/index.js

@@ -1,3 +1,5 @@
 export * from './misc.js';
 export { default as BufferedReadStream } from './buffered-read.js';
+export { default as BufferedWriteStream } from './buffered-write.js';
 export { default as OutputStream } from './output.js';
+export { default as LogUpdateStream } from './log-update.js';

package/src/stream/log-update.js

@@ -0,0 +1,47 @@
+import { Writable } from 'stream';
+import { createLogUpdate } from 'log-update';
+import { isError } from '../log/index.js'
+import { unimplemented } from '../object.js';
+
+function defaultRenderError(data = {}) {
+  if (data.level && isError(data.level)) {
+    return JSON.stringify(data) + '\n';
+  }
+  return undefined;
+}
+
+export default class LogUpdateStream extends Writable {
+
+  constructor({
+    out = process.stdout,
+    err = process.stderr,
+    render = unimplemented,
+    renderError = defaultRenderError,
+  } = {}) {
+    super({
+      objectMode: true,
+    });
+    this._update = createLogUpdate(out);
+    this._err = err;
+    this._fns = {
+      render,
+      renderError,
+    };
+  }
+
+  _write(data, _, next) {
+    const errorMsg = this._renderError(data);
+    errorMsg && this._err.write(errorMsg);
+    this._update(this._render(data));
+    next();
+  }
+
+  _render(data) {
+    return this._fns.render(data);
+  }
+
+  _renderError(data) {
+    return this._fns.renderError(data);
+  }
+
+}

package/src/stream/output.js

@@ -11,7 +11,7 @@ export default class OutputStream extends Writable {
     super({
       objectMode,
     });
-    this._format = format || (v => `${v}`);
+    this._format = format || (objectMode ? defaultObjectModeFormat : defaultNonObjectModeFormat);
     this._out = out;
     this._err = err;
   }
@@ -22,3 +22,12 @@ export default class OutputStream extends Writable {
   }
 
 }
+
+function defaultObjectModeFormat(v) {
+  typeof v === 'object' ? JSON.stringify(v) : `${v}`;
+}
+
+function defaultNonObjectModeFormat(v) {
+  // TODO: handle buffer
+  return v;
+}