@miso.ai/server-commons 0.6.6-beta.17 → 0.6.6-beta.18

package/package.json

@@ -24,5 +24,5 @@
     "uuid": "^9.0.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.6-beta.17"
+  "version": "0.6.6-beta.18"
 }

package/src/channel/buffer.js

@@ -0,0 +1,115 @@
+function defaultSerialize({ payload }) {
+  return typeof payload === 'string' ? payload : JSON.stringify(payload);
+}
+
+function normalizeOptions({
+  payloadPrefix = '[',
+  payloadSuffix = ']',
+  payloadDelimiter = ',',
+  serialize = defaultSerialize,
+  recordCap,
+  byteCap,
+} = {}) {
+  if (!recordCap && !byteCap) {
+    throw new Error('At least one of recordCap or byteCap must be specified.');
+  }
+  if (typeof serialize !== 'function') {
+    throw new Error('serialize must be a function.');
+  }
+  return {
+    payloadPrefix,
+    payloadSuffix,
+    payloadDelimiter,
+    serialize,
+    recordCap,
+    byteCap,
+  };
+}
+
+export default class WriteChannelBuffer {
+
+  constructor(options) {
+    this._options = options = normalizeOptions(options);
+    this._affixSize = (options.payloadPrefix.length + options.payloadSuffix.length) * 2;
+    this._clear();
+  }
+
+  _clear() {
+    this._records = 0;
+    this._bytes = this._affixSize;
+    this._payload = this._options.payloadPrefix;
+    this._data = [];
+  }
+
+  /**
+   * Returns true if the buffer is empty.
+   */
+  get empty() {
+    return this._records === 0;
+  }
+
+  /**
+   * Returns true if the buffer is full.
+   */
+  get full() {
+    const { recordCap, byteCap } = this._options;
+    return (recordCap !== undefined && this._records >= recordCap) || 
+      (byteCap !== undefined && this._bytes >= byteCap);
+  }
+
+  /**
+   * Take a data event and push it into the buffer.
+   * Returns an array of requests if the buffer is full, or an empty array otherwise.
+   * @returns An array of requests.
+   */
+  push(event) {
+    let dispatches = [];
+    const serializedPayload = this._options.serialize(event);
+    const contentSize = serializedPayload.length * 2;
+    const { empty } = this;
+
+    if (!empty) {
+      const delimiterSize = this._options.payloadDelimiter.length * 2;
+      if (this._bytes + delimiterSize + contentSize > this._options.byteCap) {
+        // if buffer is not empty and exceeds bytes limit after pushing, then flush right now
+        dispatches = this.flush();
+      } else {
+        // non-empty, not flushed
+        this._bytes += delimiterSize;
+        this._payload += this._options.payloadDelimiter;
+      }
+    }
+
+    this._records++;
+    this._bytes += contentSize;
+    this._payload += serializedPayload;
+    this._data.push(event);
+
+    // it's possible that the new record is so large that the buffer is instantly full even after a flush
+    // in this case, the buffer generates two dispatches from the push
+    if (this.full) {
+      dispatches = [...dispatches, ...this.flush()];
+    }
+
+    return dispatches;
+  }
+
+  /**
+   * Flush the buffer, which returns an array of requests (which can be empty) and reset the buffer to empty state.
+   * @returns An array of requests.
+   */
+  flush() {
+    if (this.empty) {
+      return [];
+    }
+    const records = this._records;
+    const bytes = this._bytes;
+    const payload = this._payload + this._options.payloadSuffix;
+    const data = this._data;
+    this._clear();
+    return [{ payload, records, bytes, data }];
+  }
+
+  destroy() {}
+
+}

package/src/channel/channel.js

@@ -0,0 +1,105 @@
+import { ChannelBase } from './component.js';
+import { LOG_LEVEL, ChannelOutput, createStartEvent, createEndEvent } from './events.js';
+
+export default class Channel extends ChannelBase {
+
+  constructor({ transform, flush, ...options } = {}) {
+    super(options);
+
+    if (transform) {
+      if (typeof transform !== 'function') {
+        throw new Error('Transform must be a function');
+      }
+      this._runCustomTransform = transform.bind(this);
+    }
+    if (flush) {
+      if (typeof flush !== 'function') {
+        throw new Error('Flush must be a function');
+      }
+      this._runCustomFlush = flush.bind(this);
+    }
+
+    this._upstream = {};
+    this.out = new ChannelOutput(this);
+  }
+
+  // customization //
+  async _runCustomTransform(event) {
+    this.out.pass(event);
+  }
+
+  async _runCustomFlush() {}
+
+  async _runFlush() {
+    await this._runCustomFlush();
+    // clear heartbeat if necessary
+    // TODO
+    if (!this._upstream.start) {
+      // not seen start event from upstream, so the upstream is empty
+      throw new Error('Start event not received from upstream');
+    }
+    if (!this._upstream.end) {
+      // not seen end event from upstream, warn
+      this.out.log(LOG_LEVEL.WARNING, 'End event not received from upstream');
+    }
+    // write my own end event
+    this.out.write(this._createEndEvent(this._upstream.end));
+
+    await super._runFlush();
+  }
+
+  _readUpstreamStartEvent(event) {
+    // take the heartbeat setting
+    // TODO
+  }
+
+
+
+  // internal //
+  async _runTransform(event) {
+    // expect the start event from upstream
+    if (!this._upstream.start) {
+      if (event.type !== 'start') {
+        throw new Error(`The first event received from upstream must be a start event: ${JSON.stringify(event)}`);
+      }
+      this._upstream.start = event;
+      this.out.write(this._createStartEvent(event));
+      this._readUpstreamStartEvent(event);
+      return;
+    }
+    switch (event.type) {
+      case 'start':
+        throw new Error(`Received a second start event from upstream: ${JSON.stringify(event)}`);
+      case 'end':
+        if (this._upstream.end) {
+          // seen end event from upstream, warn
+          this.out.log(LOG_LEVEL.WARNING, `Received a second end event from upstream: ${JSON.stringify(event)}`);
+        } else {
+          this._upstream.end = event;
+        }
+        break;
+      default:
+        await this._runCustomTransform(event);
+    }
+  }
+
+  _writePulseEvent() {
+    try {
+      this.out.write({
+        type: 'pulse',
+        ...this.pulse,
+      });
+    } catch (error) {
+      this.out.log(LOG_LEVEL.ERROR, `Failed to write pulse event: ${error.message}`, { error });
+    }
+  }
+
+  _createStartEvent(event) {
+    return createStartEvent(this, event);
+  }
+
+  _createEndEvent(event) {
+    return createEndEvent(this, event);
+  }
+
+}

package/src/channel/component.js

@@ -0,0 +1,71 @@
+import EasyTransform from '../stream/easy-transform.js';
+import { trimObj } from '../object.js';
+import { stackEventInfo } from './events.js';
+
+// TODO: camelCase to kebab-case on options keys
+function exportOptions(options) {
+  return trimObj({ ...options });
+}
+
+export class ChannelComponent {
+
+  constructor(options = {}) {
+    this._options = this._normalizeOptions(options);
+  }
+
+  _normalizeOptions({
+    ...options
+  }) {
+    return trimObj({
+      ...options,
+    });
+  }
+
+  get options() {
+    return exportOptions(this._options);
+  }
+
+}
+
+// TODO: by mixin
+
+export class ChannelBase extends EasyTransform {
+
+  constructor({ name, ...options } = {}) {
+    super({ ...options, objectMode: true });
+    this._options = this._normalizeOptions(options);
+    this._name = name || this.constructor.name;
+  }
+
+  _normalizeOptions({
+    objectMode,
+    writableObjectMode,
+    readableObjectMode,
+    ...options
+  }) {
+    return trimObj({
+      ...options,
+    });
+  }
+
+  get name() {
+    return this._name;
+  }
+
+  get options() {
+    return exportOptions(this._options);
+  }
+
+  get pulse() {
+    return {};
+  }
+
+  get result() {
+    return {};
+  }
+
+  _stackEventInfo(event, field, info) {
+    return stackEventInfo(this, event, field, info);
+  }
+
+}

package/src/channel/constants.js

@@ -0,0 +1,6 @@
+export const LOG_LEVEL = {
+  DEBUG: 'debug',
+  INFO: 'info',
+  WARNING: 'warning',
+  ERROR: 'error',
+};

package/src/channel/downgrade.js

@@ -0,0 +1,27 @@
+import EasyTransform from '../stream/easy-transform.js';
+
+export default class DowngradeChannel extends EasyTransform {
+
+  constructor({ objectMode, ...options } = {}) {
+    super({
+      ...options,
+      writableObjectMode: true,
+      readableObjectMode: objectMode,
+    });
+    this._objectMode = objectMode;
+  }
+
+  async _runTransform({ type, payload }) {
+    switch (type) {
+      case 'data':
+        payload !== undefined && this._pushBuffer(this._format(payload));
+        break;
+    }
+  }
+
+  _format(payload) {
+    // TODO: review this
+    return (this._objectMode || typeof payload === 'string') ? payload : JSON.stringify(payload);
+  }
+
+}

package/src/channel/events.js

@@ -0,0 +1,117 @@
+import { trimObj } from '../object.js';
+
+export const LOG_LEVEL = {
+  DEBUG: 'debug',
+  INFO: 'info',
+  WARNING: 'warning',
+  ERROR: 'error',
+};
+
+export function createStartEvent(channel, event = { type: 'start' }) {
+  return stackEventInfo(channel, event, 'options', channel.options);
+}
+
+export function createEndEvent(channel, event = { type: 'end' }) {
+  return stackEventInfo(channel, event, 'results', channel.result);
+}
+
+export function stackEventInfo(channel, event = {}, field, info) {
+  info = normalizeChannelInfo({
+    ...info,
+    channel: channel.name,
+  });
+  return {
+    ...event,
+    [field]: [...(event[field] || []), info],
+  };
+}
+
+export function writeChannelInfo(channel, array) {
+  if (array === undefined) {
+    return undefined;
+  }
+  return array.map(info => normalizeChannelInfo({
+    ...info,
+    channel: channel.name,
+  }));
+}
+
+export function normalizeChannelInfo({ channel, timestamp, ...info }) {
+  return trimObj({
+    channel,
+    ...info,
+  });
+}
+
+export function normalizeEvent({
+  type,
+  channel,
+  index,
+  timestamp,
+  ...rest
+} = {}) {
+  // re-order properties
+  return trimObj({
+    type,
+    channel,
+    index,
+    ...rest,
+    timestamp,
+  });
+}
+
+export function validateEvent(event) {
+  switch (event.type) {
+    case 'data':
+      if (!event.id) {
+        throw new Error('Id is required for data events');
+      }
+      break;
+  }
+}
+
+export function generateDefaultSinkResponse({ records, data }) {
+  // default implementation: assume all successful
+  return {
+    writes: 1,
+    successful: {
+      records,
+      data,
+    },
+    failed: {
+      records: 0,
+      data: [],
+    },
+  };
+}
+
+export class ChannelOutput {
+
+  constructor(channel) {
+    this._channel = channel;
+  }
+
+  pass(event) {
+    this._channel._pushBuffer(normalizeEvent(event));
+  }
+
+  write({ timestamp = Date.now(), ...rest }) {
+    const event = {
+      ...rest,
+      channel: this._channel.name,
+      timestamp,
+    };
+    validateEvent(event);
+    this._channel._pushBuffer(normalizeEvent(event));
+  }
+
+  log(level, message, data) {
+    this.write({
+      type: 'log',
+      logLevel: level,
+      message,
+      ...data,
+    });
+  }
+
+}

package/src/channel/index.js

@@ -0,0 +1,8 @@
+export { default as Channel } from './channel.js';
+export { default as WriteChannel } from './write.js';
+export { default as WriteChannelBuffer } from './buffer.js';
+export { default as WriteChannelSink } from './sink.js';
+export { default as WriteChannelSinkGate } from './sink-gate.js';
+export { default as UpgradeChannel } from './upgrade.js';
+export { default as DowngradeChannel } from './downgrade.js';
+export { default as TransformChannel } from './transform.js';

package/src/channel/sink-gate.js

@@ -0,0 +1,39 @@
+export default class WriteChannelSinkGate {
+
+  constructor(options = {}) {
+    this._options = options;
+  }
+
+  blockedTime(sinkState, now = Date.now()) {
+    const { start, started } = sinkState;
+    if (start === undefined) {
+      return 0;
+    }
+    const elapsed = now - start;
+    const targetBps = this._targetBps(now);
+    const targetRps = this._targetRps(now);
+    const targetWps = this._targetWps(now);
+
+    const shallElapsed = Math.max(started.records / targetRps, started.bytes / targetBps, started.writes / targetWps) * 1000;
+
+    const blockedTime = shallElapsed - elapsed;
+    if (blockedTime <= 1000) {
+      return 0;
+    }
+    // TODO: review this
+    return Math.ceil(blockedTime);
+  }
+
+  _targetBps(timestamp) {
+    return this._options.bytesPerSecond;
+  }
+
+  _targetRps(timestamp) {
+    return this._options.recordsPerSecond;
+  }
+
+  _targetWps(timestamp) {
+    return this._options.writesPerSecond;
+  }
+
+}

package/src/channel/sink.js

@@ -0,0 +1,103 @@
+import { trimObj } from '../object.js';
+import Resolution from '../resolution.js';
+import { ChannelComponent } from './component.js';
+import { generateDefaultSinkResponse } from './events.js';
+
+function clone(obj) {
+  return Object.freeze(trimObj({ ...obj }));
+}
+
+export default class WriteChannelSink extends ChannelComponent {
+
+  constructor({ write, ...options } = {}) {
+    super(options);
+    if (write) {
+      if (typeof write !== 'function') {
+        throw new Error('write must be a function.');
+      }
+      this._write = write.bind(this);
+    }
+    this._state = {
+      start: undefined,
+      started: { index: -1, writes: 0, records: 0, bytes: 0 },
+      finished: { index: -1, writes: 0, records: 0, bytes: 0, successful: 0, failed: 0 },
+    };
+  }
+
+  get state() {
+    const { start, started, finished } = this._state;
+    return clone({
+      start,
+      started: clone(started),
+      finished: clone(finished),
+    });
+  }
+
+  async write(request) {
+    if (this._finishedRes) {
+      this._finishedRes.reject(new Error(`More data written after finished call`));
+      this._finishedRes = undefined;
+    }
+    const { records, bytes } = request;
+    const now = Date.now();
+    const { started, finished } = this._state;
+    if (started.writes === 0) {
+      this._state.start = now;
+    }
+    started.index++;
+    started.writes++;
+    started.records += records;
+    started.bytes += bytes;
+
+    const { index } = started;
+    const response = await this._write(request);
+
+    // sanity check
+    if (!response || !response.successful || !response.failed) {
+      throw new Error(`Invalid response: ${JSON.stringify(response)}`);
+    }
+    if (response.successful.records + response.failed.records !== records) {
+      throw new Error(`Invalid response: successful records (${response.successful.records}) + failed records (${response.failed.records}) !== records (${records})`);
+    }
+
+    // a _write call may use more than 1 write
+    const writes = response.writes || 1;
+    if (writes > 1) {
+      started.writes += writes - 1; // patch the number
+    }
+    finished.index = index;
+    finished.writes += writes;
+    finished.records += records;
+    finished.bytes += bytes;
+    finished.successful += response.successful.records;
+    finished.failed += response.failed.records;
+
+    if (this._finishedRes && finished.writes === started.writes) {
+      this._finishedRes.resolve();
+      this._finishedRes = undefined;
+    }
+
+    return response;
+  }
+
+  async _write(event) {
+    // default implementation: do nothing but assume all successful
+    return generateDefaultSinkResponse(event);
+  }
+
+  get finished() {
+    const { started, finished } = this._state;
+    if (started.writes === finished.writes) {
+      return Promise.resolve();
+    }
+    return (this._finishedRes || (this._finishedRes = new Resolution())).promise;
+  }
+
+  destroy() {
+    if (this._finishedRes) {
+      this._finishedRes.reject(new Error(`Stream is destroyed.`));
+      this._finishedRes = undefined;
+    }
+  }
+
+}

package/src/channel/time.js

@@ -0,0 +1,105 @@
+import { delay } from '../async.js';
+
+export default class TimeTracker {
+
+  constructor(timestamp = Date.now()) {
+    Object.defineProperty(this, 'constructedAt', { value: timestamp });
+    this._firstWriteAt = undefined;
+    this._write = 0;
+    this._pausedAt = undefined;
+    this._willResumeAt = undefined;
+    this._paused = 0;
+  }
+
+  // operation //
+  setFirstWrite(timestamp = Date.now()) {
+    if (this._firstWriteAt !== undefined) {
+      throw new Error(`First write time already set: ${this._firstWriteAt}`);
+    }
+    this._firstWriteAt = timestamp;
+  }
+
+  addWrite(duration) {
+    this._write += duration;
+  }
+
+  async pause(pauseTime, pausedAt = Date.now()) {
+    if (pauseTime <= 0) {
+      return;
+    }
+    if (this._firstWriteAt === undefined || this._pausedAt !== undefined) {
+      throw new Error(`Cannot pause() before first write or when paused.`);
+    }
+    this._pausedAt = pausedAt;
+    this._willResumeAt = pausedAt + pauseTime;
+    await delay(pauseTime);
+    this.paused && this.resume();
+  }
+
+  resume(resumeAt = Date.now()) {
+    if (this._firstWriteAt === undefined || this._pausedAt === undefined) {
+      throw new Error(`Cannot resume() before first write or when not paused.`);
+    }
+    this._paused += resumeAt - this._pausedAt;
+    this._pausedAt = this._willResumeAt = undefined;
+  }
+
+  // stats //
+  get paused() {
+    return !!this._pausedAt;
+  }
+
+  get firstWriteAt() {
+    return this._firstWriteAt;
+  }
+
+  get pausedAt() {
+    return this._pausedAt;
+  }
+
+  get willResumeAt() {
+    return this._willResumeAt;
+  }
+
+  get write() {
+    return this._write;
+  }
+
+  total(timestamp) {
+    return timestamp - this.constructedAt;
+  }
+
+  waiting(timestamp) {
+    return (this._firstWriteAt !== undefined ? this._firstWriteAt : timestamp) - this.constructedAt;
+  }
+
+  elapsed(timestamp) {
+    return this._firstWriteAt === undefined ? 0 : timestamp - this._firstWriteAt;
+  }
+
+  paused(timestamp) {
+    return this._paused + (this._pausedAt !== undefined ? timestamp - this._pausedAt : 0);
+  }
+
+  running(timestamp) {
+    return this.elapsed(timestamp) - this.paused(timestamp);
+  }
+
+  snapshot(timestamp = Date.now()) {
+    const { constructedAt, firstWriteAt, pausedAt, willResumeAt, write } = this;
+    return Object.freeze(trimObj({
+      currentTime: timestamp,
+      constructedAt,
+      firstWriteAt,
+      pausedAt,
+      willResumeAt,
+      write,
+      total: this.total(timestamp),
+      waiting: this.waiting(timestamp),
+      elapsed: this.elapsed(timestamp),
+      paused: this.paused(timestamp),
+      running: this.running(timestamp),
+    }));
+  }
+
+}