@miso.ai/server-commons 0.6.3 → 0.6.4-beta.0

package/package.json

@@ -20,5 +20,5 @@
     "uuid": "^9.0.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.3"
+  "version": "0.6.4-beta.0"
 }

package/src/control/base.js

@@ -0,0 +1,41 @@
+import EventEmitter from 'events';
+
+export default class TaskControl {
+
+  constructor() {
+    this._events = new EventEmitter();
+    this._ready = true;
+  }
+
+  on(event, listener) {
+    this._events.on(event, listener);
+    return () => this._events.off(event, listener);
+  }
+
+  once(event, listener) {
+    this._events.once(event, listener);
+    return () => this._events.off(event, listener);
+  }
+
+  open(id, data) {}
+
+  close(id) {}
+
+  get ready() {
+    return this._ready;
+  }
+
+  _setReady(value) {
+    value = !!value
+    if (this._ready === value) {
+      return;
+    }
+    this._ready = value;
+    this._events.emit('ready', value);
+  }
+
+  destroy() {
+    this._events.removeAllListeners();
+  }
+
+}

package/src/control/collective.js

@@ -0,0 +1,38 @@
+import TaskControl from './base.js';
+
+export default class CollectiveTaskControl extends TaskControl {
+
+  constructor(...members) {
+    super();
+    this._members = members;
+    this._unsubscribes = members.map((control) => control.on('ready', () => this._syncReady()));
+    this._syncReady();
+  }
+
+  open(id, data) {
+    for (const control of this._members) {
+      control.open(id, data);
+    }
+  }
+
+  close(id) {
+    for (const control of this._members) {
+      control.close(id);
+    }
+  }
+
+  _syncReady() {
+    this._setReady(this._members.every((control) => control.ready));
+  }
+
+  destroy() {
+    for (const unsubscribe of this._unsubscribes) {
+      unsubscribe();
+    }
+    for (const control of this._members) {
+      control.destroy();
+    }
+    super.destroy();
+  }
+
+}

package/src/control/create.js

@@ -0,0 +1,22 @@
+import TaskControl from './base.js';
+import CollectiveTaskControl from './collective.js';
+import WaterMarkTaskControl from './watermark.js';
+import ThrottledTaskControl from './throttle.js';
+
+export function createTaskControl({
+  highWaterMark,
+  throttle,
+} = {}) {
+  const controls = [];
+  if (highWaterMark) {
+    controls.push(new WaterMarkTaskControl({
+      highWaterMark,
+    }));
+  }
+  if (throttle) {
+    controls.push(new ThrottledTaskControl({
+      interval: throttle,
+    }));
+  }
+  return controls.length > 0 ? new CollectiveTaskControl(...controls) : new TaskControl();
+}

package/src/control/index.js

@@ -0,0 +1,5 @@
+export { default as TaskControl } from './base.js';
+export { default as CollectiveTaskControl } from './collective.js';
+export { default as WaterMarkTaskControl } from './watermark.js';
+export { default as ThrottledTaskControl } from './throttle.js';
+export { createTaskControl } from './create.js';

package/src/control/throttle.js

@@ -0,0 +1,28 @@
+import TaskControl from './base.js';
+
+export default class ThrottledTaskControl extends TaskControl {
+
+  constructor({
+    interval,
+  }) {
+    super();
+    if (typeof interval !== 'number' || interval <= 0) {
+      throw new Error(`interval must be a positive number: ${interval}`);
+    }
+    this._interval = interval;
+  }
+
+  get interval() {
+    return this._interval;
+  }
+
+  open(id, data) {
+    this._setReady(false);
+    this._timeoutId && clearTimeout(this._timeoutId);
+    this._timeoutId = setTimeout(() => {
+      this._setReady(true);
+      this._timeoutId = undefined;
+    }, this._interval);
+  }
+
+}

package/src/control/watermark.js

@@ -0,0 +1,34 @@
+import TaskControl from './base.js';
+
+export default class WaterMarkTaskControl extends TaskControl{
+
+  constructor({
+    highWaterMark,
+  }) {
+    super();
+    if (!Number.isInteger(highWaterMark) || highWaterMark <= 0) {
+      throw new Error('highWaterMark must be a positive integer');
+    }
+    this._highWaterMark = highWaterMark;
+    this._waterLevel = 0;
+  }
+
+  get waterLevel() {
+    return this._waterLevel;
+  }
+
+  open(id, data) {
+    this._waterLevel++;
+    this._syncReady();
+  }
+
+  close(id) {
+    this._waterLevel--;
+    this._syncReady();
+  }
+
+  _syncReady() {
+    this._setReady(this._waterLevel < this._highWaterMark);
+  }
+
+}

package/src/stream/index.js

@@ -5,3 +5,4 @@ export { default as BufferedWriteStream } from './buffered-write.js';
 export { default as OutputStream } from './output.js';
 export { default as LogUpdateStream } from './log-update.js';
 export { default as DiffStream } from './diff.js';
+export { default as ParallelTransform } from './parallel-transform.js';

package/src/stream/parallel-transform.js

@@ -0,0 +1,82 @@
+import { Duplex } from 'stream';
+import { v4 as uuid } from 'uuid';
+import { createTaskControl } from '../control/index.js';
+
+export default class ParallelTransform extends Duplex {
+
+  constructor({
+    transform,
+    controls,
+    ...options
+  } = {}) {
+    super(options);
+    if (transform !== undefined && typeof transform !== 'function') {
+      throw new Error('transform must be a function');
+    }
+    this._transformFn = transform;
+    this._control = createTaskControl(controls);
+
+    this._next = undefined;
+    this._pushPaused = false;
+    this._outputBuffer;
+
+    this._control.on('ready', (ready) => ready && this._callNext());
+    this.on('drain', () => this._handleDrain());
+  }
+
+  async _transformFn(chunk, encoding) {
+    throw new Error('transform function not defined');
+  }
+
+  _write(chunk, encoding, next) {
+    const id = uuid();
+    this._next = next;
+    this._control.open(id, [chunk, encoding]);
+    (async () => {
+      try {
+        const output = await this._transformFn(chunk, encoding);
+        this._control.close(id);
+        this._writeOutput(output);
+      } catch (error) {
+        this._error(error);
+      }
+    })();
+    this._callNext();
+  }
+
+  _callNext() {
+    if (this._next && this._control.ready) {
+      const next = this._next;
+      this._next = undefined;
+      next();
+    }
+  }
+
+  _writeOutput(output) {
+    if (this._outputBuffer) {
+      this._outputBuffer.push(output);
+    } else if (!this.push(output)) {
+      this._outputBuffer = [];
+    }
+  }
+
+  _handleDrain() {
+    if (!this._outputBuffer) {
+      return;
+    }
+    const buffer = this._outputBuffer;
+    for (let i = 0, len = buffer.length; i < len; i++) {
+      if (!this.push(buffer[i])) {
+        this._outputBuffer = buffer.slice(i + 1);
+        break;
+      }
+    }
+  }
+
+  _read() {}
+
+  _error(error) {
+    this.emit('error', error);
+  }
+
+}