@miso.ai/server-commons 0.6.3-beta.9 → 0.6.4-beta.0

package/package.json

@@ -20,5 +20,5 @@
     "uuid": "^9.0.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.3-beta.9"
+  "version": "0.6.4-beta.0"
 }

package/src/control/base.js

@@ -0,0 +1,41 @@
+import EventEmitter from 'events';
+
+export default class TaskControl {
+
+  constructor() {
+    this._events = new EventEmitter();
+    this._ready = true;
+  }
+
+  on(event, listener) {
+    this._events.on(event, listener);
+    return () => this._events.off(event, listener);
+  }
+
+  once(event, listener) {
+    this._events.once(event, listener);
+    return () => this._events.off(event, listener);
+  }
+
+  open(id, data) {}
+
+  close(id) {}
+
+  get ready() {
+    return this._ready;
+  }
+
+  _setReady(value) {
+    value = !!value
+    if (this._ready === value) {
+      return;
+    }
+    this._ready = value;
+    this._events.emit('ready', value);
+  }
+
+  destroy() {
+    this._events.removeAllListeners();
+  }
+
+}

package/src/control/collective.js

@@ -0,0 +1,38 @@
+import TaskControl from './base.js';
+
+export default class CollectiveTaskControl extends TaskControl {
+
+  constructor(...members) {
+    super();
+    this._members = members;
+    this._unsubscribes = members.map((control) => control.on('ready', () => this._syncReady()));
+    this._syncReady();
+  }
+
+  open(id, data) {
+    for (const control of this._members) {
+      control.open(id, data);
+    }
+  }
+
+  close(id) {
+    for (const control of this._members) {
+      control.close(id);
+    }
+  }
+
+  _syncReady() {
+    this._setReady(this._members.every((control) => control.ready));
+  }
+
+  destroy() {
+    for (const unsubscribe of this._unsubscribes) {
+      unsubscribe();
+    }
+    for (const control of this._members) {
+      control.destroy();
+    }
+    super.destroy();
+  }
+
+}

package/src/control/create.js

@@ -0,0 +1,22 @@
+import TaskControl from './base.js';
+import CollectiveTaskControl from './collective.js';
+import WaterMarkTaskControl from './watermark.js';
+import ThrottledTaskControl from './throttle.js';
+
+export function createTaskControl({
+  highWaterMark,
+  throttle,
+} = {}) {
+  const controls = [];
+  if (highWaterMark) {
+    controls.push(new WaterMarkTaskControl({
+      highWaterMark,
+    }));
+  }
+  if (throttle) {
+    controls.push(new ThrottledTaskControl({
+      interval: throttle,
+    }));
+  }
+  return controls.length > 0 ? new CollectiveTaskControl(...controls) : new TaskControl();
+}

package/src/control/index.js

@@ -0,0 +1,5 @@
+export { default as TaskControl } from './base.js';
+export { default as CollectiveTaskControl } from './collective.js';
+export { default as WaterMarkTaskControl } from './watermark.js';
+export { default as ThrottledTaskControl } from './throttle.js';
+export { createTaskControl } from './create.js';

package/src/control/throttle.js

@@ -0,0 +1,28 @@
+import TaskControl from './base.js';
+
+export default class ThrottledTaskControl extends TaskControl {
+
+  constructor({
+    interval,
+  }) {
+    super();
+    if (typeof interval !== 'number' || interval <= 0) {
+      throw new Error(`interval must be a positive number: ${interval}`);
+    }
+    this._interval = interval;
+  }
+
+  get interval() {
+    return this._interval;
+  }
+
+  open(id, data) {
+    this._setReady(false);
+    this._timeoutId && clearTimeout(this._timeoutId);
+    this._timeoutId = setTimeout(() => {
+      this._setReady(true);
+      this._timeoutId = undefined;
+    }, this._interval);
+  }
+
+}

package/src/control/watermark.js

@@ -0,0 +1,34 @@
+import TaskControl from './base.js';
+
+export default class WaterMarkTaskControl extends TaskControl{
+
+  constructor({
+    highWaterMark,
+  }) {
+    super();
+    if (!Number.isInteger(highWaterMark) || highWaterMark <= 0) {
+      throw new Error('highWaterMark must be a positive integer');
+    }
+    this._highWaterMark = highWaterMark;
+    this._waterLevel = 0;
+  }
+
+  get waterLevel() {
+    return this._waterLevel;
+  }
+
+  open(id, data) {
+    this._waterLevel++;
+    this._syncReady();
+  }
+
+  close(id) {
+    this._waterLevel--;
+    this._syncReady();
+  }
+
+  _syncReady() {
+    this._setReady(this._waterLevel < this._highWaterMark);
+  }
+
+}

package/src/date.js

@@ -29,6 +29,10 @@ export function parseDuration(expr, unit) {
   return value * ts;
 }
 
+export function getYear(dateStr) {
+  return new Date(dateStr).getFullYear();
+}
+
 export function startOfDate(expr) {
   if (expr === undefined) {
     return undefined;

package/src/stream/buffered-read.js

@@ -24,12 +24,13 @@ export default class BufferedReadStream extends Readable {
     this._strategy.initialize(this, source);
   }
 
-  async _construct() {
+  async _construct(done) {
     if (this._source.init) {
       this._debug(`[BufferedReadStream] init source start`);
       await this._source.init();
       this._debug(`[BufferedReadStream] init source done`);
     }
+    done();
   }
 
   async _read() {

package/src/stream/buffered-write.js

@@ -76,10 +76,11 @@ export default class BufferedWriteStream extends Transform {
     done();
   }
 
-  async _transform(record, _) {
+  async _transform(record, _, next) {
     this._pushStartEventIfNecessary();
     this._dispatchAll(this._buffer.push(record));
     await this._pauseIfNecessary();
+    next();
   }
 
   async _flush(done) {

package/src/stream/diff.js

@@ -49,13 +49,14 @@ export default class DiffStream extends Transform {
     }
   }
 
-  async _transform(value, _) {
+  _transform(value, _, next) {
     if (value instanceof Buffer) {
       value = value.toString();
     }
     // dedupe
     const input = this._inputDataSet;
     if (input.has(value)) {
+      next();
       return;
     }
     input.add(value);
@@ -66,6 +67,7 @@ export default class DiffStream extends Transform {
     } else {
       this._output(true, value);
     }
+    next();
   }
 
   async _flush(done) {

package/src/stream/index.js

@@ -5,3 +5,4 @@ export { default as BufferedWriteStream } from './buffered-write.js';
 export { default as OutputStream } from './output.js';
 export { default as LogUpdateStream } from './log-update.js';
 export { default as DiffStream } from './diff.js';
+export { default as ParallelTransform } from './parallel-transform.js';

package/src/stream/misc.js

@@ -35,13 +35,6 @@ export async function pipeline(...streams) {
   return new Promise((resolve, reject) => _pipeline(...streams.filter(s => s), err => err ? reject(err) : resolve()));
 }
 
-export async function pipelineToStdout(...streams) {
-  return new Promise((resolve, reject) => 
-    _pipeline(...streams, err => err ? reject(err) : resolve())
-      .pipe(process.stdout, { end: false })
-  );
-}
-
 export async function collect(stream) {
   const records = [];
   for await (const record of stream) {
@@ -72,7 +65,7 @@ export function take(n, { transform: _, ...options } = {}) {
   let count = 0;
   return new Transform({
     ...options,
-    transform (chunk, _, next) {
+    transform(chunk, _, next) {
       if (count > n) {
         next(undefined);
         this.end();

package/src/stream/parallel-transform.js

@@ -0,0 +1,82 @@
+import { Duplex } from 'stream';
+import { v4 as uuid } from 'uuid';
+import { createTaskControl } from '../control/index.js';
+
+export default class ParallelTransform extends Duplex {
+
+  constructor({
+    transform,
+    controls,
+    ...options
+  } = {}) {
+    super(options);
+    if (transform !== undefined && typeof transform !== 'function') {
+      throw new Error('transform must be a function');
+    }
+    this._transformFn = transform;
+    this._control = createTaskControl(controls);
+
+    this._next = undefined;
+    this._pushPaused = false;
+    this._outputBuffer;
+
+    this._control.on('ready', (ready) => ready && this._callNext());
+    this.on('drain', () => this._handleDrain());
+  }
+
+  async _transformFn(chunk, encoding) {
+    throw new Error('transform function not defined');
+  }
+
+  _write(chunk, encoding, next) {
+    const id = uuid();
+    this._next = next;
+    this._control.open(id, [chunk, encoding]);
+    (async () => {
+      try {
+        const output = await this._transformFn(chunk, encoding);
+        this._control.close(id);
+        this._writeOutput(output);
+      } catch (error) {
+        this._error(error);
+      }
+    })();
+    this._callNext();
+  }
+
+  _callNext() {
+    if (this._next && this._control.ready) {
+      const next = this._next;
+      this._next = undefined;
+      next();
+    }
+  }
+
+  _writeOutput(output) {
+    if (this._outputBuffer) {
+      this._outputBuffer.push(output);
+    } else if (!this.push(output)) {
+      this._outputBuffer = [];
+    }
+  }
+
+  _handleDrain() {
+    if (!this._outputBuffer) {
+      return;
+    }
+    const buffer = this._outputBuffer;
+    for (let i = 0, len = buffer.length; i < len; i++) {
+      if (!this.push(buffer[i])) {
+        this._outputBuffer = buffer.slice(i + 1);
+        break;
+      }
+    }
+  }
+
+  _read() {}
+
+  _error(error) {
+    this.emit('error', error);
+  }
+
+}