@miso.ai/server-commons 0.6.6-beta.16 → 0.6.6-beta.18

package/src/channel/transform.js

@@ -0,0 +1,83 @@
+import Channel from './channel.js';
+import { writeChannelInfo } from './events.js';
+import { trimObj } from '../object.js';
+
+export default class TransformChannel extends Channel {
+
+  constructor({
+    transform,
+    name = 'transform',
+    ...options
+  }) {
+    super({
+      name,
+      ...options,
+    });
+    if (transform) {
+      if (typeof transform !== 'function') {
+        throw new Error(`Transform must be a function.`);
+      }
+      this._transformData = transform;
+    }
+  }
+
+  async _transformData(event) {
+    throw new Error('Not implemented');
+  }
+
+  async _runCustomTransform(event) {
+    switch (event.type) {
+      case 'data':
+        await this._runData(event);
+        return;
+    }
+    await super._runCustomTransform(event);
+  }
+
+  async _runData(event) {
+    let transformed;
+    try {
+      transformed = this._transformData(event);
+    } catch (error) {
+      // write error to source event and pass through
+      const { type, message } = error;
+      transformed = {
+        errors: writeChannelInfo(this, [{ type, message }]),
+      };
+    }
+    this._validateEvent(transformed);
+    this.out.write(this._createDataEvent(event, transformed));
+  }
+
+  _createDataEvent({ sources = [], logs, warnings, errors, ...event } = {}, transformed) {
+    sources = [...sources, event];
+    logs = mergeInfoArray(logs, writeChannelInfo(this, transformed.logs));
+    warnings = mergeInfoArray(warnings, writeChannelInfo(this, transformed.warnings));
+    errors = mergeInfoArray(errors, writeChannelInfo(this, transformed.errors));
+    return trimObj({
+      ...transformed,
+      type: 'data',
+      sources,
+      logs,
+      warnings,
+      errors,
+    });
+  }
+
+  _validateEvent(event) {
+    if (typeof event !== 'object') {
+      throw new Error('the _transformData method must return an event');
+    }
+    if (event.payload === undefined && !event.ignored) {
+      throw new Error('Event payload is undefined');
+    }
+  }
+
+}
+
+function mergeInfoArray(a = [], b = []) {
+  if (a.length === 0 && b.length === 0) {
+    return undefined;
+  }
+  return [...a, ...b];
+}

package/src/channel/upgrade.js

@@ -0,0 +1,99 @@
+import { ChannelBase } from './component.js';
+import { ChannelOutput, createStartEvent, createEndEvent } from './events.js';
+
+export default class UpgradeChannel extends ChannelBase {
+
+  constructor({ name = 'upgrade', upgrade, objectMode, ...options } = {}) {
+    super({
+      ...options,
+      name,
+      writableObjectMode: objectMode,
+      readableObjectMode: true,
+    });
+
+    this._objectMode = objectMode;
+
+    if (upgrade) {
+      if (typeof upgrade !== 'function') {
+        throw new Error('Upgrade must be a function');
+      }
+      this._upgrade = upgrade.bind(this);
+    }
+
+    this._started = false;
+    this.out = new ChannelOutput(this);
+  }
+
+  async _parse(payload, encoding) {
+    if (this._objectMode) {
+      return payload;
+    }
+    if (Buffer.isBuffer(payload)) {
+      payload = payload.toString().trim();
+    }
+    if (this._options.asId) {
+      // convert to string
+      payload = `${payload}`.trim();
+    } else {
+      if (typeof payload === 'string' && payload.charAt(0) === '{' && payload.charAt(payload.length - 1) === '}') {
+        // should be an object
+        // TODO: ad-hoc!
+        payload = JSON.parse(payload);
+      }
+    }
+    return payload;
+  }
+
+  async _id(payload) {
+    if (this._options.asId) {
+      return payload;
+    }
+    if (typeof payload !== 'object') {
+      throw new Error('Cannot determine id from payload');
+    }
+    const id = this._options.idField;
+    if (id) {
+      return payload[id];
+    }
+    // ad-hoc!
+    return payload.product_id || payload.user_id || payload.id;
+  }
+
+  async _upgrade(payload) {
+    const id = await this._id(payload);
+    if (!id) {
+      throw new Error('Id not found in payload');
+    }
+    return {
+      id,
+      payload,
+    };
+  }
+
+  _createStartEvent() {
+    return createStartEvent(this);
+  }
+
+  _createEndEvent() {
+    return createEndEvent(this);
+  }
+
+  async _runTransform(payload, encoding) {
+    // first read -> write start event
+    if (!this._started) {
+      this._started = true;
+      this.out.write(this._createStartEvent());
+    }
+    payload = await this._parse(payload, encoding);
+    this.out.write({
+      ...(await this._upgrade(payload)),
+      type: 'data',
+    });
+  }
+
+  async _runFlush() {
+    // write end event
+    this.out.write(this._createEndEvent());
+  }
+
+}

package/src/channel/write.js

@@ -0,0 +1,154 @@
+import Channel from './channel.js';
+import TimeTracker from './time.js';
+import WriteChannelBuffer from './buffer.js';
+import WriteChannelSinkGate from './sink-gate.js';
+import WriteChannelSink from './sink.js';
+
+function normalizeBuffer(buffer) {
+  if (typeof buffer !== 'object') {
+    throw new Error('A buffer instance or buffer options is required.');
+  }
+  if (typeof buffer.push !== 'function') {
+    buffer = new WriteChannelBuffer(buffer);
+  }
+  return buffer;
+}
+
+function normalizeSink(sink) {
+  if (typeof sink !== 'object') {
+    throw new Error('A sink instance or sink options is required.');
+  }
+  if (typeof sink._write !== 'function') {
+    sink = new WriteChannelSink(sink);
+  }
+  return sink;
+}
+
+function normalizeSinkGate(sinkGate) {
+  if (!sinkGate) {
+    return undefined; // can be undefined
+  }
+  if (typeof sinkGate.blockedTime !== 'function') {
+    sinkGate = new WriteChannelSinkGate(sinkGate);
+  }
+  return sinkGate;
+}
+
+// TODO: we may as well split the buffering to a preceding channel
+
+export default class WriteChannel extends Channel {
+
+  constructor({ buffer, sink, sinkGate, ...options } = {}) {
+    super(options);
+    this._payloadBuffer = normalizeBuffer(buffer);
+    this._sink = normalizeSink(sink);
+    this._sink._channel = this;
+    this._sinkGate = normalizeSinkGate(sinkGate);
+    this._time = new TimeTracker();
+    this._index = 0;
+  }
+
+  get pulse() {
+    return {
+      ...super.pulse,
+      status: this._events.end ? 'finished' : this._time.firstWriteAt === undefined ? 'waiting' : this._time.paused ? 'paused' : 'running',
+      time: this._time.snapshot(Date.now()),
+      write: this._sink.state,
+    };
+  }
+
+  async _runCustomTransform(event) {
+    switch (event.type) {
+      case 'data':
+        if (event.payload !== undefined) {
+          await this._runData(event);
+          return;
+        }
+        break;
+    }
+    await super._runCustomTransform(event);
+  }
+
+  async _runData(event) {
+    await this.writeData(event);
+  }
+
+  async writeData(event) {
+    // TODO: dedupe
+    await this._dispatchAll(this._payloadBuffer.push(event));
+  }
+
+  async _runFlush() {
+    await this._dispatchAll(this._payloadBuffer.flush());
+    await this._sink.finished;
+
+    // end event is here
+    await super._runFlush();
+
+    this._sink.destroy();
+    this._payloadBuffer.destroy();
+  }
+
+  async _pauseIfNecessary() {
+    if (!this._sinkGate) {
+      return;
+    }
+    const pauseTime = this._sinkGate.blockedTime(this._sink.state);
+    await this._time.pause(pauseTime);
+  }
+
+  async _dispatchAll(requests) {
+    for (const request of requests) {
+      await this._pauseIfNecessary();
+      this._dispatch(request); // don't wait
+    }
+  }
+
+  async _dispatch(request) {
+    const index = this._index++;
+    if (this._time.firstWriteAt === undefined) {
+      this._time.setFirstWrite();
+    }
+
+    const { data: _0, payload: _1, ...restOfRequest } = request;
+    // TODO: we want to write real request/response event, so these have to be buried into the sink
+    this.out.write({
+      ...restOfRequest,
+      type: 'request',
+      index,
+    });
+
+    const { successful, failed, ...response } = await this._sink.write(request);
+    this._time.addWrite(response.timestamp - request.timestamp);
+
+    this.out.write({
+      ...response,
+      type: 'response',
+      index,
+    });
+
+    // write write event
+    const successfulIds = (successful && successful.data && successful.data.map(d => d.id)) || [];
+    if (successfulIds.length > 0) {
+      this.out.write({
+        ...this._createWriteEvent({ index, request, response, successfulIds }),
+        type: 'write',
+      });
+    }
+    // TODO: dedupe
+
+    // recover failed data events and pass through
+    for (const event of failed.data || []) {
+      // TODO: review this
+      this.out.pass(event);
+    }
+  }
+
+  _createWriteEvent({ index, successfulIds }) {
+    return {
+      index,
+      ids: successfulIds,
+    };
+  }
+
+}

package/src/index.js

@@ -5,6 +5,7 @@ export * from './file.js';
 export * from './config.js';
 export * from './async.js';
 
+export * from './channel/index.js';
 export * as sink from './sink/index.js';
 export * as buffer from './buffer/index.js';
 export * as stream from './stream/index.js';

package/src/stream/buffered-write.js

@@ -183,7 +183,7 @@ export default class BufferedWriteStream extends Transform {
       result: failed ? 'failed' : 'successful',
       index: request.index,
       records: request.records,
-      recovered: response.recovered || { records: 0, bytes: 0 },
+      recovered: summarizeRecovered(response.recovered),
       bytes: request.bytes,
       time: response.timestamp - request.timestamp,
     });
@@ -194,3 +194,11 @@ export default class BufferedWriteStream extends Transform {
   }
 
 }
+
+function summarizeRecovered({ recovered }) {
+  if (!recovered) {
+    return { records: 0, bytes: 0 };
+  }
+  const { records, bytes } = recovered;
+  return { records, bytes };
+}

package/src/stream/easy-transform.js

@@ -0,0 +1,66 @@
+import { Transform } from 'stream';
+
+export default class EasyTransform extends Transform {
+
+  constructor({ transform, flush, ...options } = {}) {
+    super(options);
+    if (transform) {
+      if (typeof transform !== 'function') {
+        throw new Error(`transform must be a function`);
+      }
+      this._runTransform = transform.bind(this);
+    }
+    if (flush) {
+      if (typeof flush !== 'function') {
+        throw new Error(`flush must be a function`);
+      }
+      this._runFlush = flush.bind(this);
+    }
+    this._buffer = [];
+    this._callback = undefined;
+  }
+
+  // customization //
+  async _runTransform(chunk, encoding) {
+    throw new Error(`Unimplemented.`);
+  }
+
+  async _runFlush() {}
+
+  // stream internal //
+  async _transform(chunk, encoding, next) {
+    await this._runTransform(chunk, encoding);
+    this._callback = next;
+    this._flushBuffer();
+  }
+
+  async _flush(done) {
+    await this._runFlush();
+    this._callback = done;
+    this._flushBuffer();
+  }
+
+  _read() {
+    this._flushBuffer();
+  }
+
+  _pushBuffer(chunk, encoding) {
+    this._buffer.push([chunk, encoding]);
+    this._flushBuffer();
+  }
+
+  _flushBuffer() {
+    const buffer = this._buffer;
+    while (buffer.length > 0) {
+      if (!this.push(...buffer.shift())) {
+        break;
+      }
+    }
+    if (this._callback) {
+      const callback = this._callback;
+      this._callback = undefined;
+      callback();
+    }
+  }
+
+}

package/src/stream/index.js

@@ -8,3 +8,4 @@ export { default as LogUpdateStream } from './log-update.js';
 export { default as DiffStream } from './diff.js';
 export { default as ParallelTransform } from './parallel-transform.js';
 export { default as XmlParseStream } from './xml.js';
+export { default as EasyTransform } from './easy-transform.js';

package/src/stream/misc.js

@@ -1,4 +1,5 @@
-import { Transform, pipeline as _pipeline } from 'stream';
+import { Transform } from 'stream';
+import { pipeline as _pipeline } from 'stream/promises';
 import { chain as _chain } from 'stream-chain';
 
 export const chain = _chain;
@@ -34,8 +35,12 @@ export function stringify() {
   });
 }
 
+/**
+ * @deprecated Use native stream/promises pipeline instead.
+ */
 export async function pipeline(...streams) {
-  return new Promise((resolve, reject) => _pipeline(...streams.filter(s => s), err => err ? reject(err) : resolve()));
+  const flattened = streams.length === 1 && Array.isArray(streams[0]) ? streams[0] : streams;
+  return _pipeline(flattened.filter(Boolean));
 }
 
 export async function collect(stream) {