@miso.ai/server-commons 0.6.6-beta.2 → 0.6.6-beta.20

package/src/channel/time.js

@@ -0,0 +1,105 @@
+import { delay } from '../async.js';
+
+export default class TimeTracker {
+
+  constructor(timestamp = Date.now()) {
+    Object.defineProperty(this, 'constructedAt', { value: timestamp });
+    this._firstWriteAt = undefined;
+    this._write = 0;
+    this._pausedAt = undefined;
+    this._willResumeAt = undefined;
+    this._paused = 0;
+  }
+
+  // operation //
+  setFirstWrite(timestamp = Date.now()) {
+    if (this._firstWriteAt !== undefined) {
+      throw new Error(`First write time already set: ${this._firstWriteAt}`);
+    }
+    this._firstWriteAt = timestamp;
+  }
+
+  addWrite(duration) {
+    this._write += duration;
+  }
+
+  async pause(pauseTime, pausedAt = Date.now()) {
+    if (pauseTime <= 0) {
+      return;
+    }
+    if (this._firstWriteAt === undefined || this._pausedAt !== undefined) {
+      throw new Error(`Cannot pause() before first write or when paused.`);
+    }
+    this._pausedAt = pausedAt;
+    this._willResumeAt = pausedAt + pauseTime;
+    await delay(pauseTime);
+    this.paused && this.resume();
+  }
+
+  resume(resumeAt = Date.now()) {
+    if (this._firstWriteAt === undefined || this._pausedAt === undefined) {
+      throw new Error(`Cannot resume() before first write or when not paused.`);
+    }
+    this._paused += resumeAt - this._pausedAt;
+    this._pausedAt = this._willResumeAt = undefined;
+  }
+
+  // stats //
+  get paused() {
+    return !!this._pausedAt;
+  }
+
+  get firstWriteAt() {
+    return this._firstWriteAt;
+  }
+
+  get pausedAt() {
+    return this._pausedAt;
+  }
+
+  get willResumeAt() {
+    return this._willResumeAt;
+  }
+
+  get write() {
+    return this._write;
+  }
+
+  total(timestamp) {
+    return timestamp - this.constructedAt;
+  }
+
+  waiting(timestamp) {
+    return (this._firstWriteAt !== undefined ? this._firstWriteAt : timestamp) - this.constructedAt;
+  }
+
+  elapsed(timestamp) {
+    return this._firstWriteAt === undefined ? 0 : timestamp - this._firstWriteAt;
+  }
+
+  paused(timestamp) {
+    return this._paused + (this._pausedAt !== undefined ? timestamp - this._pausedAt : 0);
+  }
+
+  running(timestamp) {
+    return this.elapsed(timestamp) - this.paused(timestamp);
+  }
+
+  snapshot(timestamp = Date.now()) {
+    const { constructedAt, firstWriteAt, pausedAt, willResumeAt, write } = this;
+    return Object.freeze(trimObj({
+      currentTime: timestamp,
+      constructedAt,
+      firstWriteAt,
+      pausedAt,
+      willResumeAt,
+      write,
+      total: this.total(timestamp),
+      waiting: this.waiting(timestamp),
+      elapsed: this.elapsed(timestamp),
+      paused: this.paused(timestamp),
+      running: this.running(timestamp),
+    }));
+  }
+
+}

package/src/channel/transform.js

@@ -0,0 +1,83 @@
+import Channel from './channel.js';
+import { writeChannelInfo } from './events.js';
+import { trimObj } from '../object.js';
+
+export default class TransformChannel extends Channel {
+
+  constructor({
+    transform,
+    name = 'transform',
+    ...options
+  }) {
+    super({
+      name,
+      ...options,
+    });
+    if (transform) {
+      if (typeof transform !== 'function') {
+        throw new Error(`Transform must be a function.`);
+      }
+      this._transformData = transform;
+    }
+  }
+
+  async _transformData(event) {
+    throw new Error('Not implemented');
+  }
+
+  async _runCustomTransform(event) {
+    switch (event.type) {
+      case 'data':
+        await this._runData(event);
+        return;
+    }
+    await super._runCustomTransform(event);
+  }
+
+  async _runData(event) {
+    let transformed;
+    try {
+      transformed = this._transformData(event);
+    } catch (error) {
+      // write error to source event and pass through
+      const { type, message } = error;
+      transformed = {
+        errors: writeChannelInfo(this, [{ type, message }]),
+      };
+    }
+    this._validateEvent(transformed);
+    this.out.write(this._createDataEvent(event, transformed));
+  }
+
+  _createDataEvent({ sources = [], logs, warnings, errors, ...event } = {}, transformed) {
+    sources = [...sources, event];
+    logs = mergeInfoArray(logs, writeChannelInfo(this, transformed.logs));
+    warnings = mergeInfoArray(warnings, writeChannelInfo(this, transformed.warnings));
+    errors = mergeInfoArray(errors, writeChannelInfo(this, transformed.errors));
+    return trimObj({
+      ...transformed,
+      type: 'data',
+      sources,
+      logs,
+      warnings,
+      errors,
+    });
+  }
+
+  _validateEvent(event) {
+    if (typeof event !== 'object') {
+      throw new Error('the _transformData method must return an event');
+    }
+    if (event.payload === undefined && !event.ignored) {
+      throw new Error('Event payload is undefined');
+    }
+  }
+
+}
+
+function mergeInfoArray(a = [], b = []) {
+  if (a.length === 0 && b.length === 0) {
+    return undefined;
+  }
+  return [...a, ...b];
+}

package/src/channel/upgrade.js

@@ -0,0 +1,99 @@
+import { ChannelBase } from './component.js';
+import { ChannelOutput, createStartEvent, createEndEvent } from './events.js';
+
+export default class UpgradeChannel extends ChannelBase {
+
+  constructor({ name = 'upgrade', upgrade, objectMode, ...options } = {}) {
+    super({
+      ...options,
+      name,
+      writableObjectMode: objectMode,
+      readableObjectMode: true,
+    });
+
+    this._objectMode = objectMode;
+
+    if (upgrade) {
+      if (typeof upgrade !== 'function') {
+        throw new Error('Upgrade must be a function');
+      }
+      this._upgrade = upgrade.bind(this);
+    }
+
+    this._started = false;
+    this.out = new ChannelOutput(this);
+  }
+
+  async _parse(payload, encoding) {
+    if (this._objectMode) {
+      return payload;
+    }
+    if (Buffer.isBuffer(payload)) {
+      payload = payload.toString().trim();
+    }
+    if (this._options.asId) {
+      // convert to string
+      payload = `${payload}`.trim();
+    } else {
+      if (typeof payload === 'string' && payload.charAt(0) === '{' && payload.charAt(payload.length - 1) === '}') {
+        // should be an object
+        // TODO: ad-hoc!
+        payload = JSON.parse(payload);
+      }
+    }
+    return payload;
+  }
+
+  async _id(payload) {
+    if (this._options.asId) {
+      return payload;
+    }
+    if (typeof payload !== 'object') {
+      throw new Error('Cannot determine id from payload');
+    }
+    const id = this._options.idField;
+    if (id) {
+      return payload[id];
+    }
+    // ad-hoc!
+    return payload.product_id || payload.user_id || payload.id;
+  }
+
+  async _upgrade(payload) {
+    const id = await this._id(payload);
+    if (!id) {
+      throw new Error('Id not found in payload');
+    }
+    return {
+      id,
+      payload,
+    };
+  }
+
+  _createStartEvent() {
+    return createStartEvent(this);
+  }
+
+  _createEndEvent() {
+    return createEndEvent(this);
+  }
+
+  async _runTransform(payload, encoding) {
+    // first read -> write start event
+    if (!this._started) {
+      this._started = true;
+      this.out.write(this._createStartEvent());
+    }
+    payload = await this._parse(payload, encoding);
+    this.out.write({
+      ...(await this._upgrade(payload)),
+      type: 'data',
+    });
+  }
+
+  async _runFlush() {
+    // write end event
+    this.out.write(this._createEndEvent());
+  }
+
+}

package/src/channel/write.js

@@ -0,0 +1,154 @@
+import Channel from './channel.js';
+import TimeTracker from './time.js';
+import WriteChannelBuffer from './buffer.js';
+import WriteChannelSinkGate from './sink-gate.js';
+import WriteChannelSink from './sink.js';
+
+function normalizeBuffer(buffer) {
+  if (typeof buffer !== 'object') {
+    throw new Error('A buffer instance or buffer options is required.');
+  }
+  if (typeof buffer.push !== 'function') {
+    buffer = new WriteChannelBuffer(buffer);
+  }
+  return buffer;
+}
+
+function normalizeSink(sink) {
+  if (typeof sink !== 'object') {
+    throw new Error('A sink instance or sink options is required.');
+  }
+  if (typeof sink._write !== 'function') {
+    sink = new WriteChannelSink(sink);
+  }
+  return sink;
+}
+
+function normalizeSinkGate(sinkGate) {
+  if (!sinkGate) {
+    return undefined; // can be undefined
+  }
+  if (typeof sinkGate.blockedTime !== 'function') {
+    sinkGate = new WriteChannelSinkGate(sinkGate);
+  }
+  return sinkGate;
+}
+
+// TODO: we may as well split the buffering to a preceding channel
+
+export default class WriteChannel extends Channel {
+
+  constructor({ buffer, sink, sinkGate, ...options } = {}) {
+    super(options);
+    this._payloadBuffer = normalizeBuffer(buffer);
+    this._sink = normalizeSink(sink);
+    this._sink._channel = this;
+    this._sinkGate = normalizeSinkGate(sinkGate);
+    this._time = new TimeTracker();
+    this._index = 0;
+  }
+
+  get pulse() {
+    return {
+      ...super.pulse,
+      status: this._events.end ? 'finished' : this._time.firstWriteAt === undefined ? 'waiting' : this._time.paused ? 'paused' : 'running',
+      time: this._time.snapshot(Date.now()),
+      write: this._sink.state,
+    };
+  }
+
+  async _runCustomTransform(event) {
+    switch (event.type) {
+      case 'data':
+        if (event.payload !== undefined) {
+          await this._runData(event);
+          return;
+        }
+        break;
+    }
+    await super._runCustomTransform(event);
+  }
+
+  async _runData(event) {
+    await this.writeData(event);
+  }
+
+  async writeData(event) {
+    // TODO: dedupe
+    await this._dispatchAll(this._payloadBuffer.push(event));
+  }
+
+  async _runFlush() {
+    await this._dispatchAll(this._payloadBuffer.flush());
+    await this._sink.finished;
+
+    // end event is here
+    await super._runFlush();
+
+    this._sink.destroy();
+    this._payloadBuffer.destroy();
+  }
+
+  async _pauseIfNecessary() {
+    if (!this._sinkGate) {
+      return;
+    }
+    const pauseTime = this._sinkGate.blockedTime(this._sink.state);
+    await this._time.pause(pauseTime);
+  }
+
+  async _dispatchAll(requests) {
+    for (const request of requests) {
+      await this._pauseIfNecessary();
+      this._dispatch(request); // don't wait
+    }
+  }
+
+  async _dispatch(request) {
+    const index = this._index++;
+    if (this._time.firstWriteAt === undefined) {
+      this._time.setFirstWrite();
+    }
+
+    const { data: _0, payload: _1, ...restOfRequest } = request;
+    // TODO: we want to write real request/response event, so these have to be buried into the sink
+    this.out.write({
+      ...restOfRequest,
+      type: 'request',
+      index,
+    });
+
+    const { successful, failed, ...response } = await this._sink.write(request);
+    this._time.addWrite(response.timestamp - request.timestamp);
+
+    this.out.write({
+      ...response,
+      type: 'response',
+      index,
+    });
+
+    // write write event
+    const successfulIds = (successful && successful.data && successful.data.map(d => d.id)) || [];
+    if (successfulIds.length > 0) {
+      this.out.write({
+        ...this._createWriteEvent({ index, request, response, successfulIds }),
+        type: 'write',
+      });
+    }
+    // TODO: dedupe
+
+    // recover failed data events and pass through
+    for (const event of failed.data || []) {
+      // TODO: review this
+      this.out.pass(event);
+    }
+  }
+
+  _createWriteEvent({ index, successfulIds }) {
+    return {
+      index,
+      ids: successfulIds,
+    };
+  }
+
+}

package/src/index.js

@@ -5,6 +5,7 @@ export * from './file.js';
 export * from './config.js';
 export * from './async.js';
 
+export * from './channel/index.js';
 export * as sink from './sink/index.js';
 export * as buffer from './buffer/index.js';
 export * as stream from './stream/index.js';

package/src/stream/buffered-read.js

@@ -91,10 +91,10 @@ export default class BufferedReadStream extends Readable {
     this._debug(`[BufferedReadStream] Load response: ${JSON.stringify(response)} => data = ${data && data.length}`);
 
     // TODO: support strategy option: keepOrder = false
-    this._loads.push(request.index, () => this._resolveLoad(response, data));
+    this._loads.push(request.index, () => this._resolveLoad(response, data, request));
   }
 
-  _resolveLoad(response, records) {
+  _resolveLoad(response, records, request) {
     const state = this._state;
     const strategy = this._strategy;
 
@@ -104,15 +104,17 @@ export default class BufferedReadStream extends Readable {
     // apply terminate and filter function
     let terminate = false;
     const accepted = [];
+    let index = 0;
     for (const record of records) {
-      terminate = terminate || strategy.terminate(record, state);
-      if (!terminate && this._filter(record)) {
+      terminate = terminate || strategy.terminate(record, state, request, index);
+      if (!terminate && this._filter(record, state, request, index)) {
         state.accept();
         accepted.push(this._transform(record));
       }
       if (terminate) {
         break;
       }
+      index++;
     }
     if (terminate || (this._state.pendingLoads === 0 && this._state.exhausted)) {
       state.terminate();

package/src/stream/buffered-write.js

@@ -183,7 +183,7 @@ export default class BufferedWriteStream extends Transform {
       result: failed ? 'failed' : 'successful',
       index: request.index,
       records: request.records,
-      recovered: response.recovered || { records: 0, bytes: 0 },
+      recovered: summarizeRecovered(response.recovered),
       bytes: request.bytes,
       time: response.timestamp - request.timestamp,
     });
@@ -194,3 +194,11 @@ export default class BufferedWriteStream extends Transform {
   }
 
 }
+
+function summarizeRecovered(recovered) {
+  if (!recovered) {
+    return { records: 0, bytes: 0 };
+  }
+  const { records, bytes } = recovered;
+  return { records, bytes };
+}

package/src/stream/easy-transform.js

@@ -0,0 +1,66 @@
+import { Transform } from 'stream';
+
+export default class EasyTransform extends Transform {
+
+  constructor({ transform, flush, ...options } = {}) {
+    super(options);
+    if (transform) {
+      if (typeof transform !== 'function') {
+        throw new Error(`transform must be a function`);
+      }
+      this._runTransform = transform.bind(this);
+    }
+    if (flush) {
+      if (typeof flush !== 'function') {
+        throw new Error(`flush must be a function`);
+      }
+      this._runFlush = flush.bind(this);
+    }
+    this._buffer = [];
+    this._callback = undefined;
+  }
+
+  // customization //
+  async _runTransform(chunk, encoding) {
+    throw new Error(`Unimplemented.`);
+  }
+
+  async _runFlush() {}
+
+  // stream internal //
+  async _transform(chunk, encoding, next) {
+    await this._runTransform(chunk, encoding);
+    this._callback = next;
+    this._flushBuffer();
+  }
+
+  async _flush(done) {
+    await this._runFlush();
+    this._callback = done;
+    this._flushBuffer();
+  }
+
+  _read() {
+    this._flushBuffer();
+  }
+
+  _pushBuffer(chunk, encoding) {
+    this._buffer.push([chunk, encoding]);
+    this._flushBuffer();
+  }
+
+  _flushBuffer() {
+    const buffer = this._buffer;
+    while (buffer.length > 0) {
+      if (!this.push(...buffer.shift())) {
+        break;
+      }
+    }
+    if (this._callback) {
+      const callback = this._callback;
+      this._callback = undefined;
+      callback();
+    }
+  }
+
+}

package/src/stream/index.js

@@ -1,5 +1,6 @@
 export * from './misc.js';
 export * from './transform.js';
+export * from './joint-stream.js';
 export { default as BufferedReadStream } from './buffered-read.js';
 export { default as BufferedWriteStream } from './buffered-write.js';
 export { default as OutputStream } from './output.js';
@@ -7,3 +8,4 @@ export { default as LogUpdateStream } from './log-update.js';
 export { default as DiffStream } from './diff.js';
 export { default as ParallelTransform } from './parallel-transform.js';
 export { default as XmlParseStream } from './xml.js';
+export { default as EasyTransform } from './easy-transform.js';

package/src/stream/joint-stream.js

@@ -0,0 +1,23 @@
+import { Readable } from 'stream';
+
+export function joinStreams(streams, joinFn = defaultJoinFn) {
+  if (typeof joinFn !== 'function') {
+    throw new Error(`joinFn must be a function: ${joinFn}`);
+  }
+  return Readable.from(generate(streams, joinFn));
+}
+
+function defaultJoinFn(...args) {
+  return args;
+}
+
+async function * generate(streams, joinFn) {
+  const iterators = streams.map(stream => stream[Symbol.asyncIterator]());
+  while (true) {
+    const entries = await Promise.all(iterators.map(iterator => iterator.next()));
+    if (entries.every(entry => entry.done)) {
+      break;
+    }
+    yield joinFn(...entries.map(entry => entry.value));
+  }
+}

package/src/stream/misc.js

@@ -1,4 +1,8 @@
-import { Transform, pipeline as _pipeline } from 'stream';
+import { Transform } from 'stream';
+import { pipeline as _pipeline } from 'stream/promises';
+import { chain as _chain } from 'stream-chain';
+
+export const chain = _chain;
 
 export function parse({ lenient } = {}) {
   const parsnFn = lenient ? parseJsonIfPossible : JSON.parse;
@@ -31,8 +35,12 @@ export function stringify() {
   });
 }
 
+/**
+ * @deprecated Use native stream/promises pipeline instead.
+ */
 export async function pipeline(...streams) {
-  return new Promise((resolve, reject) => _pipeline(...streams.filter(s => s), err => err ? reject(err) : resolve()));
+  const flattened = streams.length === 1 && Array.isArray(streams[0]) ? streams[0] : streams;
+  return _pipeline(flattened.filter(Boolean));
 }
 
 export async function collect(stream) {