@miso.ai/server-sdk 0.6.0-beta.0

package/src/logger/progress.legacy.js

@@ -0,0 +1,60 @@
+import { log, stream } from '@miso.ai/server-commons';
+
+const { formatDuration, formatBytes, formatTable } = log;
+
+export default class LegacyProgressLogStream extends stream.LogUpdateStream {
+
+  constructor({
+    out = process.stdout,
+    err = process.stderr,
+  } = {}) {
+    super({
+      out,
+      err,
+    });
+  }
+
+  _renderError({ state: _, ...record }) {
+    return super._renderError(record);
+  }
+
+  _render({ state }) {
+    const { elapsed, pending, successful, failed, apiBps, sentBps } = state;
+
+    // sum pending requests
+    const pendingSums = sumMetrics(pending);
+
+    const table = formatTable([
+      ['', 'Requests', 'Records', 'Bytes', 'Server Time', 'Latency'],
+      ['Pending', pendingSums.requests, pendingSums.records, formatBytes(pendingSums.bytes), '--', '--'],
+      ['Successful', successful.requests, successful.records, formatBytes(successful.bytes), formatDuration(successful.took), formatDuration(successful.latency)],
+      ['Failed', failed.requests, failed.records, formatBytes(failed.bytes), formatDuration(failed.took), formatDuration(failed.latency)],
+    ]);
+
+    return `
+Time elapsed: ${formatDuration(elapsed)}
+API Speed: ${formatSpeed(apiBps)}
+Client Speed: ${formatSpeed(sentBps)}
+
+${table}
+`;
+  }
+
+}
+
+function sumMetrics(requests) {
+  return requests.reduce((acc, req) => {
+    acc.requests ++;
+    acc.records += req.records;
+    acc.bytes += req.bytes;
+    return acc;
+  }, {
+    requests: 0,
+    records: 0,
+    bytes: 0,
+  });
+}
+
+function formatSpeed(value) {
+  return isNaN(value) ? 'N/A' : `${formatBytes(value)}/s`;
+}

package/src/logger/standard.js

@@ -0,0 +1,58 @@
+import { Writable } from 'stream';
+import { log } from '@miso.ai/server-commons';
+import { FORMAT } from './constants.js';
+
+export default class StandardLogStream extends Writable {
+
+  constructor({
+    level = log.INFO,
+    format = FORMAT.JSON,
+    out = process.stdout,
+    err = process.stderr,
+  } = {}) {
+    super({
+      objectMode: true,
+    });
+    if (log.LEVELS.indexOf(level) < 0) {
+      throw new Error(`Unrecognized log level: ${level}`);
+    }
+    this._level = level;
+    this._debug = log.reachesThreshold(log.DEBUG, level);
+    this._formatter = getFormatter(format);
+    this._out = out;
+    this._err = err;
+  }
+
+  _write(record, _, next) {
+    const { level } = record;
+    if (!log.reachesThreshold(level, this._level)) {
+      next();
+      return;
+    }
+    if (!this._debug) {
+      delete record.state;
+    }
+    (log.isError(level) ? this._err : this._out).write(this._formatter(record) + '\n');
+    next();
+  }
+
+}
+
+function getFormatter(format) {
+  switch (format) {
+    case FORMAT.TEXT:
+      return formatText;
+    case FORMAT.JSON:
+      return formatJson;
+    default:
+      throw new Error(`Unrecognized format: ${format}`);
+  }
+}
+
+function formatJson(record) {
+  return JSON.stringify(record);
+}
+
+function formatText({ level, timestamp, event, ...record }) {
+  return `[${new Date(timestamp).toISOString()}][${level.toUpperCase()}] ${event}, ${JSON.stringify(record)}`;
+}

package/src/stream/api-sink.js

@@ -0,0 +1,66 @@
+import { sink, trimObj } from '@miso.ai/server-commons';
+import ServiceStats from './service-stats.js';
+
+export default class ApiSink extends sink.BpsSink {
+
+  constructor(client, options) {
+    super(options);
+    this._client = client;
+    this._serviceStats = new ServiceStats();
+  }
+
+  _normalizeOptions({
+    params,
+    ...options
+  } = {}) {
+    return {
+      ...super._normalizeOptions(options),
+      ...trimObj({
+        params: this._normalizeParams(params),
+      }),
+    };
+  }
+
+  _normalizeParams(params) {
+    if (!params || params.length === 0) {
+      return undefined;
+    }
+    return params.reduce((acc, param) => {
+      const [key, value = '1'] = param.split('=');
+      acc[key] = value;
+      return acc;
+    }, {});
+  }
+
+  get serviceStats() {
+    return this._serviceStats.snapshot();
+  }
+
+  async _write(payload, { records, bytes }) {
+    let data;
+    try {
+      data = await this._execute(payload);
+    } catch(error) {
+      // not axios-handled error
+      if (!error.response) {
+        throw error;
+      }
+      data = error.response.data;
+      if (typeof data!== 'object') {
+        data = trimObj({ errors: true, cause: data });
+      }
+    }
+
+    // keep track of service stats on successful calls
+    if (!data.errors) {
+      this._serviceStats.track({ records, bytes, took: data.took });
+    }
+
+    return data;
+  }
+
+  async _execute(payload) {
+    throw new Error(`Unimplemented.`);
+  }
+
+}

package/src/stream/delete-sink.js

@@ -0,0 +1,26 @@
+import ApiSink from './api-sink.js';
+
+export default class DeleteSink extends ApiSink {
+
+  constructor(client, options) {
+    super(client, options);
+  }
+
+  _normalizeOptions({
+    ...options
+  } = {}) {
+    if (!options.type) {
+      throw new Error(`Type is required.`);
+    }
+    return {
+      ...super._normalizeOptions(options),
+    };
+  }
+
+  async _execute(payload) {
+    const { type, params } = this._options;
+    const response = await this._client._delete(type, payload, { params });
+    return response.data;
+  }
+
+}

package/src/stream/delete.js

@@ -0,0 +1,111 @@
+import { stream, buffer } from '@miso.ai/server-commons';
+import version from '../version.js';
+import DeleteSink from './delete-sink.js';
+import DeletionStats from './deletion-state.js';
+
+export default class DeleteStream extends stream.BufferedWriteStream {
+
+  constructor(client, type, {
+    name,
+    // super
+    objectMode,
+    heartbeatInterval,
+    // sink
+    recordsPerSecond,
+    // buffer
+    recordsPerRequest,
+  }) {
+    super({
+      name,
+      type,
+      version,
+      objectMode,
+      heartbeatInterval,
+    });
+    if (type !== 'products' && type !== 'users') {
+      throw new Error(`Unsupported type: ${type}`);
+    }
+
+    this._client = client;
+    this._type = type;
+
+    this._buffer = createBuffer(type, {
+      recordsPerRequest,
+    });
+
+    this._sink = new DeleteSink(client, {
+      type,
+      recordsPerSecond,
+    });
+
+    this._deletionStats = new DeletionStats();
+  }
+
+  get serviceStats() {
+    return this._sink.serviceStats;
+  }
+
+  get deletionStats() {
+    return this._deletionStats.snapshot();
+  }
+
+  _exportConfig() {
+    return {
+      client: this._client.options,
+      ...super._exportConfig(),
+    }
+  }
+
+  async _writeToSink(payload, request) {
+    const response = await super._writeToSink(payload, request);
+
+    this._deletionStats.track(request, response);
+
+    return response;
+  }
+
+  _output(message, args) {
+    const output = super._output(message, args);
+
+    // if upload fails, emit extracted payload at response event
+    if (message.event === 'response' && args.response && args.response.errors && args.payload) {
+      output.payload = JSON.parse(args.payload);
+    }
+
+    // TODO: we should find a way to place deletion stats as a main part in the events
+
+    // add upload stats
+    output.state = {
+      ...output.state,
+      stats: {
+        service: this.serviceStats,
+        deletion: this.deletionStats,
+      },
+    };
+
+    return output;
+  }
+
+}
+
+const DEFAULT_BUFFER_OPTIONS = Object.freeze({
+  suffix: ']}}',
+  separator: ',',
+  recordsLimit: 1000,
+});
+
+function createBuffer(type, {
+  recordsPerRequest,
+}) {
+  const key = type === 'products' ? 'product_ids' : 'user_ids';
+  const options = {
+    ...DEFAULT_BUFFER_OPTIONS,
+    objectMode: true,
+    prefix: `{"data":{"${key}":[`,
+    transform: v => v.toString(),
+  };
+  if (recordsPerRequest) {
+    options.recordsLimit = recordsPerRequest;
+  }
+  return new buffer.JsonBuffer(options);
+}

package/src/stream/deletion-state.js

@@ -0,0 +1,32 @@
+export default class DeletionStats {
+
+  constructor() {
+    this._records = 0;
+    this._deleted = 0;
+    this._notFound = 0;
+  }
+
+  get records() {
+    return this._records;
+  }
+
+  get deleted() {
+    return this._deleted;
+  }
+
+  get notFound() {
+    return this._notFound;
+  }
+
+  track(request, { data: response }) {
+    this._records += request.records;
+    this._deleted += (response.deleted && response.deleted.count) || 0;
+    this._notFound += (response.not_found && response.not_found.count) || 0;
+  }
+
+  snapshot() {
+    const { records, deleted, notFound } = this;
+    return Object.freeze({ records, deleted, notFound });
+  }
+
+}

package/src/stream/service-stats.js

@@ -0,0 +1,43 @@
+export default class ServiceStats {
+
+  constructor() {
+    this._requests = this._records = this._bytes = this._took = 0;
+  }
+
+  track({ records, bytes, took } = {}) {
+    if (isNaN(took) || took <= 0) {
+      return;
+    }
+    this._requests++;
+    this._bytes += records;
+    this._bytes += bytes;
+    this._took += took;
+  }
+
+  get requests() {
+    return this._requests;
+  }
+
+  get records() {
+    return this._records;
+  }
+
+  get bytes() {
+    return this._bytes;
+  }
+
+  get took() {
+    return this._took;
+  }
+
+  get bps() {
+    const { took, bytes } = this;
+    return took > 0 ? (bytes / took * 1000) : NaN;
+  }
+
+  snapshot() {
+    const { requests, records, bytes, took, bps } = this;
+    return Object.freeze({ requests, records, bytes, took, bps });
+  }
+
+}

package/src/stream/upload-buffer.js

@@ -0,0 +1,34 @@
+import { buffer } from '@miso.ai/server-commons';
+
+const DEFAULT_OPTIONS = Object.freeze({
+  prefix: '{"data":[',
+  suffix: ']}',
+  separator: ',',
+  bytesLimit: 1024 * 1024,
+});
+
+export default function create(type, { objectMode, recordsPerRequest, bytesPerRequest } = {}) {
+  const options = {
+    ...DEFAULT_OPTIONS,
+    objectMode,
+  };
+  // TODO: validate RPR, BPR values
+  if (bytesPerRequest) {
+    options.bytesLimit = bytesPerRequest;
+  }
+
+  switch (type) {
+    case 'users':
+    case 'products':
+      options.recordsLimit = recordsPerRequest || 200;
+      return new buffer.JsonBuffer(options);
+    case 'interactions':
+      options.recordsLimit = recordsPerRequest || 1000;
+      return new buffer.JsonBuffer(options);
+    case 'experiment-events':
+      options.recordsLimit = 1;
+      return new buffer.JsonBuffer(options);
+    default:
+      throw new Error(`Unrecognized type: ${type}`);
+  }
+}

package/src/stream/upload-sink.js

@@ -0,0 +1,104 @@
+import ApiSink from './api-sink.js';
+
+class UploadSink extends ApiSink {
+
+  constructor(client, options) {
+    super(client, options);
+  }
+
+  _normalizeOptions({
+    async,
+    dryRun,
+    ...options
+  } = {}) {
+    if (!options.type) {
+      throw new Error(`Type is required.`);
+    }
+    return {
+      ...super._normalizeOptions(options),
+      async: !!async,
+      dryRun: !!dryRun,
+    };
+  }
+
+  async _execute(payload) {
+    const { type, async, dryRun, params } = this._options;
+    const response = await this._client.upload(type, payload, { async, dryRun, params });
+    return response.data;
+  }
+
+}
+
+class DataSetUploadSink extends UploadSink {
+
+  constructor(client, options) {
+    super(client, options);
+    this._stats.api = { count: 0, bytes: 0, took: 0 };
+  }
+
+  _normalizeOptions({
+    apiBpsRate,
+    apiBpsSampleThreshold = 1024 * 1024,
+    ...options
+  } = {}) {
+    return {
+      ...super._normalizeOptions(options),
+      apiBpsRate: this._normalizeApiBpsRate(apiBpsRate, options),
+      apiBpsSampleThreshold,
+    };
+  }
+
+  _normalizeApiBpsRate(apiBpsRate, options) {
+    if (options.type === 'interactions' || options.async) {
+      // in async mode, apiBps is meaningless
+      return false;
+    }
+    if (apiBpsRate === undefined || apiBpsRate === null) {
+      // default to 1 if absent
+      return 1;
+    }
+    if (apiBpsRate === false || (!isNaN(apiBpsRate) && apiBpsRate > 0)) {
+      // legal values
+      return apiBpsRate;
+    }
+    throw new Error(`Illegal apiBpsRate value: ${apiBpsRate}`);
+  }
+
+  _targetBps() {
+    const { bytesPerSecond: configuredBps, apiBpsRate, apiBpsSampleThreshold } = this._options;
+    if (apiBpsRate && this._stats.api.bytes < apiBpsSampleThreshold) {
+      // use configured BPS until we have enough data from API response
+      return configuredBps;
+    }
+    const apiBps = this._serviceStats.bps;
+    return !isNaN(apiBps) ? Math.max(apiBps * apiBpsRate, configuredBps) : configuredBps;
+  }
+
+}
+
+class ExperimentEventUploadSink extends UploadSink {
+
+  constructor(client, options) {
+    super(client, { ...options, type: 'experiment-events' });
+  }
+
+  async _execute(payload) {
+    const { experimentId } = this._options;
+    const response = await this._client.uploadExperimentEvent(experimentId, payload);
+    return response.data;
+  }
+
+}
+
+export default function create(client, type, options) {
+  switch (type) {
+    case 'users':
+    case 'products':
+    case 'interactions':
+      return new DataSetUploadSink(client, { ...options, type });
+    case 'experiment-events':
+      return new ExperimentEventUploadSink(client, options);
+    default:
+      throw new Error(`Unrecognized type: ${type}`);
+  }
+}

package/src/stream/upload.js

@@ -0,0 +1,81 @@
+import { stream } from '@miso.ai/server-commons';
+import version from '../version.js';
+import createSink from './upload-sink.js';
+import createBuffer from './upload-buffer.js';
+
+export default class UploadStream extends stream.BufferedWriteStream {
+
+  constructor(client, type, {
+    name,
+    // super
+    objectMode,
+    heartbeatInterval,
+    // sink
+    async,
+    dryRun,
+    params,
+    experimentId,
+    recordsPerSecond,
+    bytesPerSecond,
+    // buffer
+    recordsPerRequest,
+    bytesPerRequest,
+  } = {}) {
+    super({
+      name,
+      type,
+      version,
+      objectMode,
+      heartbeatInterval,
+    });
+
+    this._client = client;
+    this._type = type;
+
+    this._sink = createSink(client, type, {
+      async, 
+      dryRun,
+      params,
+      experimentId,
+      recordsPerSecond,
+      bytesPerSecond,
+    });
+
+    this._buffer = createBuffer(type, {
+      objectMode,
+      recordsPerRequest,
+      bytesPerRequest,
+    });
+  }
+
+  get serviceStats() {
+    return this._sink.serviceStats;
+  }
+
+  _exportConfig() {
+    return {
+      client: this._client.options,
+      ...super._exportConfig(),
+    }
+  }
+
+  _output(message, args) {
+    const output = super._output(message, args);
+
+    // if upload fails, emit extracted payload at response event
+    if (message.event === 'response' && args.response && args.response.errors && args.payload) {
+      output.payload = JSON.parse(args.payload);
+    }
+
+    // add upload stats
+    output.state = {
+      ...output.state,
+      stats: {
+        service: this.serviceStats,
+      },
+    };
+
+    return output;
+  }
+
+}