@miso.ai/server-sdk 0.6.0-beta.1 → 0.6.0

package/cli/transform.js

@@ -32,7 +32,22 @@ async function run({ file }) {
 
 async function getTransformStream(loc) {
   const mod = await import(join(PWD, loc));
-  return mod.default ? new mod.default() : new Transform({ objectMode: mod.objectMode !== false, ...mod });
+  return mod.default ? new mod.default() : new Transform(normalizeOptions(mod));
+}
+
+function normalizeOptions({
+  objectMode,
+  readableObjectMode,
+  writableObjectMode,
+  ...options
+} = {}) {
+  readableObjectMode = readableObjectMode !== undefined ? readableObjectMode : objectMode !== undefined ? objectMode : true;
+  writableObjectMode = writableObjectMode !== undefined ? writableObjectMode : objectMode !== undefined ? objectMode : true;
+  return {
+    readableObjectMode,
+    writableObjectMode,
+    ...options
+  };
 }
 
 export default {

package/cli/upload.js

@@ -1,6 +1,6 @@
 import split2 from 'split2';
 import { log, stream } from '@miso.ai/server-commons';
-import { MisoClient, logger } from '../src/index.js';
+import { MisoClient, logger, normalize } from '../src/index.js';
 
 function build(yargs) {
   return yargs
@@ -12,6 +12,10 @@ function build(yargs) {
       alias: ['dry'],
       describe: 'Dry run mode',
     })
+    .option('lenient', {
+      describe: 'Accept some enient record schema',
+      type: 'boolean',
+    })
     .option('records-per-request', {
       alias: ['rpr'],
       describe: 'How many records to send in a request',
@@ -37,11 +41,6 @@ function build(yargs) {
       alias: ['name'],
       describe: 'Stream name that shows up in log messages',
     })
-    .option('legacy', {
-      type: 'boolean',
-      default: false,
-    })
-    .hide('legacy')
     .option('log-level', {
       describe: 'Log level',
     })
@@ -56,6 +55,7 @@ const run = type => async ({
   param: params,
   async,
   ['dry-run']: dryRun,
+  lenient,
   ['records-per-request']: recordsPerRequest,
   ['bytes-per-request']: bytesPerRequest,
   ['bytes-per-second']: bytesPerSecond,
@@ -63,7 +63,6 @@ const run = type => async ({
   debug,
   progress,
   ['stream-name']: name,
-  legacy,
   ['log-level']: loglevel,
   ['log-format']: logFormat,
 }) => {
@@ -73,8 +72,10 @@ const run = type => async ({
 
   const client = new MisoClient({ key, server });
 
+  const uploadStreamObjectMode = lenient;
+
   const uploadStream = client.createUploadStream(type, {
-    legacy,
+    objectMode: uploadStreamObjectMode,
     name,
     async, 
     dryRun,
@@ -85,19 +86,29 @@ const run = type => async ({
     bytesPerRequest,
     bytesPerSecond,
     experimentId,
+    extra: {
+      lenient,
+    },
   });
 
   const logStream = logger.createLogStream({
     api: 'upload',
     type,
-    legacy,
     level: loglevel,
     format: logFormat,
   });
 
+  // standard: stdin -> split2 -> upload -> log
+  // lenient: stdin -> split2 -> parse -> normalize -> upload -> log
+  // notice that the output of split2 are strings, while input/output of normalize are objects
+
   await stream.pipeline(
     process.stdin,
     split2(),
+    ...(lenient ? [
+      stream.parse(),
+      new normalize.Stream(type),
+    ] : []),
     uploadStream,
     logStream,
   );

package/package.json

@@ -16,11 +16,11 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.0-beta.1",
+    "@miso.ai/server-commons": "0.6.0",
     "axios": "^0.27.2",
     "dotenv": "^16.0.1",
     "split2": "^4.1.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.0-beta.1"
+  "version": "0.6.0"
 }

package/src/api/base.js

@@ -0,0 +1,62 @@
+import axios from 'axios';
+import { upload, batchDelete, buildUrl } from './helpers.js';
+import UploadStream from '../stream/upload.js';
+import DeleteStream from '../stream/delete.js';
+
+export class Queries {
+
+  constructor(client, group) {
+    this._client = client;
+    this._group = group;
+  }
+
+  async _run(path, payload, options) {
+    // TODO: options
+    const url = buildUrl(this._client, `${this._group}/${path}`);
+    return (await axios.post(url, payload)).data.data;
+  }
+
+}
+
+export class Writable {
+
+  constructor(client, type) {
+    this._client = client;
+    this._type = type;
+  }
+
+  async upload(records, options) {
+    return (await upload(this._client, this._type, records, options)).data;
+  }
+
+  uploadStream(options = {}) {
+    return new UploadStream(this._client, this._type, options);
+  }
+
+}
+
+export class Entities extends Writable {
+
+  constructor(client, type) {
+    super(client, type);
+  }
+
+  async get(id) {
+    const url = buildUrl(this._client, `${this._type}/${id}`);
+    return (await axios.get(url)).data.data;
+  }
+
+  async ids() {
+    const url = buildUrl(this._client, `${this._type}/_ids`);
+    return (await axios.get(url)).data.data.ids;
+  }
+
+  async delete(ids, options = {}) {
+    return batchDelete(this._client, this._type, ids, options);
+  }
+
+  deleteStream(options = {}) {
+    return new DeleteStream(this._client, this._type, options);
+  }
+
+}

package/src/api/experiments.js

@@ -0,0 +1,20 @@
+import axios from 'axios';
+import { buildUrl } from './helpers.js';
+
+export default class Experiments {
+
+  constructor(client) {
+    this._client = client;
+  }
+
+  async uploadEvent(experimentId, record) {
+    // TODO: support non-string record
+    const url = buildUrl(this._client, `experiments/${experimentId}/events`);
+    // TODO: make content type header global
+    const headers = { 'Content-Type': 'application/json' };
+    const response = await axios.post(url, record, { headers });
+    // 200 response body does not have .data layer
+    return response.data ? response : { data: response };
+  }
+
+}

package/src/api/helpers.js

@@ -0,0 +1,63 @@
+import { asArray } from '@miso.ai/server-commons';
+import axios from 'axios';
+import { Buffer } from 'buffer';
+
+export async function upload(client, type, records, options = {}) {
+  const url = buildUrl(client, type, options);
+  const payload = buildUploadPayload(records);
+  return axios.post(url, payload);
+}
+
+export async function batchDelete(client, type, ids, options = {}) {
+  const url = buildUrl(client, `${type}/_delete`, options);
+  const payload = buildBatchDeletePayload(type, ids);
+  // TODO: organize axios
+  const { data } = await axios.post(url, payload, {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  });
+  return data;
+}
+
+export function buildUrl(client, path, { async, dryRun, params: extraParams } = {}) {
+  let { server, key } = client._options;
+  let params = `?api_key=${key}`;
+  if (dryRun) {
+    params += '&dry_run=1';
+  } else if (async) {
+    params += '&async=1';
+  }
+  if (extraParams) {
+    for (const key in extraParams) {
+      // TODO: deal with encodeURIComponent
+      params += `&${key}=${extraParams[key]}`;
+    }
+  }
+  return `${server}/v1/${path}${params}`;
+}
+
+export function buildUploadPayload(records) {
+  return typeof records === 'string' ? records :
+    Buffer.isBuffer(records) ? records.toString() :
+    { data: Array.isArray(records)? records : [records] };
+}
+
+export function buildBatchDeletePayload(type, ids) {
+  if (type !== 'products' && type !== 'users' && type !== 'interactions') {
+    throw new Error(`Unsupported type: ${type}`);
+  }
+  if ((typeof ids === 'string' && ids[0] === '{') || typeof ids === 'object' && !Array.isArray(ids)) {
+    return ids;
+  }
+  ids = asArray(ids);
+  if (ids.length === 0) {
+    return { data: {} };
+  }
+  // interactions are deleted by user_ids
+  return {
+    data: {
+      [type === 'products' ? 'product_ids' : 'user_ids']: ids,
+    },
+  };
+}

package/src/api/index.js

@@ -0,0 +1,19 @@
+import Products from './products.js';
+import Users from './users.js';
+import Interactions from './interactions.js';
+import Experiments from './experiments.js';
+import Search from './search.js';
+import Recommendation from './recommendation.js';
+
+export default class Api {
+
+  constructor(client) {
+    this.products = new Products(client);
+    this.users = new Users(client);
+    this.interactions = new Interactions(client);
+    this.experiments = new Experiments(client);
+    this.search = new Search(client);
+    this.recommendation = new Recommendation(client);
+  }
+
+}

package/src/api/interactions.js

@@ -0,0 +1,11 @@
+import { Writable } from './base.js';
+
+export default class Interactions extends Writable {
+
+  constructor(client) {
+    super(client, 'interactions');
+  }
+
+  // TODO: delete (by user_ids)
+
+}

package/src/api/products.js

@@ -0,0 +1,9 @@
+import { Entities } from './base.js';
+
+export default class Products extends Entities {
+
+  constructor(client) {
+    super(client, 'products');
+  }
+
+}

package/src/api/recommendation.js

@@ -0,0 +1,33 @@
+import { Queries } from './base.js';
+
+export default class Recommendation extends Queries {
+
+  constructor(client) {
+    super(client, 'recommendation');
+  }
+
+  async userToProducts(payload, options) {
+    return this._run('user_to_products', payload, options);
+  }
+
+  async userToCategories(payload, options) {
+    return this._run('user_to_categories', payload, options);
+  }
+
+  async userToAttributes(payload, options) {
+    return this._run('user_to_attributes', payload, options);
+  }
+
+  async userToTrending(payload, options) {
+    return this._run('user_to_trending', payload, options);
+  }
+
+  async userToHistory(payload, options) {
+    return this._run('user_to_history', payload, options);
+  }
+
+  async productToProducts(payload, options) {
+    return this._run('product_to_products', payload, options);
+  }
+
+}

package/src/api/search.js

@@ -0,0 +1,21 @@
+import { Queries } from './base.js';
+
+export default class Search extends Queries {
+
+  constructor(client) {
+    super(client, 'search');
+  }
+
+  async search(payload, options = {}) {
+    return this._run('search', payload, options);
+  }
+
+  async autocomplete(payload, options = {}) {
+    return this._run('autocomplete', payload, options);
+  }
+
+  async multipleGet(payload, options = {}) {
+    return this._run('mget', payload, options);
+  }
+
+}

package/src/api/users.js

@@ -0,0 +1,9 @@
+import { Entities } from './base.js';
+
+export default class Users extends Entities {
+
+  constructor(client) {
+    super(client, 'users');
+  }
+
+}

package/src/client.js

@@ -3,9 +3,9 @@ import { asArray } from '@miso.ai/server-commons';
 import { Buffer } from 'buffer';
 import axios from 'axios';
 import version from './version.js';
-import LegacyUploadStream from './stream/upload.legacy.js';
 import UploadStream from './stream/upload.js';
 import DeleteStream from './stream/delete.js';
+import Api from './api/index.js';
 
 export default class MisoClient {
 
@@ -14,6 +14,7 @@ export default class MisoClient {
   constructor(options) {
     this._options = normalizeOptions(options);
     this.version = version;
+    this.api = new Api(this);
   }
 
   async upload(type, records, options) {
@@ -65,11 +66,6 @@ export default class MisoClient {
   }
 
   createUploadStream(type, options) {
-    if (options.legacy) {
-      options.heartbeat = options.heartbeatInvertal;
-      delete options.heartbeatInvertal;
-      return new LegacyUploadStream(this, type, options);
-    }
     return new UploadStream(this, type, options);
   }
 

package/src/index.js

@@ -1,2 +1,3 @@
 export { default as MisoClient } from './client.js';
+export * as normalize from './normalize/index.js';
 export * as logger from './logger/index.js';

package/src/logger/api-progress.js

@@ -24,22 +24,55 @@ export default class ApiProgressLogStream extends stream.LogUpdateStream {
 
   _sections({ config, state }) {
     return [
-      this._configTable(config),
+      this._configTableCached(config),
       this._statusTable(state),
       this._timeStatsTable(state),
       this._dataStatsTable(state),
     ];
   }
 
-  _configTable(config) {
-    const { name = '(anonymous)', client = {} } = config || {};
+  _configTableCached(config) {
+    return this._configTableString || (this._configTableString = this._configTable(config));
+  }
+
+  _configTable(config = {}) {
+    const { name, id, client = {}, experimentId } = config;
+    const rows = [];
+    if (experimentId) {
+      rows.push(['Exp. ID:', experimentId]);
+    }
+    const props = this._configProps(config);
+    if (props.length) {
+      rows.push(['Config:', props.join(', ')]);
+    }
     return formatTable([
-      ['Job:', `${name}`],
+      ['Job:', `${name || id}`],
+      ...rows,
       ['Server:', `${client.server || '(default)'}`],
       ['API Key:', `${client.keyMasked}`],
     ]);
   }
 
+  _configProps(config = {}) {
+    const { sink = {}, extra = {} } = config;
+    const { dryRun, async, params } = sink;
+    const { lenient } = extra;
+    const props = [];
+    if (dryRun) {
+      props.push('dry-run');
+    }
+    if (async) {
+      props.push('async');
+    }
+    if (lenient) {
+      props.push('lenient');
+    }
+    if (params && Object.keys(params).length) {
+      props.push(`params = ${JSON.stringify(params)}`);
+    }
+    return props;
+  }
+
   _statusTable(state) {
     const { bps, stats } = state;
     const serviceBps = stats && stats.service && stats.service.bps;

package/src/logger/index.js

@@ -1,6 +1,5 @@
 import { FORMAT } from './constants.js';
 import StandardLogStream from './standard.js';
-import LegacyProgressLogStream from './progress.legacy.js';
 import ApiProgressLogStream from './api-progress.js';
 import DeleteProgressLogStream from './delete-progress.js';
 
@@ -8,7 +7,6 @@ export * from './constants.js';
 
 export function createLogStream({
   api,
-  legacy,
   level,
   format,
   out,
@@ -18,9 +16,6 @@ export function createLogStream({
     case FORMAT.PROGRESS:
       switch (api) {
         case 'upload':
-          if (legacy) {
-            return new LegacyProgressLogStream({ out, err });
-          }
           return new ApiProgressLogStream({ out, err });
         case 'delete':
           return new DeleteProgressLogStream({ out, err });

package/src/normalize/index.js

@@ -0,0 +1,74 @@
+import { Transform } from 'stream';
+import { trimObj } from '@miso.ai/server-commons';
+import * as shim from './shim.js';
+
+export function products({
+  cover_image,
+  url,
+  ...record
+}) {
+  // category -> assume single, categories -> assume multiple
+  return trimObj({
+    cover_image: shim.url(cover_image),
+    url: shim.url(url),
+    ...record,
+  });
+}
+
+export function users({
+  ...record
+}) {
+  return trimObj({
+    ...record,
+  });
+}
+
+export function interactions({
+  ...record
+}) {
+  return trimObj({
+    ...record,
+  });
+}
+
+export function experimentEvents({
+  ...record
+}) {
+  return trimObj({
+    ...record,
+  });
+}
+
+function getTransformFunction(type) {
+  switch (type) {
+    case 'products':
+      return products;
+    case 'users':
+      return users;
+    case 'interactions':
+      return interactions;
+    case 'experiment-events':
+      return experimentEvents;
+    default:
+      throw new Error(`Unrecognized record type: ${type}`);
+  }
+}
+
+export class Stream extends Transform {
+
+  constructor(type) {
+    super({
+      objectMode: true,
+    });
+    this._transformFn = getTransformFunction(type);
+  }
+
+  async _transform(record, _, next) {
+    try {
+      next(undefined, this._transformFn(record));
+    } catch (error) {
+      next(error);
+    }
+  }
+
+}

package/src/normalize/shim.js

@@ -0,0 +1,72 @@
+import { asArray } from '@miso.ai/server-commons';
+
+export const number = _applyGeneralPass(_number);
+export const string = _applyGeneralPass(_string);
+export const array = _applyGeneralPass(_array);
+export const doubleArray = _applyGeneralPass(_doubleArray);
+export const time = _applyGeneralPass(_time);
+export const url = _applyGeneralPass(_url);
+
+function _number(value) {
+  value = Number(value);
+  if (isNaN(value)) {
+    throw new Error();
+  }
+  return value;
+}
+
+function _string(value) {
+  // TODO
+  return value;
+}
+
+function _array(value) {
+  value = asArray(value).filter(isNotBlank);
+  return value.length ? value : undefined;
+}
+
+function _doubleArray(value) {
+  // TODO
+  value;
+}
+
+function _time(value) {
+  // TODO: how to handle timezone
+  return value;
+}
+
+function _url(value) {
+  // blank safe
+  if (isBlank(value) || typeof value !== 'string') {
+    return undefined;
+  }
+  // a wild guess on whether the URL has been encoded
+  if (!value.includes('%')) {
+    value = encodeURI(value);
+  }
+  // bounce on invalid URL
+  new URL(value);
+
+  return value;
+}
+
+function isNotBlank(value) {
+  return !isBlank(value);
+}
+
+function isBlank(value) {
+  return value === undefined || value === null || value === '';
+}
+
+function _applyGeneralPass(fn) {
+  return value => {
+    if (isBlank(value)) {
+      return undefined;
+    }
+    try {
+      return fn(value);
+    } catch(_) {
+      return value;
+    }
+  };
+}

package/src/stream/api-sink.js

@@ -46,7 +46,7 @@ export default class ApiSink extends sink.BpsSink {
         throw error;
       }
       data = error.response.data;
-      if (typeof data!== 'object') {
+      if (typeof data !== 'object') {
         data = trimObj({ errors: true, cause: data });
       }
     }

package/src/stream/delete-sink.js

@@ -1,4 +1,5 @@
 import ApiSink from './api-sink.js';
+import { batchDelete } from '../api/helpers.js';
 
 export default class DeleteSink extends ApiSink {
 
@@ -19,8 +20,7 @@ export default class DeleteSink extends ApiSink {
 
   async _execute(payload) {
     const { type, params } = this._options;
-    const response = await this._client._delete(type, payload, { params });
-    return response.data;
+    return batchDelete(this._client, type, payload, { params });
   }
 
 }

package/src/stream/delete.js

@@ -1,7 +1,7 @@
 import { stream, buffer } from '@miso.ai/server-commons';
 import version from '../version.js';
 import DeleteSink from './delete-sink.js';
-import DeletionStats from './deletion-state.js';
+import DeletionStats from './deletion-stats.js';
 
 export default class DeleteStream extends stream.BufferedWriteStream {
 

package/src/stream/upload-sink.js

@@ -1,4 +1,5 @@
 import ApiSink from './api-sink.js';
+import { upload } from '../api/helpers.js';
 
 class UploadSink extends ApiSink {
 
@@ -23,7 +24,7 @@ class UploadSink extends ApiSink {
 
   async _execute(payload) {
     const { type, async, dryRun, params } = this._options;
-    const response = await this._client.upload(type, payload, { async, dryRun, params });
+    const response = await upload(this._client, type, payload, { async, dryRun, params });
     return response.data;
   }
 
@@ -84,7 +85,7 @@ class ExperimentEventUploadSink extends UploadSink {
 
   async _execute(payload) {
     const { experimentId } = this._options;
-    const response = await this._client.uploadExperimentEvent(experimentId, payload);
+    const response = await this._client.api.experiments.uploadEvent(experimentId, payload);
     return response.data;
   }
 

package/src/stream/upload.js

@@ -20,6 +20,7 @@ export default class UploadStream extends stream.BufferedWriteStream {
     // buffer
     recordsPerRequest,
     bytesPerRequest,
+    extra,
   } = {}) {
     super({
       name,
@@ -27,6 +28,7 @@ export default class UploadStream extends stream.BufferedWriteStream {
       version,
       objectMode,
       heartbeatInterval,
+      extra,
     });
 
     this._client = client;

package/src/version.js

@@ -1 +1 @@
-export default '0.6.0-beta.1';
+export default '0.6.0';

package/src/logger/progress.legacy.js

@@ -1,60 +0,0 @@
-import { log, stream } from '@miso.ai/server-commons';
-
-const { formatDuration, formatBytes, formatTable } = log;
-
-export default class LegacyProgressLogStream extends stream.LogUpdateStream {
-
-  constructor({
-    out = process.stdout,
-    err = process.stderr,
-  } = {}) {
-    super({
-      out,
-      err,
-    });
-  }
-
-  _renderError({ state: _, ...record }) {
-    return super._renderError(record);
-  }
-
-  _render({ state }) {
-    const { elapsed, pending, successful, failed, apiBps, sentBps } = state;
-
-    // sum pending requests
-    const pendingSums = sumMetrics(pending);
-
-    const table = formatTable([
-      ['', 'Requests', 'Records', 'Bytes', 'Server Time', 'Latency'],
-      ['Pending', pendingSums.requests, pendingSums.records, formatBytes(pendingSums.bytes), '--', '--'],
-      ['Successful', successful.requests, successful.records, formatBytes(successful.bytes), formatDuration(successful.took), formatDuration(successful.latency)],
-      ['Failed', failed.requests, failed.records, formatBytes(failed.bytes), formatDuration(failed.took), formatDuration(failed.latency)],
-    ]);
-
-    return `
-Time elapsed: ${formatDuration(elapsed)}
-API Speed: ${formatSpeed(apiBps)}
-Client Speed: ${formatSpeed(sentBps)}
-
-${table}
-`;
-  }
-
-}
-
-function sumMetrics(requests) {
-  return requests.reduce((acc, req) => {
-    acc.requests ++;
-    acc.records += req.records;
-    acc.bytes += req.bytes;
-    return acc;
-  }, {
-    requests: 0,
-    records: 0,
-    bytes: 0,
-  });
-}
-
-function formatSpeed(value) {
-  return isNaN(value) ? 'N/A' : `${formatBytes(value)}/s`;
-}

package/src/stream/upload.legacy.js

@@ -1,410 +0,0 @@
-import { Transform } from 'stream';
-import { trimObj, log } from '@miso.ai/server-commons';
-import version from '../version.js';
-
-function getDefaultRecordsPerRequest(type) {
-  switch (type) {
-    case 'interactions':
-      return 1000;
-    default:
-      return 200;
-  }
-}
-
-function normalizeParams(params = []) {
-  return params.reduce((acc, param) => {
-    const [key, value = '1'] = param.split('=');
-    acc[key] = value;
-    return acc;
-  }, {});
-}
-
-const PAYLOAD_PREFIX = '{"data":[';
-const PAYLOAD_SUFFIX = ']}';
-const PAYLOAD_OVERHEAD_BYTES = (PAYLOAD_PREFIX.length + PAYLOAD_SUFFIX.length) * 2;
-
-const MIN_HREATBEAT = 100;
-
-const requestPromises = new WeakMap();
-
-export default class LegacyUploadStream extends Transform {
-
-  constructor(client, type, {
-    name,
-    objectMode,
-    async, 
-    dryRun,
-    params,
-    heartbeat,
-    recordsPerRequest,
-    bytesPerRequest,
-    bytesPerSecond,
-    experimentId,
-  } = {}) {
-    super({
-      readableObjectMode: true,
-      writableObjectMode: objectMode,
-    });
-    this._client = client;
-    this._type = type;
-    this._options = {
-      name,
-      objectMode: !!objectMode,
-      async: !!async, 
-      dryRun: !!dryRun,
-      params: normalizeParams(params),
-      heartbeat,
-      recordsPerRequest: recordsPerRequest || getDefaultRecordsPerRequest(type),
-      bytesPerRequest: bytesPerRequest || 1024 * 1024,
-      bytesPerSecond: bytesPerSecond || 4 * 1024 * 1024,
-      experimentId,
-    };
-    if (heartbeat !== undefined && (typeof heartbeat !== 'number') || heartbeat < MIN_HREATBEAT) {
-      throw new Error(`Heartbeat must be a number at least ${MIN_HREATBEAT}: ${heartbeat}`);
-    }
-    this._singleRecordPerRequest = false;
-
-    switch (type) {
-      case 'experiment-events':
-        if (!experimentId) {
-          throw new Error(`Experiment id is required for experiment APIs`);
-        }
-        this._singleRecordPerRequest = true;
-    }
-
-    this._state = new State();
-    this._resetBuffer();
-    // log functions
-    for (const level of log.LEVELS) {
-      this[`_${level}`] = this._log.bind(this, level);
-    }
-  }
-
-  _construct(done) {
-    const { config } = this;
-    this._info('construct', { config });
-    const heartbeat = this._options.heartbeat;
-    if (heartbeat) {
-      this._heartbeatIntervalId = setInterval(this._heartbeat.bind(this), heartbeat);
-    }
-    done();
-  }
-
-  _transform(record, _, next) {
-    this._pushStartEventIfNecessary();
-    const { objectMode, bytesPerRequest } = this._options;
-    const str = objectMode ? JSON.stringify(record) : record;
-    const newSize = str.length * 2;
-
-    if (this._singleRecordPerRequest) {
-      this._buffer.content = str;
-      this._buffer.bytes = newSize;
-      this._buffer.records = 1;
-      this._dispatch();
-
-    } else {
-      if (this._buffer.records && this._buffer.bytes + newSize >= bytesPerRequest) {
-        // flush previous records if this record is large enough to exceed BPR threshold
-        this._dispatch();
-      }
-      if (this._buffer.records > 0) {
-        this._buffer.content += ',';
-      }
-      this._buffer.content += str;
-      this._buffer.bytes += newSize;
-      this._buffer.records++;
-  
-      this._dispatchIfNecessary();
-    }
-
-    const restTime = this._state.restTime(this._getBpsLimit());
-    if (restTime > 0) {
-      this._debug('rest', { restTime });
-      setTimeout(next, restTime);
-    } else if (this._state._pending.length > 15) {
-      // TODO: figure out best strategy on this
-      // release event loop for downstream
-      setTimeout(next);
-    } else {
-      next();
-    }
-  }
-
-  _log(level, event, args = {}) {
-    this.push(trimObj({
-      name: this._options.name,
-      level,
-      event,
-      timestamp: Date.now(),
-      ...args,
-      state: this.state,
-    }));
-  }
-
-  _heartbeat() {
-    this._log(log.DEBUG, 'heartbeat');
-  }
-
-  async _flush(done) {
-    this._pushStartEventIfNecessary();
-    this._dispatch();
-    await Promise.all(this._state.pending.map(r => requestPromises.get(r)));
-    const { successful, failed } = this.state;
-
-    if (this._heartbeatIntervalId) {
-      clearInterval(this._heartbeatIntervalId);
-      delete this._heartbeatIntervalId;
-    }
-
-    this._info('end', { successful, failed });
-    done();
-  }
-
-  get state() {
-    return this._state.export();
-  }
-
-  get config() {
-    return Object.freeze({
-      version,
-      type: this._type,
-      ...this._client.options,
-      ...this._options,
-    });
-  }
-
-  // helper //
-  _pushStartEventIfNecessary() {
-    if (this._state.next.request === 0 && this._buffer.records === 0) {
-      this._info('start');
-    }
-  }
-
-  _dispatchIfNecessary() {
-    const { records, bytes } = this._buffer;
-    const { recordsPerRequest, bytesPerRequest } = this._options;
-    if (records > 0 && (records >= recordsPerRequest || bytes >= bytesPerRequest)) {
-      this._dispatch();
-    }
-  }
-
-  _dispatch() {
-    const singleRecord = this._singleRecordPerRequest;
-    const { records, bytes, content } = this._resetBuffer();
-    if (records === 0) {
-      return;
-    }
-    const request = this._state.createRequest(records, bytes);
-
-    let requestResolve;
-    requestPromises.set(request, new Promise(r => {
-      requestResolve = r;
-    }));
-
-    this._debug('request', { request });
-
-    this._state.open(request);
-
-    const payload = singleRecord ? content : content + PAYLOAD_SUFFIX;
-
-    (async () => {
-      let response;
-      try {
-        response = await this._upload(payload);
-      } catch(error) {
-        response = !error.response ? trimObj({ errors: true, cause: error.message }) :
-          typeof error.response.data !== 'object' ? trimObj({ errors: true, cause: error.response.data }) :
-          error.response.data;
-      }
-      response.timestamp = Date.now();
-      response.took = response.took || 0; // TODO: ad-hoc
-
-      requestResolve();
-      this._state.close(request, response);
-
-      const failed = response.errors;
-
-      (failed ? this._error : this._debug)('response', { request, response, payload: failed ? JSON.parse(payload) : undefined });
-      this._info('upload', {
-        result: failed ? 'failed' : 'successful',
-        index: request.index,
-        records: request.records,
-        bytes: request.bytes,
-        took: response.took,
-        latency: response.timestamp - request.timestamp - response.took
-      });
-    })();
-  }
-
-  async _upload(payload) {
-    switch (this._type) {
-      case 'experiment-events':
-        const { experimentId } = this._options;
-        return (await this._client.uploadExperimentEvent(experimentId, payload)).data;
-      default:
-        const { async, dryRun, params } = this._options;
-        const response = await this._client.upload(this._type, payload, { async, dryRun, params });
-        return response.data;
-    }
-  }
-
-  _resetBuffer() {
-    const buffer = { ...this._buffer };
-    this._buffer = {
-      records: 0,
-      bytes: PAYLOAD_OVERHEAD_BYTES,
-      content: PAYLOAD_PREFIX,
-    };
-    return buffer;
-  }
-
-  /**
-   * Note that when API is effectively in async mode, apiBps will be overestimated, but there is no harm to respect it.
-   */
-  _getBpsLimit() {
-    const { bytesPerSecond } = this._options;
-    const { pending, apiBps, completed } = this.state;
-    // TODO: threshold to switch to real API BPS should be based on bytes, not requests
-    // respect API BPS only when
-    // 1. has pending requests
-    // 2. has enouch data points from completed requests
-    return pending.length > 0 && completed.requests > 9 && !isNaN(apiBps) && apiBps < bytesPerSecond ? apiBps : bytesPerSecond;
-  }
-
-}
-
-class State {
-
-  constructor() {
-    this._start = Date.now();
-    this._next = Object.freeze({
-      request: 0,
-      record: 0,
-    });
-    this._pending = [];
-    this._successful = {
-      requests: 0,
-      records: 0,
-      bytes: 0,
-      took: 0,
-      latency: 0,
-    };
-    this._failed = {
-      requests: 0,
-      records: 0,
-      bytes: 0,
-      took: 0,
-      latency: 0,
-    };
-  }
-
-  get start() {
-    return this._start;
-  }
-
-  get next() {
-    return this._next;
-  }
-
-  get pending() {
-    return Object.freeze(this._pending.slice());
-  }
-
-  get successful() {
-    return Object.freeze({ ...this._successful });
-  }
-
-  get failed() {
-    return Object.freeze({ ...this._failed });
-  }
-
-  elapsed(time) {
-    return (time || Date.now()) - this.start;
-  }
-
-  createRequest(records, bytes) {
-    const { next } = this;
-    const request = Object.freeze({
-      index: next.request,
-      recordOffset: next.record,
-      records,
-      bytes,
-      timestamp: Date.now(),
-    });
-    this._next = Object.freeze({
-      request: next.request + 1,
-      record: next.record + records,
-    });
-    return request;
-  }
-
-  get sent() {
-    const { _pending, completed } = this;
-    return Object.freeze(_pending.reduce((acc, request) => {
-      acc.requests++;
-      acc.records += request.records;
-      acc.bytes += request.bytes;
-      return acc;
-    }, {
-      requests: completed.requests,
-      records: completed.records,
-      bytes: completed.bytes,
-    }));
-  }
-
-  get completed() {
-    const { _successful, _failed } = this;
-    return Object.freeze({
-      requests: _successful.requests + _failed.requests,
-      records: _successful.records + _failed.records,
-      bytes: _successful.bytes + _failed.bytes,
-      took: _successful.took + _failed.took,
-      latency: _successful.latency + _failed.latency,
-    });
-  }
-
-  sentBps(timestamp) {
-    const { sent } = this;
-    const elapsed = this.elapsed(timestamp);
-    return elapsed > 0 ? sent.bytes / elapsed * 1000 : NaN;
-  }
-
-  get apiBps() {
-    const { _successful, _failed } = this;
-    const took = _successful.took + _failed.took;
-    const bytes = _successful.bytes + _failed.bytes;
-    return took > 0 ? bytes / took * 1000 : NaN;
-  }
-
-  restTime(bps) {
-    const elapsed = this.elapsed();
-    const { sent } = this;
-    return Math.max(0, sent.bytes / bps * 1000 - elapsed) * 1.05;
-  }
-
-  export(timestamp) {
-    const { next, pending, successful, failed, sent, completed, apiBps } = this;
-    timestamp = timestamp || Date.now();
-    return Object.freeze({
-      next, pending, sent, successful, failed, completed,
-      elapsed: this.elapsed(timestamp),
-      apiBps,
-      sentBps: this.sentBps(timestamp),
-    });
-  }
-
-  open(request) {
-    this._pending.push(request);
-  }
-
-  close(request, response) {
-    this._pending = this._pending.filter(r => r.index !== request.index);
-    const category = response.errors ? this._failed : this._successful;
-    category.requests++;
-    category.records += request.records;
-    category.bytes += request.bytes;
-    category.took += response.took;
-    category.latency += response.timestamp - request.timestamp - response.took;
-  }
-
-}