@miso.ai/server-sdk 0.6.0-beta.0

package/cli/delete.js

@@ -0,0 +1,85 @@
+import split2 from 'split2';
+import { log, stream } from '@miso.ai/server-commons';
+import { MisoClient, logger } from '../src/index.js';
+
+function build(yargs) {
+  return yargs
+    .option('records-per-request', {
+      alias: ['rpr'],
+      describe: 'How many records to send in a request',
+    })
+    .option('records-per-second', {
+      alias: ['rps'],
+      describe: 'How many records to send per second',
+    })
+    .option('debug', {
+      describe: 'Set log level to debug',
+      type: 'boolean',
+    })
+    .option('progress', {
+      alias: ['p'],
+      describe: 'Set log format progress',
+      type: 'boolean',
+    })
+    .option('stream-name', {
+      alias: ['name'],
+      describe: 'Stream name that shows up in log messages',
+    })
+    .option('log-level', {
+      describe: 'Log level',
+    })
+    .option('log-format', {
+      describe: 'Log format',
+    });
+}
+
+const run = type => async ({
+  key,
+  server,
+  param: params,
+  ['records-per-request']: recordsPerRequest,
+  ['records-per-second']: recordsPerSecond,
+  debug,
+  progress,
+  ['stream-name']: name,
+  ['log-level']: loglevel,
+  ['log-format']: logFormat,
+}) => {
+
+  loglevel = (debug || progress) ? log.DEBUG : loglevel;
+  logFormat = progress ? logger.FORMAT.PROGRESS : logFormat;
+
+  const client = new MisoClient({ key, server });
+
+  const deleteStream = client.createDeleteStream(type, {
+    name,
+    params,
+    heartbeatInterval: logFormat === logger.FORMAT.PROGRESS ? 250 : false,
+    recordsPerRequest,
+    recordsPerSecond,
+  });
+
+  const logStream = logger.createLogStream({
+    api: 'delete',
+    type,
+    level: loglevel,
+    format: logFormat,
+  });
+
+  await stream.pipeline(
+    process.stdin,
+    split2(),
+    deleteStream,
+    logStream,
+  );
+};
+
+export default function(type) {
+  return {
+    command: 'delete',
+    aliases: ['d'],
+    description: `Delete ${type}`,
+    builder: build,
+    handler: run(type),
+  };
+}

package/cli/ids.js

@@ -0,0 +1,32 @@
+import { Readable } from 'stream';
+import { stream } from '@miso.ai/server-commons';
+import { MisoClient } from '../src/index.js';
+
+function build(yargs) {
+  return yargs;
+}
+
+const run = type => async ({
+  key,
+  server,
+}) => {
+  const client = new MisoClient({ key, server });
+  const ids = await client.ids(type);
+
+  const readStream = Readable.from(ids);
+  const outputStream = new stream.OutputStream();
+
+  await stream.pipeline(
+    readStream,
+    outputStream,
+  );
+};
+
+export default function(type) {
+  return {
+    command: 'ids',
+    description: false,
+    builder: build,
+    handler: run(type),
+  };
+}

package/cli/index.js

@@ -0,0 +1,102 @@
+#!/usr/bin/env node
+import 'dotenv/config';
+import yargs from 'yargs/yargs';
+import { hideBin } from 'yargs/helpers';
+import upload from './upload.js';
+import _delete from './delete.js';
+import ids from './ids.js';
+import version from '../src/version.js';
+
+const interactions = {
+  command: 'interactions',
+  aliases: ['interaction', 'i'],
+  description: 'Interaction commands',
+  builder: yargs => _buildBase(yargs)
+    .command(upload('interactions')),
+};
+
+const products = {
+  command: 'products',
+  aliases: ['product', 'p'],
+  description: 'Product commands',
+  builder: yargs => _buildBase(yargs)
+    .command(upload('products'))
+    .command(_delete('products'))
+    .command(ids('products')),
+};
+
+const users = {
+  command: 'users',
+  aliases: ['user', 'u'],
+  description: 'User commands',
+  builder: yargs => _buildBase(yargs)
+    .command(upload('users'))
+    .command(_delete('users'))
+    .command(ids('users')),
+};
+
+const experiments = {
+  command: 'experiments',
+  aliases: ['experiment'],
+  description: 'Experiment commands',
+  builder: yargs => _buildBase(yargs)
+    .option('experiment-id', {
+      alias: ['exp-id'],
+      describe: 'Experiment ID for experiment API',
+    })
+    .command({
+      command: 'events',
+      builder: yargs => yargs
+        .command(upload('experiment-events')),
+    }),
+};
+
+yargs(hideBin(process.argv))
+  .env('MISO')
+  .command(interactions)
+  .command(products)
+  .command(users)
+  .command(experiments)
+  .demandCommand(2)
+  .version(version)
+  .help()
+  .fail(_handleFail)
+  .parse();
+
+
+
+// helpers //
+function _buildBase(yargs) {
+  return yargs
+    .option('key', {
+      alias: ['k', 'api-key'],
+      describe: 'API key',
+    })
+    .option('server', {
+      alias: ['api-server'],
+      describe: 'API server',
+    })
+    .option('param', {
+      alias: ['v', 'var'],
+      describe: 'Extra URL parameters',
+      type: 'array',
+      coerce: _coerceToArray,
+    })
+    .demandOption(['key'], 'API key is required.');
+}
+
+function _coerceToArray(arg) {
+  return Array.isArray(arg) ? arg :
+    typeof arg === 'string' ? arg.split(',') :
+    arg === undefined || arg === null ? [] : [arg];
+}
+
+function _handleFail(msg, err) {
+  if (err) {
+    throw err;
+  }
+  console.error(msg);
+  process.exit(1);
+}
+
+process.stdout.on('error', err => err.code == 'EPIPE' && process.exit(0));

package/cli/upload.js

@@ -0,0 +1,114 @@
+import split2 from 'split2';
+import { log, stream } from '@miso.ai/server-commons';
+import { MisoClient, logger } from '../src/index.js';
+
+function build(yargs) {
+  return yargs
+    .option('async', {
+      alias: ['a'],
+      describe: 'Asynchrnous mode',
+    })
+    .option('dry-run', {
+      alias: ['dry'],
+      describe: 'Dry run mode',
+    })
+    .option('records-per-request', {
+      alias: ['rpr'],
+      describe: 'How many records to send in a request',
+    })
+    .option('bytes-per-request', {
+      alias: ['bpr'],
+      describe: 'How many bytes to send in a request',
+    })
+    .option('bytes-per-second', {
+      alias: ['bps'],
+      describe: 'How many bytes to send per second',
+    })
+    .option('debug', {
+      describe: 'Set log level to debug',
+      type: 'boolean',
+    })
+    .option('progress', {
+      alias: ['p'],
+      describe: 'Set log format progress',
+      type: 'boolean',
+    })
+    .option('stream-name', {
+      alias: ['name'],
+      describe: 'Stream name that shows up in log messages',
+    })
+    .option('legacy', {
+      type: 'boolean',
+      default: false,
+    })
+    .hide('legacy')
+    .option('log-level', {
+      describe: 'Log level',
+    })
+    .option('log-format', {
+      describe: 'Log format',
+    });
+}
+
+const run = type => async ({
+  key,
+  server,
+  param: params,
+  async,
+  ['dry-run']: dryRun,
+  ['records-per-request']: recordsPerRequest,
+  ['bytes-per-request']: bytesPerRequest,
+  ['bytes-per-second']: bytesPerSecond,
+  ['experiment-id']: experimentId,
+  debug,
+  progress,
+  ['stream-name']: name,
+  legacy,
+  ['log-level']: loglevel,
+  ['log-format']: logFormat,
+}) => {
+
+  loglevel = (debug || progress) ? log.DEBUG : loglevel;
+  logFormat = progress ? logger.FORMAT.PROGRESS : logFormat;
+
+  const client = new MisoClient({ key, server });
+
+  const uploadStream = client.createUploadStream(type, {
+    legacy,
+    name,
+    async, 
+    dryRun,
+    params,
+    heartbeatInterval: logFormat === logger.FORMAT.PROGRESS ? 250 : false,
+    //heartbeat: logFormat === logger.FORMAT.PROGRESS ? 250 : undefined,
+    recordsPerRequest,
+    bytesPerRequest,
+    bytesPerSecond,
+    experimentId,
+  });
+
+  const logStream = logger.createLogStream({
+    api: 'upload',
+    type,
+    legacy,
+    level: loglevel,
+    format: logFormat,
+  });
+
+  await stream.pipeline(
+    process.stdin,
+    split2(),
+    uploadStream,
+    logStream,
+  );
+};
+
+export default function(type) {
+  return {
+    command: 'upload',
+    aliases: ['u'],
+    description: `Upload ${type}`,
+    builder: build,
+    handler: run(type),
+  };
+}

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@miso.ai/server-sdk",
+  "description": "Miso SDK for Node.js",
+  "type": "module",
+  "main": "src/index.js",
+  "bin": {
+    "miso": "cli/index.js"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {},
+  "repository": "MisoAI/miso-server-js-sdk",
+  "license": "MIT",
+  "contributors": [
+    "simonpai <simon.pai@askmiso.com>"
+  ],
+  "dependencies": {
+    "@miso.ai/server-commons": "0.6.0-beta.0",
+    "axios": "^0.27.2",
+    "dotenv": "^16.0.1",
+    "split2": "^4.1.0",
+    "yargs": "^17.5.1"
+  },
+  "version": "0.6.0-beta.0"
+}

package/src/client.js

@@ -0,0 +1,124 @@
+import { asArray } from '@miso.ai/server-commons';
+//import { createHash } from 'crypto';
+import { Buffer } from 'buffer';
+import axios from 'axios';
+import version from './version.js';
+import LegacyUploadStream from './stream/upload.legacy.js';
+import UploadStream from './stream/upload.js';
+import DeleteStream from './stream/delete.js';
+
+export default class MisoClient {
+
+  static version = version;
+
+  constructor(options) {
+    this._options = normalizeOptions(options);
+    this.version = version;
+  }
+
+  async upload(type, records, options) {
+    const url = buildUrl(this, type, options);
+    const payload = buildPayload(records);
+    return await axios.post(url, payload);
+  }
+
+  // TODO: extract to .experiment() later
+  async uploadExperimentEvent(experimentId, record) {
+    // TODO: support non-string record
+    const url = buildUrl(this, `experiments/${experimentId}/events`);
+    // TODO: make content type header global
+    const headers = { 'Content-Type': 'application/json' };
+    const response = await axios.post(url, record, { headers });
+    // 200 response body does not have .data layer
+    return response.data ? response : { data: response };
+  }
+
+  async ids(type) {
+    const url = buildUrl(this, `${type}/_ids`);
+    return (await axios.get(url)).data.data.ids;
+  }
+
+  async delete(type, ids) {
+    if (type !== 'products' && type !== 'users') {
+      throw new Error(`Only products and users are supported: ${type}`);
+    }
+    ids = asArray(ids);
+    if (ids.length === 0) {
+      return { data: {} };
+    }
+    const payload = {
+      data: {
+        [type === 'products' ? 'product_ids' : 'user_ids']: ids,
+      },
+    };
+    return this._delete(type, payload);
+  }
+
+  async _delete(type, payload, options) {
+    const url = buildUrl(this, `${type}/_delete`, options);
+    const { data } = await axios.post(url, payload, {
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+    return { data };
+  }
+
+  createUploadStream(type, options) {
+    if (options.legacy) {
+      options.heartbeat = options.heartbeatInvertal;
+      delete options.heartbeatInvertal;
+      return new LegacyUploadStream(this, type, options);
+    }
+    return new UploadStream(this, type, options);
+  }
+
+  createDeleteStream(type, options) {
+    return new DeleteStream(this, type, options);
+  }
+
+  get options() {
+    const { server, key } = this._options;
+    return Object.freeze({
+      server,
+      keyMasked: key.substring(0, 4) + '*'.repeat(Math.max(0, key.length - 4)),
+      //keyMd5: createHash('md5').update(key).digest('hex'),
+    });
+  }
+
+}
+
+function normalizeOptions(options) {
+  if (typeof options === 'string') {
+    options = { key: options };
+  }
+  if (!options.key || typeof options.key !== 'string') {
+    throw new Error(`API key is required.`);
+  }
+  options.server = options.server || 'https://api.askmiso.com'
+
+  return options;
+}
+
+function buildUrl(client, path, { async, dryRun, params: extraParams } = {}) {
+  let { server, key } = client._options;
+  let params = `?api_key=${key}`;
+  if (dryRun) {
+    params += '&dry_run=1';
+  } else if (async) {
+    params += '&async=1';
+  }
+  if (extraParams) {
+    for (const key in extraParams) {
+      // TODO: deal with encodeURIComponent
+      params += `&${key}=${extraParams[key]}`;
+    }
+  }
+  return `${server}/v1/${path}${params}`;
+}
+
+function buildPayload(records) {
+  return typeof records === 'string' ? records :
+    Buffer.isBuffer(records) ? records.toString() :
+    { data: Array.isArray(records)? records : [records] };
+}

package/src/index.js

@@ -0,0 +1,2 @@
+export { default as MisoClient } from './client.js';
+export * as logger from './logger/index.js';

package/src/logger/api-progress.js

@@ -0,0 +1,106 @@
+import { log, stream } from '@miso.ai/server-commons';
+
+const { formatDuration, formatBytes, formatTable } = log;
+
+export default class ApiProgressLogStream extends stream.LogUpdateStream {
+
+  _renderError({ state: _, ...record }) {
+    return super._renderError(record);
+  }
+
+  _render(args) {
+    args = this._normalizeInput(args);
+    return this._sections(args)
+      .map(s => `\n${s}\n`)
+      .join('');
+  }
+
+  _normalizeInput({ config, ...args } = {}) {
+    return {
+      config: (this._config = (config || this._config)),
+      ...args,
+    };
+  }
+
+  _sections({ config, state }) {
+    return [
+      this._configTable(config),
+      this._statusTable(state),
+      this._timeStatsTable(state),
+      this._dataStatsTable(state),
+    ];
+  }
+
+  _configTable(config) {
+    const { name = '(anonymous)', client = {} } = config || {};
+    return formatTable([
+      ['Job:', `${name}`],
+      ['Server:', `${client.server || '(default)'}`],
+      ['API Key:', `${client.keyMasked}`],
+    ]);
+  }
+
+  _statusTable(state) {
+    const { bps, stats } = state;
+    const serviceBps = stats && stats.service && stats.service.bps;
+    return formatTable([
+      ['Status:', `${this._statusLine(state)}`],
+      ['Service Speed:', this._formatBps(serviceBps)],
+      ['Client Speed:', this._formatBps(bps)],
+    ]);
+  }
+
+  _statusLine({ status, time }) {
+    switch (status) {
+      case 'paused':
+        const { currentTime, willResumeAt } = time;
+        const willResumeIn = Math.max(0, willResumeAt - currentTime);
+        return `${status.toUpperCase()} (resume in ${formatDuration(willResumeIn)})`;
+      default:
+        return status.toUpperCase();
+    }
+  }
+
+  _timeStatsTable({ time }) {
+    const { total, waiting, running, paused } = time;
+    return formatTable([
+      ['Total Time', 'Preparing', 'Running', 'Paused'],
+      [formatDuration(total), formatDuration(waiting), formatDuration(running), formatDuration(paused)],
+    ]);
+  }
+
+  _dataStatsTable({ pending, successful, failed }) {
+    const pendingSums = this._sumMetrics(pending);
+
+    return formatTable([
+      ['', 'Requests', 'Records', 'Bytes'],
+      ['Pending', pendingSums.requests, pendingSums.records, formatBytes(pendingSums.bytes)],
+      ['Successful', successful.requests, successful.records, formatBytes(successful.bytes)],
+      ['Failed', failed.requests, failed.records, formatBytes(failed.bytes)],
+    ]);
+  }
+
+  // helper //
+  _sumMetrics(requests) {
+    return requests.reduce((acc, req) => {
+      acc.requests ++;
+      acc.records += req.records;
+      acc.bytes += req.bytes;
+      return acc;
+    }, {
+      requests: 0,
+      records: 0,
+      bytes: 0,
+    });
+  }
+
+  // TODO: move to log utils
+  _formatBps(value) {
+    return isNaN(value) ? '--' : `${formatBytes(value)}/s`;
+  }
+  
+  _formatRps(value) {
+    return isNaN(value) ? '--' : `${value} records/s`;
+  }
+  
+}

package/src/logger/constants.js

@@ -0,0 +1,5 @@
+export const FORMAT = {
+  TEXT: 'text',
+  JSON: 'json',
+  PROGRESS: 'progress',
+};

package/src/logger/delete-progress.js

@@ -0,0 +1,22 @@
+import { log } from '@miso.ai/server-commons';
+import ApiProgressLogStream from './api-progress.js';
+
+const { formatTable } = log;
+
+export default class DeleteProgressLogStream extends ApiProgressLogStream {
+
+  _dataStatsTable({ pending, successful, failed, stats }) {
+    const { deletion } = stats;
+    const pendingSums = this._sumMetrics(pending);
+
+    return formatTable([
+      ['', 'Requests', 'Records'],
+      ['Pending', pendingSums.requests, pendingSums.records],
+      ['Successful', successful.requests, successful.records],
+      ['- Deleted', '', deletion.deleted],
+      ['- Not Found', '', deletion.notFound],
+      ['Failed', failed.requests, failed.records],
+    ]);
+  }
+
+}

package/src/logger/index.js

@@ -0,0 +1,41 @@
+import { FORMAT } from './constants.js';
+import StandardLogStream from './standard.js';
+import LegacyProgressLogStream from './progress.legacy.js';
+import ApiProgressLogStream from './api-progress.js';
+import DeleteProgressLogStream from './delete-progress.js';
+
+export * from './constants.js';
+
+export function createLogStream({
+  api,
+  legacy,
+  level,
+  format,
+  out,
+  err,
+}) {
+  switch (format || FORMAT.JSON) {
+    case FORMAT.PROGRESS:
+      switch (api) {
+        case 'upload':
+          if (legacy) {
+            return new LegacyProgressLogStream({ out, err });
+          }
+          return new ApiProgressLogStream({ out, err });
+        case 'delete':
+          return new DeleteProgressLogStream({ out, err });
+        default:
+          throw new Error(`Unsupported API: ${api}`);
+      }
+    case FORMAT.TEXT:
+    case FORMAT.JSON:
+      return new StandardLogStream({
+        level,
+        format,
+        out,
+        err,
+      });
+    default:
+      throw new Error(`Unrecognized log format: ${format}`);
+  }
+}