@miso.ai/server-sdk 0.6.2-beta.0 → 0.6.3-beta.0

package/cli/delete.js

@@ -51,7 +51,7 @@ const run = type => async ({
 
   const client = new MisoClient({ key, server });
 
-  const deleteStream = client.createDeleteStream(type, {
+  const deleteStream = client.api[type].deleteStream({
     name,
     params,
     heartbeatInterval: logFormat === logger.FORMAT.PROGRESS ? 250 : false,

package/cli/ids-diff.js

@@ -34,7 +34,7 @@ const run = type => async ({
   output = output || (plus ? 'plus' : minus ? 'minus' : undefined);
 
   const client = new MisoClient({ key, server });
-  const misoIds = await client.ids(type);
+  const misoIds = await client.api[type].ids();
 
   const diffStream = new stream.DiffStream(misoIds, { output });
   const outputStream = new stream.OutputStream({ objectMode: false });

package/cli/ids.js

@@ -13,7 +13,13 @@ const run = type => async ({
   server,
 }) => {
   const client = new MisoClient({ key, server });
-  const ids = await client.ids(type);
+  let ids;
+  try {
+    ids = await client.api[type].ids();
+  } catch (err) {
+    console.error(err);
+    throw err;
+  }
 
   const readStream = Readable.from(ids);
   const outputStream = new stream.OutputStream();

package/cli/index.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import { yargs } from '@miso.ai/server-commons';
+import { MisoClient } from '../src/index.js';
 import upload from './upload.js';
 import del from './delete.js';
 import ids from './ids.js';
 import transform from './transform.js';
-import version from '../src/version.js';
 
 const interactions = {
   command: 'interactions',
@@ -58,7 +58,7 @@ yargs.build(yargs => {
     .command(users)
     .command(experiments)
     .command(transform)
-    .version(version);
+    .version(MisoClient.version);
 });
 
 

package/cli/transform.js

@@ -27,7 +27,7 @@ async function run({ file }) {
     objectMode: transform.readableObjectMode,
   }));
 
-  await stream.pipeline(streams);
+  await stream.pipeline(...streams);
 }
 
 async function getTransformStream(loc) {

package/cli/upload.js

@@ -4,16 +4,12 @@ import { MisoClient, logger, normalize } from '../src/index.js';
 
 function build(yargs) {
   return yargs
-    .option('async', {
-      alias: ['a'],
-      describe: 'Asynchrnous mode',
-    })
     .option('dry-run', {
       alias: ['dry'],
       describe: 'Dry run mode',
     })
     .option('lenient', {
-      describe: 'Accept some enient record schema',
+      describe: 'Accept some lenient record schema',
       type: 'boolean',
     })
     .option('records-per-request', {
@@ -53,7 +49,6 @@ const run = type => async ({
   key,
   server,
   param: params,
-  async,
   ['dry-run']: dryRun,
   lenient,
   ['records-per-request']: recordsPerRequest,
@@ -74,10 +69,9 @@ const run = type => async ({
 
   const uploadStreamObjectMode = lenient;
 
-  const uploadStream = client.createUploadStream(type, {
+  const uploadStream = client.api[type].uploadStream({
     objectMode: uploadStreamObjectMode,
     name,
-    async, 
     dryRun,
     params,
     heartbeatInterval: logFormat === logger.FORMAT.PROGRESS ? 250 : false,

package/package.json

@@ -16,11 +16,11 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.2-beta.0",
+    "@miso.ai/server-commons": "0.6.3-beta.0",
     "axios": "^0.27.2",
     "dotenv": "^16.0.1",
     "split2": "^4.1.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.2-beta.0"
+  "version": "0.6.3-beta.0"
 }

package/src/api/helpers.js

@@ -1,15 +1,48 @@
-import { asArray } from '@miso.ai/server-commons';
+import { asArray, trimObj, computeIfAbsent } from '@miso.ai/server-commons';
 import axios from 'axios';
 import { Buffer } from 'buffer';
 
 export async function upload(client, type, records, options = {}) {
-  const url = buildUrl(client, type, options);
+  const url = buildUrl(client, type, { ...options, async: true });
   const payload = buildUploadPayload(records);
   return axios.post(url, payload);
 }
 
+const RE_422_MSG_LINE = /^\s*data\.(\d+)(?:\.(\S+))?\s+is\s+invalid\.\s+(.*)$/;
+
+export function process422ResponseBody(payload, { data } = {}) {
+  const records = extractUploadPayload(payload);
+  const unrecognized = [];
+  const groupsMap = new Map();
+  for (const line of data) {
+    const m = line.match(RE_422_MSG_LINE);
+    if (!m) {
+      unrecognized.push(line);
+      continue;
+    }
+    let [_, index, path, message] = m;
+    index = Number(index);
+    const { violations } = computeIfAbsent(groupsMap, index, index => ({
+      index,
+      violations: [],
+      record: records[index],
+    }));
+    violations.push({
+      path,
+      message,
+    });
+  }
+  const groups = [...groupsMap.keys()]
+    .sort((a, b) => a - b)
+    .map(groupsMap.get.bind(groupsMap));
+  return trimObj({
+    groups,
+    unrecognized: unrecognized.length ? unrecognized : undefined,
+  });
+}
+
 export async function batchDelete(client, type, ids, options = {}) {
-  const url = buildUrl(client, `${type}/_delete`, options);
+  const url = buildUrl(client, `${type}/_delete`, { ...options, async: true });
   const payload = buildBatchDeletePayload(type, ids);
   // TODO: organize axios
   const { data } = await axios.post(url, payload, {
@@ -23,10 +56,11 @@ export async function batchDelete(client, type, ids, options = {}) {
 export function buildUrl(client, path, { async, dryRun, params: extraParams } = {}) {
   let { server, key } = client._options;
   let params = `?api_key=${key}`;
+  if (async) {
+    params += '&async=1';
+  }
   if (dryRun) {
     params += '&dry_run=1';
-  } else if (async) {
-    params += '&async=1';
   }
   if (extraParams) {
     for (const key in extraParams) {
@@ -40,7 +74,23 @@ export function buildUrl(client, path, { async, dryRun, params: extraParams } =
 export function buildUploadPayload(records) {
   return typeof records === 'string' ? records :
     Buffer.isBuffer(records) ? records.toString() :
-    { data: Array.isArray(records)? records : [records] };
+    { data: Array.isArray(records) ? records : [records] };
+}
+
+export function extractUploadPayload(records) {
+  if (Buffer.isBuffer(records)) {
+    records = records.toString();
+  }
+  if (typeof records === 'string') {
+    records = JSON.parse(records);
+  }
+  if (records.data) {
+    records = records.data;
+  }
+  if (!Array.isArray(records)) {
+    records = [records];
+  }
+  return records;
 }
 
 export function buildBatchDeletePayload(type, ids) {

package/src/client.js

@@ -1,10 +1,4 @@
-import { asArray } from '@miso.ai/server-commons';
-//import { createHash } from 'crypto';
-import { Buffer } from 'buffer';
-import axios from 'axios';
 import version from './version.js';
-import UploadStream from './stream/upload.js';
-import DeleteStream from './stream/delete.js';
 import Api from './api/index.js';
 
 export default class MisoClient {
@@ -17,62 +11,6 @@ export default class MisoClient {
     this.api = new Api(this);
   }
 
-  async upload(type, records, options) {
-    const url = buildUrl(this, type, options);
-    const payload = buildPayload(records);
-    return await axios.post(url, payload);
-  }
-
-  // TODO: extract to .experiment() later
-  async uploadExperimentEvent(experimentId, record) {
-    // TODO: support non-string record
-    const url = buildUrl(this, `experiments/${experimentId}/events`);
-    // TODO: make content type header global
-    const headers = { 'Content-Type': 'application/json' };
-    const response = await axios.post(url, record, { headers });
-    // 200 response body does not have .data layer
-    return response.data ? response : { data: response };
-  }
-
-  async ids(type) {
-    const url = buildUrl(this, `${type}/_ids`);
-    return (await axios.get(url)).data.data.ids;
-  }
-
-  async delete(type, ids) {
-    if (type !== 'products' && type !== 'users') {
-      throw new Error(`Only products and users are supported: ${type}`);
-    }
-    ids = asArray(ids);
-    if (ids.length === 0) {
-      return { data: {} };
-    }
-    const payload = {
-      data: {
-        [type === 'products' ? 'product_ids' : 'user_ids']: ids,
-      },
-    };
-    return this._delete(type, payload);
-  }
-
-  async _delete(type, payload, options) {
-    const url = buildUrl(this, `${type}/_delete`, options);
-    const { data } = await axios.post(url, payload, {
-      headers: {
-        'Content-Type': 'application/json',
-      },
-    });
-    return { data };
-  }
-
-  createUploadStream(type, options) {
-    return new UploadStream(this, type, options);
-  }
-
-  createDeleteStream(type, options) {
-    return new DeleteStream(this, type, options);
-  }
-
   get options() {
     const { server, key } = this._options;
     return Object.freeze({
@@ -95,26 +33,3 @@ function normalizeOptions(options) {
 
   return options;
 }
-
-function buildUrl(client, path, { async, dryRun, params: extraParams } = {}) {
-  let { server, key } = client._options;
-  let params = `?api_key=${key}`;
-  if (dryRun) {
-    params += '&dry_run=1';
-  } else if (async) {
-    params += '&async=1';
-  }
-  if (extraParams) {
-    for (const key in extraParams) {
-      // TODO: deal with encodeURIComponent
-      params += `&${key}=${extraParams[key]}`;
-    }
-  }
-  return `${server}/v1/${path}${params}`;
-}
-
-function buildPayload(records) {
-  return typeof records === 'string' ? records :
-    Buffer.isBuffer(records) ? records.toString() :
-    { data: Array.isArray(records)? records : [records] };
-}

package/src/logger/api-progress.js

@@ -55,15 +55,12 @@ export default class ApiProgressLogStream extends stream.LogUpdateStream {
 
   _configProps(config = {}) {
     const { sink = {}, extra = {} } = config;
-    const { dryRun, async, params } = sink;
+    const { dryRun, params } = sink;
     const { lenient } = extra;
     const props = [];
     if (dryRun) {
       props.push('dry-run');
     }
-    if (async) {
-      props.push('async');
-    }
     if (lenient) {
       props.push('lenient');
     }

package/src/stream/api-sink.js

@@ -45,9 +45,12 @@ export default class ApiSink extends sink.BpsSink {
       if (!error.response) {
         throw error;
       }
+      const status = error.response.status;
       data = error.response.data;
       if (typeof data !== 'object') {
-        data = trimObj({ errors: true, cause: data });
+        data = trimObj({ status, errors: true, cause: data });
+      } else if (status) {
+        data = { status, ...data };
       }
     }
 

package/src/stream/upload-sink.js

@@ -8,7 +8,6 @@ class UploadSink extends ApiSink {
   }
 
   _normalizeOptions({
-    async,
     dryRun,
     ...options
   } = {}) {
@@ -17,62 +16,14 @@ class UploadSink extends ApiSink {
     }
     return {
       ...super._normalizeOptions(options),
-      async: !!async,
       dryRun: !!dryRun,
     };
   }
 
   async _execute(payload) {
-    const { type, async, dryRun, params } = this._options;
-    const response = await upload(this._client, type, payload, { async, dryRun, params });
-    return response.data;
-  }
-
-}
-
-class DataSetUploadSink extends UploadSink {
-
-  constructor(client, options) {
-    super(client, options);
-    this._stats.api = { count: 0, bytes: 0, took: 0 };
-  }
-
-  _normalizeOptions({
-    apiBpsRate,
-    apiBpsSampleThreshold = 1024 * 1024,
-    ...options
-  } = {}) {
-    return {
-      ...super._normalizeOptions(options),
-      apiBpsRate: this._normalizeApiBpsRate(apiBpsRate, options),
-      apiBpsSampleThreshold,
-    };
-  }
-
-  _normalizeApiBpsRate(apiBpsRate, options) {
-    if (options.type === 'interactions' || options.async) {
-      // in async mode, apiBps is meaningless
-      return false;
-    }
-    if (apiBpsRate === undefined || apiBpsRate === null) {
-      // default to 1 if absent
-      return 1;
-    }
-    if (apiBpsRate === false || (!isNaN(apiBpsRate) && apiBpsRate > 0)) {
-      // legal values
-      return apiBpsRate;
-    }
-    throw new Error(`Illegal apiBpsRate value: ${apiBpsRate}`);
-  }
-
-  _targetBps() {
-    const { bytesPerSecond: configuredBps, apiBpsRate, apiBpsSampleThreshold } = this._options;
-    if (apiBpsRate && this._stats.api.bytes < apiBpsSampleThreshold) {
-      // use configured BPS until we have enough data from API response
-      return configuredBps;
-    }
-    const apiBps = this._serviceStats.bps;
-    return !isNaN(apiBps) ? Math.max(apiBps * apiBpsRate, configuredBps) : configuredBps;
+    const { type, dryRun, params } = this._options;
+    const { data } = await upload(this._client, type, payload, { dryRun, params });
+    return data;
   }
 
 }
@@ -96,7 +47,7 @@ export default function create(client, type, options) {
     case 'users':
     case 'products':
     case 'interactions':
-      return new DataSetUploadSink(client, { ...options, type });
+      return new UploadSink(client, { ...options, type });
     case 'experiment-events':
       return new ExperimentEventUploadSink(client, options);
     default:

package/src/stream/upload.js

@@ -2,6 +2,7 @@ import { stream } from '@miso.ai/server-commons';
 import version from '../version.js';
 import createSink from './upload-sink.js';
 import createBuffer from './upload-buffer.js';
+import { process422ResponseBody } from '../api/helpers.js';
 
 export default class UploadStream extends stream.BufferedWriteStream {
 
@@ -11,7 +12,6 @@ export default class UploadStream extends stream.BufferedWriteStream {
     objectMode,
     heartbeatInterval,
     // sink
-    async,
     dryRun,
     params,
     experimentId,
@@ -35,7 +35,6 @@ export default class UploadStream extends stream.BufferedWriteStream {
     this._type = type;
 
     this._sink = createSink(client, type, {
-      async, 
       dryRun,
       params,
       experimentId,
@@ -65,8 +64,14 @@ export default class UploadStream extends stream.BufferedWriteStream {
     const output = super._output(message, args);
 
     // if upload fails, emit extracted payload at response event
-    if (message.event === 'response' && args.response && args.response.errors && args.payload) {
-      output.payload = JSON.parse(args.payload);
+    if (message.event === 'response') {
+      const { response, payload } = args;
+      if (response && response.errors && payload) {
+        output.payload = JSON.parse(payload); // TODO: call extractUploadPayload
+        if (response.status === 422) {
+          output.issues = process422ResponseBody(payload, response);
+        }
+      }
     }
 
     // add upload stats

package/src/version.js

@@ -1 +1 @@
-export default '0.6.2-beta.0';
+export default '0.6.3-beta.0';