@miso.ai/server-sdk 0.6.6-beta.6 → 0.6.6

package/cli/delete.js

@@ -1,21 +1,11 @@
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
-import { log, stream } from '@miso.ai/server-commons';
+import { log, stream, splitObj } from '@miso.ai/server-commons';
 import { MisoClient, logger } from '../src/index.js';
+import { buildForWrite } from './utils.js';
 
 function build(yargs) {
-  return yargs
-    .option('records-per-request', {
-      alias: ['rpr'],
-      describe: 'How many records to send in a request',
-    })
-    .option('records-per-second', {
-      alias: ['rps'],
-      describe: 'How many records to send per second',
-    })
-    .option('debug', {
-      describe: 'Set log level to debug',
-      type: 'boolean',
-    })
+  return buildForWrite(yargs)
     .option('progress', {
       alias: ['p'],
       describe: 'Set log format progress',
@@ -34,8 +24,35 @@ function build(yargs) {
 }
 
 const run = type => async ({
+  env,
   key,
   server,
+  channel,
+  ...options
+}) => {
+  const { debug } = options;
+  const client = new MisoClient({ env, key, server, debug });
+
+  if (channel) {
+    await runChannel(client, type, options);
+  } else {
+    await runStream(client, type, options);
+  }
+};
+
+async function runChannel(client, type, options) {
+  const [deleteOptions] = splitObj(options, ['params', 'requestsPerSecond', 'bytesPerSecond', 'recordsPerRequest', 'bytesPerRequest', 'debug']);
+  const deleteChannel = client.api[type].deleteChannel(deleteOptions);
+
+  await pipeline(
+    process.stdin,
+    split2(JSON.parse),
+    deleteChannel,
+    new stream.OutputStream({ objectMode: true }),
+  );
+}
+
+async function runStream(client, type, {
   param: params,
   ['records-per-request']: recordsPerRequest,
   ['records-per-second']: recordsPerSecond,
@@ -44,13 +61,10 @@ const run = type => async ({
   ['stream-name']: name,
   ['log-level']: loglevel,
   ['log-format']: logFormat,
-}) => {
-
+}) {
   loglevel = (debug || progress) ? log.DEBUG : loglevel;
   logFormat = progress ? logger.FORMAT.PROGRESS : logFormat;
 
-  const client = new MisoClient({ key, server, debug });
-
   const deleteStream = client.api[type].deleteStream({
     name,
     params,
@@ -66,13 +80,13 @@ const run = type => async ({
     format: logFormat,
   });
 
-  await stream.pipeline(
+  await pipeline(
     process.stdin,
     split2(),
     deleteStream,
     logStream,
   );
-};
+}
 
 export default function(type) {
   return {

package/cli/get.js

@@ -4,12 +4,13 @@ import { formatError } from './utils.js';
 const build = yargs => yargs;
 
 const run = type => async ({
+  env,
   key,
   server,
   id,
   debug,
 }) => {
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
   try {
     const entity = await client.api[type].get(id);
     console.log(JSON.stringify(entity));

package/cli/hybrid-search.js

@@ -1,4 +1,5 @@
 import { Readable } from 'stream';
+import { pipeline } from 'stream/promises';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
 import { buildForApi, buildForSearch } from './utils.js';
@@ -15,13 +16,13 @@ function build(yargs) {
     });
 }
 
-async function run({ query, fq, fl, rows, answer, key, server, debug }) {
-  const client = new MisoClient({ key, server, debug });
+async function run({ query, fq, fl, rows, answer, env, key, server, debug }) {
+  const client = new MisoClient({ env, key, server, debug });
   const { products } = await client.api.ask.search({ q: query, fq, fl, rows, answer });
   const readStream = Readable.from(products);
   const outputStream = new stream.OutputStream();
 
-  await stream.pipeline(
+  await pipeline(
     readStream,
     outputStream,
   );

package/cli/ids-diff.js

@@ -1,3 +1,4 @@
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
@@ -25,6 +26,7 @@ function build(yargs) {
 };
 
 const run = type => async ({
+  env,
   key,
   server,
   output,
@@ -34,13 +36,13 @@ const run = type => async ({
 }) => {
   output = output || (plus ? 'plus' : minus ? 'minus' : undefined);
 
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
   const misoIds = await client.api[type].ids();
 
   const diffStream = new stream.DiffStream(misoIds, { output });
   const outputStream = new stream.OutputStream({ objectMode: false });
 
-  await stream.pipeline(
+  await pipeline(
     process.stdin,
     split2(),
     diffStream,

package/cli/ids.js

@@ -1,4 +1,5 @@
 import { Readable } from 'stream';
+import { pipeline } from 'stream/promises';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
 import diff from './ids-diff.js';
@@ -19,12 +20,13 @@ const build = type => yargs => {
 };
 
 const run = type => async ({
+  env,
   key,
   server,
   type: recordType,
   debug,
 }) => {
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
   let ids;
   try {
     const options = recordType ? { type: recordType } : {};
@@ -37,7 +39,7 @@ const run = type => async ({
   const readStream = Readable.from(ids);
   const outputStream = new stream.OutputStream();
 
-  await stream.pipeline(
+  await pipeline(
     readStream,
     outputStream,
   );

package/cli/index.js

@@ -12,6 +12,7 @@ import status from './status.js';
 import get from './get.js';
 import search from './search.js';
 import hybridSearch from './hybrid-search.js';
+import upgrade from './upgrade.js';
 
 const interactions = {
   command: 'interactions',
@@ -47,30 +48,14 @@ const users = {
     .command(status('users')),
 };
 
-const experiments = {
-  command: 'experiments',
-  aliases: ['experiment'],
-  description: 'Experiment commands',
-  builder: yargs => buildForApi(yargs)
-    .option('experiment-id', {
-      alias: ['exp-id'],
-      describe: 'Experiment ID for experiment API',
-    })
-    .command({
-      command: 'events',
-      builder: yargs => yargs
-        .command(upload('experiment-events')),
-    }),
-};
-
 yargs.build(yargs => {
   yargs
     .env('MISO')
     .command(interactions)
     .command(products)
     .command(users)
-    .command(experiments)
     .command(transform)
+    .command(upgrade)
     .command(mergeLocal)
     .command(search)
     .command(hybridSearch)

package/cli/merge.js

@@ -1,6 +1,7 @@
 import { join } from 'path';
 import { createReadStream } from 'fs';
 import { createGunzip } from 'zlib';
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
@@ -23,6 +24,7 @@ function build(yargs) {
 }
 
 const run = type => async ({
+  env,
   key,
   server,
   file,
@@ -32,13 +34,12 @@ const run = type => async ({
 }) => {
   const mergeFn = await getMergeFn(file);
   const records = await buildBaseRecords(base);
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
   const mergeStream = client.api[type].mergeStream({ ...options, mergeFn, records });
   const outputStream = new stream.OutputStream({ objectMode: true });
-  await stream.pipeline(
+  await pipeline(
     process.stdin,
-    split2(),
-    stream.parse(),
+    split2(JSON.parse),
     mergeStream,
     outputStream
   );

package/cli/search.js

@@ -1,4 +1,5 @@
 import { Readable } from 'stream';
+import { pipeline } from 'stream/promises';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
 import { buildForApi, buildForSearch } from './utils.js';
@@ -10,13 +11,13 @@ function build(yargs) {
     });
 }
 
-async function run({ query, fq, fl, rows, key, server, debug }) {
-  const client = new MisoClient({ key, server, debug });
+async function run({ query, fq, fl, rows, env, key, server, debug }) {
+  const client = new MisoClient({ env, key, server, debug });
   const records = await client.api.search.search({ q: query, fq, fl, rows });
   const readStream = Readable.from(records);
   const outputStream = new stream.OutputStream();
 
-  await stream.pipeline(
+  await pipeline(
     readStream,
     outputStream,
   );

package/cli/status.js

@@ -1,3 +1,4 @@
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
@@ -7,12 +8,13 @@ const build = type => yargs => {
 };
 
 const run = type => async ({
+  env,
   key,
   server,
   taskId,
   debug,
 }) => {
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
   if (taskId) {
     runOne(client, type, taskId);
   } else {
@@ -30,14 +32,14 @@ async function runOne(client, type, taskId) {
 }
 
 async function runStream(client, type) {
-  await stream.pipeline([
+  await pipeline(
     process.stdin,
     split2(),
     client.api[type].statusStream(),
     new stream.OutputStream({
       objectMode: true,
     }),
-  ]);
+  );
 }
 
 export default function(type) {

package/cli/transform.js

@@ -1,3 +1,4 @@
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
 import { stream } from '@miso.ai/server-commons';
 
@@ -11,19 +12,15 @@ function build(yargs) {
 
 async function run({ file }) {
   const transform = await stream.getTransformStream(file);
-  const streams = [
-    process.stdin,
-    split2(),
-  ];
-  if (transform.writableObjectMode) {
-    streams.push(stream.parse());
-  }
-  streams.push(transform);
-  streams.push(new stream.OutputStream({
-    objectMode: transform.readableObjectMode,
-  }));
 
-  await stream.pipeline(...streams);
+  await pipeline(
+    process.stdin,
+    transform.writableObjectMode ? split2(JSON.parse) : split2(),
+    transform,
+    new stream.OutputStream({
+      objectMode: transform.readableObjectMode,
+    }),
+  );
 }
 
 export default {

package/cli/upgrade.js

@@ -0,0 +1,37 @@
+import { pipeline } from 'stream/promises';
+import split2 from 'split2';
+import { stream, UpgradeChannel, splitObj } from '@miso.ai/server-commons';
+
+function build(yargs) {
+  return yargs
+    .option('name', {
+      describe: 'Channel name',
+      type: 'string',
+    })
+    .option('id-field', {
+      alias: 'id',
+      describe: 'Payload field to use as ID',
+      type: 'string',
+    })
+    .option('as-id', {
+      describe: 'Upgrade as ID',
+      type: 'boolean',
+    });
+}
+
+async function run(options) {
+  const [upgradeOptions] = splitObj(options, ['name', 'asId', 'idField']);
+  await pipeline(
+    process.stdin,
+    split2(),
+    new UpgradeChannel({ objectMode: false, ...upgradeOptions }),
+    new stream.OutputStream({ objectMode: true }),
+  );
+}
+
+export default {
+  command: ['upgrade', 'up'],
+  description: `Upgrade the stream to a channel`,
+  builder: build,
+  handler: run,
+};

package/cli/upload.js

@@ -1,33 +1,20 @@
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
-import { log, stream } from '@miso.ai/server-commons';
+import { log, stream, splitObj } from '@miso.ai/server-commons';
 import { MisoClient, logger, normalize } from '../src/index.js';
+import { buildForWrite } from './utils.js';
 
 function build(yargs) {
-  return yargs
+  return buildForWrite(yargs)
     .option('dry-run', {
       alias: ['dry'],
       describe: 'Dry run mode',
+      type: 'boolean',
     })
     .option('lenient', {
       describe: 'Accept some lenient record schema',
       type: 'boolean',
     })
-    .option('requests-per-second', {
-      alias: ['rps'],
-      describe: 'How many requests to send per second',
-    })
-    .option('records-per-request', {
-      alias: ['rpr'],
-      describe: 'How many records to send in a request',
-    })
-    .option('bytes-per-request', {
-      alias: ['bpr'],
-      describe: 'How many bytes to send in a request',
-    })
-    .option('bytes-per-second', {
-      alias: ['bps'],
-      describe: 'How many bytes to send per second',
-    })
     .option('progress', {
       alias: ['p'],
       describe: 'Set log format progress',
@@ -46,8 +33,35 @@ function build(yargs) {
 }
 
 const run = type => async ({
+  env,
   key,
   server,
+  channel,
+  ...options
+}) => {
+  const { debug } = options;
+  const client = new MisoClient({ env, key, server, debug });
+
+  if (channel) {
+    await runChannel(client, type, options);
+  } else {
+    await runStream(client, type, options);
+  }
+};
+
+async function runChannel(client, type, options) {
+  const [uploadOptions] = splitObj(options, ['dryRun', 'params', 'requestsPerSecond', 'bytesPerSecond', 'recordsPerRequest', 'bytesPerRequest', 'debug']);
+  const uploadChannel = client.api[type].uploadChannel(uploadOptions);
+
+  await pipeline(
+    process.stdin,
+    split2(JSON.parse),
+    uploadChannel,
+    new stream.OutputStream({ objectMode: true }),
+  );
+}
+
+async function runStream(client, type, {
   param: params,
   ['dry-run']: dryRun,
   lenient,
@@ -55,19 +69,16 @@ const run = type => async ({
   ['records-per-request']: recordsPerRequest,
   ['bytes-per-request']: bytesPerRequest,
   ['bytes-per-second']: bytesPerSecond,
-  ['experiment-id']: experimentId,
-  debug,
   progress,
+  debug,
+  ['experiment-id']: experimentId,
   ['stream-name']: name,
   ['log-level']: loglevel,
   ['log-format']: logFormat,
-}) => {
-
+}) {
   loglevel = (debug || progress) ? log.DEBUG : loglevel;
   logFormat = progress ? logger.FORMAT.PROGRESS : logFormat;
 
-  const client = new MisoClient({ key, server, debug });
-
   const uploadStreamObjectMode = lenient;
 
   const uploadStream = client.api[type].uploadStream({
@@ -97,17 +108,18 @@ const run = type => async ({
   // lenient: stdin -> split2 -> parse -> normalize -> upload -> log
   // notice that the output of split2 are strings, while input/output of normalize are objects
 
-  await stream.pipeline(
+  await pipeline(
     process.stdin,
-    split2(),
     ...(lenient ? [
-      stream.parse(),
+      split2(JSON.parse),
       new normalize.Stream(type),
-    ] : []),
+    ] : [
+      split2(),
+    ]),
     uploadStream,
     logStream,
   );
-};
+}
 
 export default function(type) {
   return {

package/cli/utils.js

@@ -6,6 +6,10 @@ export function buildForApi(yargs) {
       alias: ['k', 'api-key'],
       describe: 'API key',
     })
+    .option('env', {
+      describe: 'Environment',
+      type: 'string',
+    })
     .option('server', {
       alias: ['api-server'],
       describe: 'API server',
@@ -13,14 +17,13 @@ export function buildForApi(yargs) {
     .option('param', {
       alias: ['v', 'var'],
       describe: 'Extra URL parameters',
-      type: 'array',
+      type: 'string',
       coerce: _yargs.coerceToArray,
     })
     .option('debug', {
       describe: 'Set log level to debug',
       type: 'boolean',
-    })
-    .demandOption(['key'], 'API key is required.');
+    });
 }
 
 export function buildForSearch(yargs) {
@@ -46,6 +49,36 @@ export function buildForSearch(yargs) {
     });
 }
 
+export function buildForWrite(yargs) {
+  return yargs
+    .option('channel', {
+      alias: ['c'],
+      describe: 'Use channel protocol',
+      type: 'boolean',
+      default: false,
+    })
+    .option('requests-per-second', {
+      alias: ['rps'],
+      describe: 'How many requests to send per second',
+      type: 'number',
+    })
+    .option('bytes-per-second', {
+      alias: ['bps'],
+      describe: 'How many bytes to send per second',
+      type: 'number',
+    })
+    .option('records-per-request', {
+      alias: ['rpr'],
+      describe: 'How many records to send in a request',
+      type: 'number',
+    })
+    .option('bytes-per-request', {
+      alias: ['bpr'],
+      describe: 'How many bytes to send in a request',
+      type: 'number',
+    });
+}
+
 export function formatError(err) {
   const { response } = err;
   if (response) {

package/package.json

@@ -16,12 +16,12 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.6-beta.6",
+    "@miso.ai/server-commons": "0.6.6",
     "axios": "^1.6.2",
     "axios-retry": "^4.5.0",
     "dotenv": "^16.0.1",
     "split2": "^4.1.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.6-beta.6"
+  "version": "0.6.6"
 }

package/src/api/base.js

@@ -4,6 +4,7 @@ import UploadStream from '../stream/upload.js';
 import DeleteStream from '../stream/delete.js';
 import StatusStream from '../stream/status.js';
 import MergeStream from '../stream/merge.js';
+import { UploadChannel, DeleteChannel } from '../channel/index.js';
 
 export class Queries {
 
@@ -35,6 +36,10 @@ export class Writable {
     return new UploadStream(this._client, this._type, options);
   }
 
+  uploadChannel(options = {}) {
+    return new UploadChannel(this._client, this._type, options);
+  }
+
 }
 
 export class Entities extends Writable {
@@ -44,7 +49,7 @@ export class Entities extends Writable {
   }
 
   async get(id) {
-    const url = buildUrl(this._client, `${this._type}/${id}`);
+    const url = buildUrl(this._client, `${this._type}/${encodeURIComponent(id)}`);
     return (await this._client._axios.get(url)).data.data;
   }
 
@@ -62,6 +67,10 @@ export class Entities extends Writable {
     return new DeleteStream(this._client, this._type, options);
   }
 
+  deleteChannel(options = {}) {
+    return new DeleteChannel(this._client, this._type, options);
+  }
+
   async status(taskId) {
     const url = buildUrl(this._client, `${this._type}/_status/${taskId}`);
     return (await this._client._axios.get(url)).data;

package/src/api/helpers.js

@@ -14,12 +14,15 @@ export async function upload(client, type, records, options = {}) {
 }
 
 async function recoverValidRecords(client, type, records, options, response) {
-  if (!response || response.status !== 422 || !options.recoverValidRecordsOn422) {
+  if (!response || response.status !== 422) {
     return;
   }
   records = extractRecordsFromUploadPayload(records);
   // try to collect valid records and resend them, which should pass the validation
-  const { groups = [], unrecognized = [] } = process422ResponseBody(records, response.data); // it takes records too
+  const { groups = [], unrecognized = [] } = response.issues = process422ResponseBody(records, response.data); // it takes records too
+  if (!options.recoverValidRecordsOn422) {
+    return; // still write issues to response
+  }
   if (groups.length === 0 || groups.length === records.length || unrecognized.length > 0) {
     // if there are unrecognized messages, it's hard to tell which records are valid
     return;
@@ -31,15 +34,18 @@ async function recoverValidRecords(client, type, records, options, response) {
   }
   const url = buildUrl(client, type, options);
   const validPayload = buildUploadPayload(validRecords);
+  let secondResponse;
   try {
-    await client._axios.post(url, validPayload);
+    secondResponse = await client._axios.post(url, validPayload);
   } catch (_) {
     return; // still fail, never mind...
   }
-  response.recovered = {
+  response.recovered = trimObj({
+    ...secondResponse,
+    product_ids: validRecords.map(record => record.product_id),
     records: validRecords.length,
     bytes: validPayload.length * 2,
-  };
+  });
 }
 
 export async function merge(client, type, record, { mergeFn = defaultMerge } = {}) {
@@ -131,18 +137,15 @@ export function process422ResponseBody(payload, { data } = {}) {
 }
 
 export async function batchDelete(client, type, ids, options = {}) {
-  const url = buildUrl(client, `${type}/_delete`, { ...options, async: true });
+  const url = buildUrl(client, `${type}/_delete`, options);
   const payload = buildBatchDeletePayload(type, ids);
   const { data } = await client._axios.post(url, payload);
   return data;
 }
 
-export function buildUrl(client, path, { async, dryRun, params: extraParams } = {}) {
+export function buildUrl(client, path, { dryRun, params: extraParams } = {}) {
   let { server, key } = client._options;
   let params = `?api_key=${key}`;
-  if (async) {
-    params += '&async=1';
-  }
   if (dryRun) {
     params += '&dry_run=1';
   }

package/src/api/index.js

@@ -1,7 +1,6 @@
 import Products from './products.js';
 import Users from './users.js';
 import Interactions from './interactions.js';
-import Experiments from './experiments.js';
 import Ask from './ask.js';
 import Search from './search.js';
 import Recommendation from './recommendation.js';
@@ -12,7 +11,6 @@ export default class Api {
     this.products = new Products(client);
     this.users = new Users(client);
     this.interactions = new Interactions(client);
-    this.experiments = new Experiments(client);
     this.ask = new Ask(client);
     this.search = new Search(client);
     this.recommendation = new Recommendation(client);

package/src/channel/api.js

@@ -0,0 +1,194 @@
+import { WriteChannelSink, WriteChannel, trimObj } from '@miso.ai/server-commons';
+
+export function normalizeApiSinkGateOptions({
+  writesPerSecond = 10,
+  recordsPerSecond = 100000,
+  bytesPerSecond = 100 * 1024 * 1024,
+  ...options
+} = {}) {
+  return {
+    writesPerSecond,
+    recordsPerSecond,
+    bytesPerSecond,
+    ...options,
+  };
+}
+
+function normalizeParams(params) {
+  if (!params || params.length === 0) {
+    return undefined;
+  }
+  return params.reduce((acc, param) => {
+    const [key, value = '1'] = param.split('=');
+    acc[key] = value;
+    return acc;
+  }, {});
+}
+
+function splitData(data, ids) {
+  ids = new Set(ids);
+  const positive = [];
+  const negative = [];
+  for (const record of data) {
+    if (ids.has(record.id)) {
+      positive.push(record);
+    } else {
+      negative.push(record);
+    }
+  }
+  return [positive, negative];
+}
+
+function writeIssuesToData(data, { groups = [] } = {}, { name: channel } = {}) {
+  for (const { index, violations = [] } of groups) {
+    try {
+      const errors = (data[index].errors || (data[index].errors = []));
+      errors.push(...(violations.map(v => trimObj({ channel, status: 422, ...v }))));
+    } catch(_) {}
+  }
+}
+
+function writeResponseErrorToFailedData(response, { name: channel } = {}) {
+  // 422 errors are already handled by writeIssuesToData
+  if (!response.error && (response.status < 400 || response.status === 422)) {
+    return;
+  }
+  const error = trimObj({
+    channel, 
+    status: response.status,
+    message: response.error || response.statusText,
+  });
+  for (const event of response.failed.data) {
+    (event.errors || (event.errors = [])).push({ ...error });
+  }
+}
+
+export class ChannelApiSink extends WriteChannelSink {
+
+  constructor(client, options) {
+    super(options);
+    this._client = client;
+    // TODO: take axios options?
+  }
+
+  _normalizeOptions({
+    params,
+    dryRun,
+    ...options
+  } = {}) {
+    return trimObj({
+      ...super._normalizeOptions(options),
+      params: normalizeParams(params),
+      dryRun: !!dryRun,
+    });
+  }
+
+  async _write(request) {
+    const { payload, records, data } = request;
+    let response;
+    try {
+      response = await this._send(payload);
+      response = processMisoApiResponse(response);
+      response.writes = 1;
+      response.successful = { records, data };
+      response.failed = { records: 0, data: [] };
+    } catch(error) {
+      // not axios-handled error
+      if (!error.response) {
+        throw error;
+      }
+      response = processMisoApiResponse(error.response);
+      if (typeof response !== 'object') {
+        response = trimObj({ error: response });
+      }
+      const { recovered, issues } = error.response;
+      // write issues into failed data events
+      // TODO: need to add channel name
+      writeIssuesToData(data, issues, this._channel);
+      // TODO: handle issue.unrecognized
+
+      if (recovered) {
+        const [positive, negative] = splitData(data, recovered.product_ids);
+        response.writes = 2;
+        response.successful = { records: positive.length, data: positive };
+        response.failed = { records: negative.length, data: negative };
+        response.recovered = processMisoApiResponse(recovered);
+      } else {
+        response.writes = 1;
+        response.successful = { records: 0, data: [] };
+        response.failed = { records, data };
+      }
+    }
+
+    // TODO: need to add channel name
+    writeResponseErrorToFailedData(response, this._channel);
+
+    return response;
+  }
+
+  async _send(payload) {
+    throw new Error(`Unimplemented.`);
+  }
+
+}
+
+export class ApiWriteChannel extends WriteChannel {
+
+  constructor(client, type, options = {}) {
+    super({
+      ...options,
+      sinkGate: normalizeApiSinkGateOptions(options),
+    });
+    this._client = client;
+    this._type = type;
+  }
+
+  async _runCustomTransform(event) {
+    switch (event.type) {
+      case 'data':
+        await this._runData(event);
+        return;
+    }
+    await super._runCustomTransform(event);
+  }
+
+  async _runData(event) {
+    await this.writeData(event);
+  }
+
+  _createWriteEvent(context) {
+    return trimObj({
+      ...super._createWriteEvent(context),
+      taskId: getTaskIdFromResponse(context.response),
+    });
+  }
+
+}
+
+function getTaskIdFromResponse(response) {
+  return getTaskIdFromResponse0(response) || getTaskIdFromResponse0(response.recovered);
+}
+
+function getTaskIdFromResponse0({ body } = {}) {
+  return body && body.data && body.data.task_id;
+}
+
+function maskApiKeyInMisoUrl(url) {
+  // mask the api_key from the url
+  return url.replace(/api_key=\w+/, 'api_key=****');
+}
+
+export function processMisoApiResponse(response) {
+  if (typeof response !== 'object') {
+    return response;
+  }
+  const { data: body, status, statusText, config = {} } = response;
+  const { method, url } = config;
+  return trimObj({
+    status,
+    statusText,
+    method,
+    url: maskApiKeyInMisoUrl(url),
+    body,
+  });
+}

package/src/channel/delete.js

@@ -0,0 +1,68 @@
+import { ChannelApiSink, ApiWriteChannel } from './api.js';
+import { batchDelete } from '../api/helpers.js';
+
+// channel //
+export default class DeleteChannel extends ApiWriteChannel {
+
+  constructor(client, type, { name = 'delete', ...options } = {}) {
+    super(client, type, {
+      ...options,
+      name,
+      buffer: normalizeBufferOptions(type, options),
+      sink: createSink(client, type, options),
+    });
+  }
+
+}
+
+// buffer //
+const DEFAULT_BUFFER_OPTIONS = Object.freeze({
+  payloadSuffix: ']}}',
+  payloadDelimiter: ',',
+  serialize: event => JSON.stringify(event.id),
+  recordCap: 1000,
+});
+
+function normalizeBufferOptions(type, {
+  bytesPerRequest,
+  recordsPerRequest,
+  ...options
+}) {
+  const key = type === 'products' ? 'product_ids' : 'user_ids';
+  options = {
+    ...DEFAULT_BUFFER_OPTIONS,
+    ...options,
+    payloadPrefix: `{"data":{"${key}":[`,
+  };
+  if (recordsPerRequest) {
+    options.recordCap = recordsPerRequest;
+  }
+  if (bytesPerRequest) {
+    options.byteCap = bytesPerRequest;
+  }
+  return options;
+}
+
+// sink //
+function createSink(client, type, options) {
+  switch (type) {
+    case 'users':
+    case 'products':
+      return new ChannelDeleteSink(client, { ...options, type });
+    default:
+      throw new Error(`Unrecognized type: ${type}`);
+  }
+}
+
+class ChannelDeleteSink extends ChannelApiSink {
+
+  constructor(client, options) {
+    super(client, options);
+  }
+
+  async _send(payload) {
+    const { type, params } = this._options;
+    return await batchDelete(this._client, type, payload, { params });
+  }
+
+}

package/src/channel/index.js

@@ -0,0 +1,2 @@
+export { default as DeleteChannel } from './delete.js';
+export { default as UploadChannel } from './upload.js';

package/src/channel/upload.js

@@ -0,0 +1,68 @@
+import { ChannelApiSink, ApiWriteChannel } from './api.js';
+import { upload } from '../api/helpers.js';
+
+// channel //
+export default class UploadChannel extends ApiWriteChannel {
+
+  constructor(client, type, { name = 'upload', ...options } = {}) {
+    super(client, type, {
+      ...options,
+      name,
+      buffer: normalizeBufferOptions(type, options),
+      sink: createSink(client, type, options),
+    });
+  }
+
+}
+
+// buffer //
+const DEFAULT_BUFFER_OPTIONS = Object.freeze({
+  payloadPrefix: '{"data":[',
+  payloadSuffix: ']}',
+  payloadDelimiter: ',',
+  byteCap: 1024 * 1024,
+});
+
+function normalizeBufferOptions(type, { recordsPerRequest, bytesPerRequest, ...options } = {}) {
+  options = { ...DEFAULT_BUFFER_OPTIONS, ...options };
+  switch (type) {
+    case 'users':
+    case 'products':
+      options.recordCap = recordsPerRequest || 200;
+      break;
+    case 'interactions':
+      options.recordCap = recordsPerRequest || 1000;
+      break;
+    default:
+      throw new Error(`Unrecognized type: ${type}`);
+  }
+  if (bytesPerRequest) {
+    options.byteCap = bytesPerRequest;
+  }
+  return options;
+}
+
+// sink //
+function createSink(client, type, options) {
+  switch (type) {
+    case 'users':
+    case 'products':
+    case 'interactions':
+      return new ChannelUploadSink(client, { ...options, type });
+    default:
+      throw new Error(`Unrecognized type: ${type}`);
+  }
+}
+
+class ChannelUploadSink extends ChannelApiSink {
+
+  constructor(client, options) {
+    super(client, options);
+  }
+
+  async _send(payload) {
+    const { type, dryRun, params } = this._options;
+    return await upload(this._client, type, payload, { recoverValidRecordsOn422: true, dryRun, params });
+  }
+
+}

package/src/client.js

@@ -28,6 +28,7 @@ function normalizeOptions(options) {
   if (typeof options === 'string') {
     options = { key: options };
   }
+  options.key = getApiKeyFromEnv(options.env, options.key);
   if (!options.key || typeof options.key !== 'string') {
     throw new Error(`API key is required.`);
   }
@@ -35,3 +36,10 @@ function normalizeOptions(options) {
 
   return options;
 }
+
+function getApiKeyFromEnv(env, key) {
+  if (!env) {
+    return key;
+  }
+  return process.env[`MISO_${env.toUpperCase()}_API_KEY`] || undefined;
+}

package/src/stream/api-sink.js

@@ -62,10 +62,12 @@ export default class ApiSink extends sink.BpsSink {
       if (error.response.recovered) {
         data.recovered = error.response.recovered;
       }
+      if (error.response.issues) {
+        data.issues = error.response.issues;
+      }
     }
 
     // keep track of service stats on successful calls
-    // TODO: handle recovered records?
     if (!data.errors) {
       this._serviceStats.track({ records, bytes, took: data.took });
     }

package/src/stream/delete.js

@@ -11,6 +11,8 @@ export default class DeleteStream extends stream.BufferedWriteStream {
     objectMode,
     heartbeatInterval,
     // sink
+    dryRun,
+    params,
     recordsPerSecond,
     // buffer
     recordsPerRequest,
@@ -35,6 +37,8 @@ export default class DeleteStream extends stream.BufferedWriteStream {
 
     this._sink = new DeleteSink(client, {
       type,
+      dryRun,
+      params,
       recordsPerSecond,
     });
 

package/src/stream/upload-sink.js

@@ -28,28 +28,12 @@ class UploadSink extends ApiSink {
 
 }
 
-class ExperimentEventUploadSink extends UploadSink {
-
-  constructor(client, options) {
-    super(client, { ...options, type: 'experiment-events' });
-  }
-
-  async _execute(payload) {
-    const { experimentId } = this._options;
-    const { data } = await this._client.api.experiments.uploadEvent(experimentId, payload);
-    return data;
-  }
-
-}
-
 export default function create(client, type, options) {
   switch (type) {
     case 'users':
     case 'products':
     case 'interactions':
       return new UploadSink(client, { ...options, type });
-    case 'experiment-events':
-      return new ExperimentEventUploadSink(client, options);
     default:
       throw new Error(`Unrecognized type: ${type}`);
   }

package/src/stream/upload.js

@@ -2,7 +2,6 @@ import { stream } from '@miso.ai/server-commons';
 import version from '../version.js';
 import createSink from './upload-sink.js';
 import createBuffer from './upload-buffer.js';
-import { process422ResponseBody } from '../api/helpers.js';
 
 export default class UploadStream extends stream.BufferedWriteStream {
 
@@ -67,13 +66,12 @@ export default class UploadStream extends stream.BufferedWriteStream {
 
     // if upload fails, emit extracted payload at response event
     if (message.event === 'response') {
-      // TODO: we can do these near recoverValidRecords()
       const { response, payload } = args;
       if (payload) {
         output.payload = JSON.parse(payload);
-        if (response && response.status === 422) {
-          output.issues = process422ResponseBody(payload, response);
-        }
+      }
+      if (response && response.issues) {
+        output.issues = response.issues;
       }
     }
 

package/src/version.js

@@ -1 +1 @@
-export default '0.6.6-beta.6';
+export default '0.6.6';

package/src/api/experiments.js

@@ -1,19 +0,0 @@
-import { buildUrl } from './helpers.js';
-
-export default class Experiments {
-
-  constructor(client) {
-    this._client = client;
-  }
-
-  async uploadEvent(experimentId, record) {
-    // TODO: support non-string record
-    const url = buildUrl(this._client, `experiments/${experimentId}/events`);
-    // TODO: make content type header global
-    const headers = { 'Content-Type': 'application/json' };
-    const response = await this._client._axios.post(url, record, { headers });
-    // 200 response body does not have .data layer
-    return response.data ? response : { data: response };
-  }
-
-}