@miso.ai/server-sdk 0.6.6-beta.17 → 0.6.6-beta.18

package/cli/delete.js

@@ -1,21 +1,11 @@
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
-import { log, stream } from '@miso.ai/server-commons';
+import { log, stream, splitObj } from '@miso.ai/server-commons';
 import { MisoClient, logger } from '../src/index.js';
+import { buildForWrite } from './utils.js';
 
 function build(yargs) {
-  return yargs
-    .option('records-per-request', {
-      alias: ['rpr'],
-      describe: 'How many records to send in a request',
-    })
-    .option('records-per-second', {
-      alias: ['rps'],
-      describe: 'How many records to send per second',
-    })
-    .option('debug', {
-      describe: 'Set log level to debug',
-      type: 'boolean',
-    })
+  return buildForWrite(yargs)
     .option('progress', {
       alias: ['p'],
       describe: 'Set log format progress',
@@ -37,6 +27,32 @@ const run = type => async ({
   env,
   key,
   server,
+  channel,
+  ...options
+}) => {
+  const { debug } = options;
+  const client = new MisoClient({ env, key, server, debug });
+
+  if (channel) {
+    await runChannel(client, type, options);
+  } else {
+    await runStream(client, type, options);
+  }
+};
+
+async function runChannel(client, type, options) {
+  const [deleteOptions] = splitObj(options, ['params', 'requestsPerSecond', 'bytesPerSecond', 'recordsPerRequest', 'bytesPerRequest', 'debug']);
+  const deleteChannel = client.api[type].deleteChannel(deleteOptions);
+
+  await pipeline(
+    process.stdin,
+    split2(JSON.parse),
+    deleteChannel,
+    new stream.OutputStream({ objectMode: true }),
+  );
+}
+
+async function runStream(client, type, {
   param: params,
   ['records-per-request']: recordsPerRequest,
   ['records-per-second']: recordsPerSecond,
@@ -45,13 +61,10 @@ const run = type => async ({
   ['stream-name']: name,
   ['log-level']: loglevel,
   ['log-format']: logFormat,
-}) => {
-
+}) {
   loglevel = (debug || progress) ? log.DEBUG : loglevel;
   logFormat = progress ? logger.FORMAT.PROGRESS : logFormat;
 
-  const client = new MisoClient({ env, key, server, debug });
-
   const deleteStream = client.api[type].deleteStream({
     name,
     params,
@@ -67,13 +80,13 @@ const run = type => async ({
     format: logFormat,
   });
 
-  await stream.pipeline(
+  await pipeline(
     process.stdin,
     split2(),
     deleteStream,
     logStream,
   );
-};
+}
 
 export default function(type) {
   return {

package/cli/hybrid-search.js

@@ -1,4 +1,5 @@
 import { Readable } from 'stream';
+import { pipeline } from 'stream/promises';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
 import { buildForApi, buildForSearch } from './utils.js';
@@ -21,7 +22,7 @@ async function run({ query, fq, fl, rows, answer, env, key, server, debug }) {
   const readStream = Readable.from(products);
   const outputStream = new stream.OutputStream();
 
-  await stream.pipeline(
+  await pipeline(
     readStream,
     outputStream,
   );

package/cli/ids-diff.js

@@ -1,3 +1,4 @@
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
@@ -41,7 +42,7 @@ const run = type => async ({
   const diffStream = new stream.DiffStream(misoIds, { output });
   const outputStream = new stream.OutputStream({ objectMode: false });
 
-  await stream.pipeline(
+  await pipeline(
     process.stdin,
     split2(),
     diffStream,

package/cli/ids.js

@@ -1,4 +1,5 @@
 import { Readable } from 'stream';
+import { pipeline } from 'stream/promises';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
 import diff from './ids-diff.js';
@@ -38,7 +39,7 @@ const run = type => async ({
   const readStream = Readable.from(ids);
   const outputStream = new stream.OutputStream();
 
-  await stream.pipeline(
+  await pipeline(
     readStream,
     outputStream,
   );

package/cli/index.js

@@ -12,6 +12,7 @@ import status from './status.js';
 import get from './get.js';
 import search from './search.js';
 import hybridSearch from './hybrid-search.js';
+import upgrade from './upgrade.js';
 
 const interactions = {
   command: 'interactions',
@@ -54,6 +55,7 @@ yargs.build(yargs => {
     .command(products)
     .command(users)
     .command(transform)
+    .command(upgrade)
     .command(mergeLocal)
     .command(search)
     .command(hybridSearch)

package/cli/merge.js

@@ -1,6 +1,7 @@
 import { join } from 'path';
 import { createReadStream } from 'fs';
 import { createGunzip } from 'zlib';
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
@@ -36,10 +37,9 @@ const run = type => async ({
   const client = new MisoClient({ env, key, server, debug });
   const mergeStream = client.api[type].mergeStream({ ...options, mergeFn, records });
   const outputStream = new stream.OutputStream({ objectMode: true });
-  await stream.pipeline(
+  await pipeline(
     process.stdin,
-    split2(),
-    stream.parse(),
+    split2(JSON.parse),
     mergeStream,
     outputStream
   );

package/cli/search.js

@@ -1,4 +1,5 @@
 import { Readable } from 'stream';
+import { pipeline } from 'stream/promises';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
 import { buildForApi, buildForSearch } from './utils.js';
@@ -16,7 +17,7 @@ async function run({ query, fq, fl, rows, env, key, server, debug }) {
   const readStream = Readable.from(records);
   const outputStream = new stream.OutputStream();
 
-  await stream.pipeline(
+  await pipeline(
     readStream,
     outputStream,
   );

package/cli/status.js

@@ -1,3 +1,4 @@
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
@@ -31,14 +32,14 @@ async function runOne(client, type, taskId) {
 }
 
 async function runStream(client, type) {
-  await stream.pipeline([
+  await pipeline(
     process.stdin,
     split2(),
     client.api[type].statusStream(),
     new stream.OutputStream({
       objectMode: true,
     }),
-  ]);
+  );
 }
 
 export default function(type) {

package/cli/transform.js

@@ -1,3 +1,4 @@
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
 import { stream } from '@miso.ai/server-commons';
 
@@ -11,19 +12,15 @@ function build(yargs) {
 
 async function run({ file }) {
   const transform = await stream.getTransformStream(file);
-  const streams = [
-    process.stdin,
-    split2(),
-  ];
-  if (transform.writableObjectMode) {
-    streams.push(stream.parse());
-  }
-  streams.push(transform);
-  streams.push(new stream.OutputStream({
-    objectMode: transform.readableObjectMode,
-  }));
 
-  await stream.pipeline(...streams);
+  await pipeline(
+    process.stdin,
+    transform.writableObjectMode ? split2(JSON.parse) : split2(),
+    transform,
+    new stream.OutputStream({
+      objectMode: transform.readableObjectMode,
+    }),
+  );
 }
 
 export default {

package/cli/upgrade.js

@@ -0,0 +1,37 @@
+import { pipeline } from 'stream/promises';
+import split2 from 'split2';
+import { stream, UpgradeChannel, splitObj } from '@miso.ai/server-commons';
+
+function build(yargs) {
+  return yargs
+    .option('name', {
+      describe: 'Channel name',
+      type: 'string',
+    })
+    .option('id-field', {
+      alias: 'id',
+      describe: 'Payload field to use as ID',
+      type: 'string',
+    })
+    .option('as-id', {
+      describe: 'Upgrade as ID',
+      type: 'boolean',
+    });
+}
+
+async function run(options) {
+  const [upgradeOptions] = splitObj(options, ['name', 'asId', 'idField']);
+  await pipeline(
+    process.stdin,
+    split2(),
+    new UpgradeChannel({ objectMode: false, ...upgradeOptions }),
+    new stream.OutputStream({ objectMode: true }),
+  );
+}
+
+export default {
+  command: ['upgrade', 'up'],
+  description: `Upgrade the stream to a channel`,
+  builder: build,
+  handler: run,
+};

package/cli/upload.js

@@ -1,33 +1,20 @@
+import { pipeline } from 'stream/promises';
 import split2 from 'split2';
-import { log, stream } from '@miso.ai/server-commons';
+import { log, stream, splitObj } from '@miso.ai/server-commons';
 import { MisoClient, logger, normalize } from '../src/index.js';
+import { buildForWrite } from './utils.js';
 
 function build(yargs) {
-  return yargs
+  return buildForWrite(yargs)
     .option('dry-run', {
       alias: ['dry'],
       describe: 'Dry run mode',
+      type: 'boolean',
     })
     .option('lenient', {
       describe: 'Accept some lenient record schema',
       type: 'boolean',
     })
-    .option('requests-per-second', {
-      alias: ['rps'],
-      describe: 'How many requests to send per second',
-    })
-    .option('records-per-request', {
-      alias: ['rpr'],
-      describe: 'How many records to send in a request',
-    })
-    .option('bytes-per-request', {
-      alias: ['bpr'],
-      describe: 'How many bytes to send in a request',
-    })
-    .option('bytes-per-second', {
-      alias: ['bps'],
-      describe: 'How many bytes to send per second',
-    })
     .option('progress', {
       alias: ['p'],
       describe: 'Set log format progress',
@@ -49,6 +36,32 @@ const run = type => async ({
   env,
   key,
   server,
+  channel,
+  ...options
+}) => {
+  const { debug } = options;
+  const client = new MisoClient({ env, key, server, debug });
+
+  if (channel) {
+    await runChannel(client, type, options);
+  } else {
+    await runStream(client, type, options);
+  }
+};
+
+async function runChannel(client, type, options) {
+  const [uploadOptions] = splitObj(options, ['dryRun', 'params', 'requestsPerSecond', 'bytesPerSecond', 'recordsPerRequest', 'bytesPerRequest', 'debug']);
+  const uploadChannel = client.api[type].uploadChannel(uploadOptions);
+
+  await pipeline(
+    process.stdin,
+    split2(JSON.parse),
+    uploadChannel,
+    new stream.OutputStream({ objectMode: true }),
+  );
+}
+
+async function runStream(client, type, {
   param: params,
   ['dry-run']: dryRun,
   lenient,
@@ -56,19 +69,16 @@ const run = type => async ({
   ['records-per-request']: recordsPerRequest,
   ['bytes-per-request']: bytesPerRequest,
   ['bytes-per-second']: bytesPerSecond,
-  ['experiment-id']: experimentId,
-  debug,
   progress,
+  debug,
+  ['experiment-id']: experimentId,
   ['stream-name']: name,
   ['log-level']: loglevel,
   ['log-format']: logFormat,
-}) => {
-
+}) {
   loglevel = (debug || progress) ? log.DEBUG : loglevel;
   logFormat = progress ? logger.FORMAT.PROGRESS : logFormat;
 
-  const client = new MisoClient({ env, key, server, debug });
-
   const uploadStreamObjectMode = lenient;
 
   const uploadStream = client.api[type].uploadStream({
@@ -98,17 +108,18 @@ const run = type => async ({
   // lenient: stdin -> split2 -> parse -> normalize -> upload -> log
   // notice that the output of split2 are strings, while input/output of normalize are objects
 
-  await stream.pipeline(
+  await pipeline(
     process.stdin,
-    split2(),
     ...(lenient ? [
-      stream.parse(),
+      split2(JSON.parse),
       new normalize.Stream(type),
-    ] : []),
+    ] : [
+      split2(),
+    ]),
     uploadStream,
     logStream,
   );
-};
+}
 
 export default function(type) {
   return {

package/cli/utils.js

@@ -49,6 +49,36 @@ export function buildForSearch(yargs) {
     });
 }
 
+export function buildForWrite(yargs) {
+  return yargs
+    .option('channel', {
+      alias: ['c'],
+      describe: 'Use channel protocol',
+      type: 'boolean',
+      default: false,
+    })
+    .option('requests-per-second', {
+      alias: ['rps'],
+      describe: 'How many requests to send per second',
+      type: 'number',
+    })
+    .option('bytes-per-second', {
+      alias: ['bps'],
+      describe: 'How many bytes to send per second',
+      type: 'number',
+    })
+    .option('records-per-request', {
+      alias: ['rpr'],
+      describe: 'How many records to send in a request',
+      type: 'number',
+    })
+    .option('bytes-per-request', {
+      alias: ['bpr'],
+      describe: 'How many bytes to send in a request',
+      type: 'number',
+    });
+}
+
 export function formatError(err) {
   const { response } = err;
   if (response) {

package/package.json

@@ -16,12 +16,12 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.6-beta.17",
+    "@miso.ai/server-commons": "0.6.6-beta.18",
     "axios": "^1.6.2",
     "axios-retry": "^4.5.0",
     "dotenv": "^16.0.1",
     "split2": "^4.1.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.6-beta.17"
+  "version": "0.6.6-beta.18"
 }

package/src/api/base.js

@@ -4,6 +4,7 @@ import UploadStream from '../stream/upload.js';
 import DeleteStream from '../stream/delete.js';
 import StatusStream from '../stream/status.js';
 import MergeStream from '../stream/merge.js';
+import { UploadChannel, DeleteChannel } from '../channel/index.js';
 
 export class Queries {
 
@@ -35,6 +36,10 @@ export class Writable {
     return new UploadStream(this._client, this._type, options);
   }
 
+  uploadChannel(options = {}) {
+    return new UploadChannel(this._client, this._type, options);
+  }
+
 }
 
 export class Entities extends Writable {
@@ -62,6 +67,10 @@ export class Entities extends Writable {
     return new DeleteStream(this._client, this._type, options);
   }
 
+  deleteChannel(options = {}) {
+    return new DeleteChannel(this._client, this._type, options);
+  }
+
   async status(taskId) {
     const url = buildUrl(this._client, `${this._type}/_status/${taskId}`);
     return (await this._client._axios.get(url)).data;

package/src/api/helpers.js

@@ -34,16 +34,18 @@ async function recoverValidRecords(client, type, records, options, response) {
   }
   const url = buildUrl(client, type, options);
   const validPayload = buildUploadPayload(validRecords);
+  let secondResponse;
   try {
-    await client._axios.post(url, validPayload);
+    secondResponse = await client._axios.post(url, validPayload);
   } catch (_) {
     return; // still fail, never mind...
   }
-  response.recovered = {
+  response.recovered = trimObj({
+    ...secondResponse,
     product_ids: validRecords.map(record => record.product_id),
     records: validRecords.length,
     bytes: validPayload.length * 2,
-  };
+  });
 }
 
 export async function merge(client, type, record, { mergeFn = defaultMerge } = {}) {

package/src/channel/api.js

@@ -0,0 +1,194 @@
+import { WriteChannelSink, WriteChannel, trimObj } from '@miso.ai/server-commons';
+
+export function normalizeApiSinkGateOptions({
+  writesPerSecond = 10,
+  recordsPerSecond = 100000,
+  bytesPerSecond = 100 * 1024 * 1024,
+  ...options
+} = {}) {
+  return {
+    writesPerSecond,
+    recordsPerSecond,
+    bytesPerSecond,
+    ...options,
+  };
+}
+
+function normalizeParams(params) {
+  if (!params || params.length === 0) {
+    return undefined;
+  }
+  return params.reduce((acc, param) => {
+    const [key, value = '1'] = param.split('=');
+    acc[key] = value;
+    return acc;
+  }, {});
+}
+
+function splitData(data, ids) {
+  ids = new Set(ids);
+  const positive = [];
+  const negative = [];
+  for (const record of data) {
+    if (ids.has(record.id)) {
+      positive.push(record);
+    } else {
+      negative.push(record);
+    }
+  }
+  return [positive, negative];
+}
+
+function writeIssuesToData(data, { groups = [] } = {}, { name: channel } = {}) {
+  for (const { index, violations = [] } of groups) {
+    try {
+      const errors = (data[index].errors || (data[index].errors = []));
+      errors.push(...(violations.map(v => trimObj({ channel, status: 422, ...v }))));
+    } catch(_) {}
+  }
+}
+
+function writeResponseErrorToFailedData(response, { name: channel } = {}) {
+  // 422 errors are already handled by writeIssuesToData
+  if (!response.error && (response.status < 400 || response.status === 422)) {
+    return;
+  }
+  const error = trimObj({
+    channel, 
+    status: response.status,
+    message: response.error || response.statusText,
+  });
+  for (const event of response.failed.data) {
+    (event.errors || (event.errors = [])).push({ ...error });
+  }
+}
+
+export class ChannelApiSink extends WriteChannelSink {
+
+  constructor(client, options) {
+    super(options);
+    this._client = client;
+    // TODO: take axios options?
+  }
+
+  _normalizeOptions({
+    params,
+    dryRun,
+    ...options
+  } = {}) {
+    return trimObj({
+      ...super._normalizeOptions(options),
+      params: normalizeParams(params),
+      dryRun: !!dryRun,
+    });
+  }
+
+  async _write(request) {
+    const { payload, records, data } = request;
+    let response;
+    try {
+      response = await this._send(payload);
+      response = processMisoApiResponse(response);
+      response.writes = 1;
+      response.successful = { records, data };
+      response.failed = { records: 0, data: [] };
+    } catch(error) {
+      // not axios-handled error
+      if (!error.response) {
+        throw error;
+      }
+      response = processMisoApiResponse(error.response);
+      if (typeof response !== 'object') {
+        response = trimObj({ error: response });
+      }
+      const { recovered, issues } = error.response;
+      // write issues into failed data events
+      // TODO: need to add channel name
+      writeIssuesToData(data, issues, this._channel);
+      // TODO: handle issue.unrecognized
+
+      if (recovered) {
+        const [positive, negative] = splitData(data, recovered.product_ids);
+        response.writes = 2;
+        response.successful = { records: positive.length, data: positive };
+        response.failed = { records: negative.length, data: negative };
+        response.recovered = processMisoApiResponse(recovered);
+      } else {
+        response.writes = 1;
+        response.successful = { records: 0, data: [] };
+        response.failed = { records, data };
+      }
+    }
+
+    // TODO: need to add channel name
+    writeResponseErrorToFailedData(response, this._channel);
+
+    return response;
+  }
+
+  async _send(payload) {
+    throw new Error(`Unimplemented.`);
+  }
+
+}
+
+export class ApiWriteChannel extends WriteChannel {
+
+  constructor(client, type, options = {}) {
+    super({
+      ...options,
+      sinkGate: normalizeApiSinkGateOptions(options),
+    });
+    this._client = client;
+    this._type = type;
+  }
+
+  async _runCustomTransform(event) {
+    switch (event.type) {
+      case 'data':
+        await this._runData(event);
+        return;
+    }
+    await super._runCustomTransform(event);
+  }
+
+  async _runData(event) {
+    await this.writeData(event);
+  }
+
+  _createWriteEvent(context) {
+    return trimObj({
+      ...super._createWriteEvent(context),
+      taskId: getTaskIdFromResponse(context.response),
+    });
+  }
+
+}
+
+function getTaskIdFromResponse(response) {
+  return getTaskIdFromResponse0(response) || getTaskIdFromResponse0(response.recovered);
+}
+
+function getTaskIdFromResponse0({ body } = {}) {
+  return body && body.data && body.data.task_id;
+}
+
+function maskApiKeyInMisoUrl(url) {
+  // mask the api_key from the url
+  return url.replace(/api_key=\w+/, 'api_key=****');
+}
+
+export function processMisoApiResponse(response) {
+  if (typeof response !== 'object') {
+    return response;
+  }
+  const { data: body, status, statusText, config = {} } = response;
+  const { method, url } = config;
+  return trimObj({
+    status,
+    statusText,
+    method,
+    url: maskApiKeyInMisoUrl(url),
+    body,
+  });
+}

package/src/channel/delete.js

@@ -0,0 +1,68 @@
+import { ChannelApiSink, ApiWriteChannel } from './api.js';
+import { batchDelete } from '../api/helpers.js';
+
+// channel //
+export default class DeleteChannel extends ApiWriteChannel {
+
+  constructor(client, type, { name = 'delete', ...options } = {}) {
+    super(client, type, {
+      ...options,
+      name,
+      buffer: normalizeBufferOptions(type, options),
+      sink: createSink(client, type, options),
+    });
+  }
+
+}
+
+// buffer //
+const DEFAULT_BUFFER_OPTIONS = Object.freeze({
+  payloadSuffix: ']}}',
+  payloadDelimiter: ',',
+  serialize: event => JSON.stringify(event.id),
+  recordCap: 1000,
+});
+
+function normalizeBufferOptions(type, {
+  bytesPerRequest,
+  recordsPerRequest,
+  ...options
+}) {
+  const key = type === 'products' ? 'product_ids' : 'user_ids';
+  options = {
+    ...DEFAULT_BUFFER_OPTIONS,
+    ...options,
+    payloadPrefix: `{"data":{"${key}":[`,
+  };
+  if (recordsPerRequest) {
+    options.recordCap = recordsPerRequest;
+  }
+  if (bytesPerRequest) {
+    options.byteCap = bytesPerRequest;
+  }
+  return options;
+}
+
+// sink //
+function createSink(client, type, options) {
+  switch (type) {
+    case 'users':
+    case 'products':
+      return new ChannelDeleteSink(client, { ...options, type });
+    default:
+      throw new Error(`Unrecognized type: ${type}`);
+  }
+}
+
+class ChannelDeleteSink extends ChannelApiSink {
+
+  constructor(client, options) {
+    super(client, options);
+  }
+
+  async _send(payload) {
+    const { type, params } = this._options;
+    return await batchDelete(this._client, type, payload, { params });
+  }
+
+}

package/src/channel/index.js

@@ -0,0 +1,2 @@
+export { default as DeleteChannel } from './delete.js';
+export { default as UploadChannel } from './upload.js';

package/src/channel/upload.js

@@ -0,0 +1,68 @@
+import { ChannelApiSink, ApiWriteChannel } from './api.js';
+import { upload } from '../api/helpers.js';
+
+// channel //
+export default class UploadChannel extends ApiWriteChannel {
+
+  constructor(client, type, { name = 'upload', ...options } = {}) {
+    super(client, type, {
+      ...options,
+      name,
+      buffer: normalizeBufferOptions(type, options),
+      sink: createSink(client, type, options),
+    });
+  }
+
+}
+
+// buffer //
+const DEFAULT_BUFFER_OPTIONS = Object.freeze({
+  payloadPrefix: '{"data":[',
+  payloadSuffix: ']}',
+  payloadDelimiter: ',',
+  byteCap: 1024 * 1024,
+});
+
+function normalizeBufferOptions(type, { recordsPerRequest, bytesPerRequest, ...options } = {}) {
+  options = { ...DEFAULT_BUFFER_OPTIONS, ...options };
+  switch (type) {
+    case 'users':
+    case 'products':
+      options.recordCap = recordsPerRequest || 200;
+      break;
+    case 'interactions':
+      options.recordCap = recordsPerRequest || 1000;
+      break;
+    default:
+      throw new Error(`Unrecognized type: ${type}`);
+  }
+  if (bytesPerRequest) {
+    options.byteCap = bytesPerRequest;
+  }
+  return options;
+}
+
+// sink //
+function createSink(client, type, options) {
+  switch (type) {
+    case 'users':
+    case 'products':
+    case 'interactions':
+      return new ChannelUploadSink(client, { ...options, type });
+    default:
+      throw new Error(`Unrecognized type: ${type}`);
+  }
+}
+
+class ChannelUploadSink extends ChannelApiSink {
+
+  constructor(client, options) {
+    super(client, options);
+  }
+
+  async _send(payload) {
+    const { type, dryRun, params } = this._options;
+    return await upload(this._client, type, payload, { recoverValidRecordsOn422: true, dryRun, params });
+  }
+
+}

package/src/version.js

@@ -1 +1 @@
-export default '0.6.6-beta.17';
+export default '0.6.6-beta.18';