@miso.ai/server-sdk 0.6.5-beta.1 → 0.6.5-beta.11

package/cli/delete.js

@@ -49,7 +49,7 @@ const run = type => async ({
   loglevel = (debug || progress) ? log.DEBUG : loglevel;
   logFormat = progress ? logger.FORMAT.PROGRESS : logFormat;
 
-  const client = new MisoClient({ key, server });
+  const client = new MisoClient({ key, server, debug });
 
   const deleteStream = client.api[type].deleteStream({
     name,

package/cli/get.js

@@ -1,4 +1,5 @@
 import { MisoClient } from '../src/index.js';
+import { formatError } from './utils.js';
 
 const build = yargs => yargs;
 
@@ -8,9 +9,13 @@ const run = type => async ({
   id,
   debug,
 }) => {
-  const client = new MisoClient({ key, server });
-  const entity = await client.api[type].get(id);
-  console.log(JSON.stringify(entity));
+  const client = new MisoClient({ key, server, debug });
+  try {
+    const entity = await client.api[type].get(id);
+    console.log(JSON.stringify(entity));
+  } catch (err) {
+    console.error(formatError(err));
+  }
 };
 
 export default function(type) {

package/cli/hybrid-search.js

@@ -0,0 +1,36 @@
+import { Readable } from 'stream';
+import { stream } from '@miso.ai/server-commons';
+import { MisoClient } from '../src/index.js';
+import { buildForApi, buildForSearch } from './utils.js';
+
+function build(yargs) {
+  return buildForSearch(buildForApi(yargs))
+    .positional('query', {
+      describe: 'Search query',
+    })
+    .option('answer', {
+      type: 'boolean',
+      describe: 'Return answer',
+      default: false,
+    });
+}
+
+async function run({ query, fq, fl, rows, answer, key, server, debug }) {
+  const client = new MisoClient({ key, server, debug });
+  const { products } = await client.api.ask.search({ q: query, fq, fl, rows, answer });
+  const readStream = Readable.from(products);
+  const outputStream = new stream.OutputStream();
+
+  await stream.pipeline(
+    readStream,
+    outputStream,
+  );
+}
+
+export default {
+  command: 'hybrid-search [query]',
+  aliases: ['hs'],
+  description: `Hybrid search records by a query.`,
+  builder: build,
+  handler: run,
+};

package/cli/ids-diff.js

@@ -30,10 +30,11 @@ const run = type => async ({
   output,
   plus,
   minus,
+  debug,
 }) => {
   output = output || (plus ? 'plus' : minus ? 'minus' : undefined);
 
-  const client = new MisoClient({ key, server });
+  const client = new MisoClient({ key, server, debug });
   const misoIds = await client.api[type].ids();
 
   const diffStream = new stream.DiffStream(misoIds, { output });

package/cli/ids.js

@@ -22,8 +22,9 @@ const run = type => async ({
   key,
   server,
   type: recordType,
+  debug,
 }) => {
-  const client = new MisoClient({ key, server });
+  const client = new MisoClient({ key, server, debug });
   let ids;
   try {
     const options = recordType ? { type: recordType } : {};
@@ -45,7 +46,7 @@ const run = type => async ({
 export default function(type) {
   return {
     command: 'ids',
-    description: false,
+    description: `Get all ids in the catalog`,
     builder: build(type),
     handler: run(type),
   };

package/cli/index.js

@@ -9,6 +9,8 @@ import ids from './ids.js';
 import transform from './transform.js';
 import status from './status.js';
 import get from './get.js';
+import search from './search.js';
+import hybridSearch from './hybrid-search.js';
 
 const interactions = {
   command: 'interactions',
@@ -68,5 +70,7 @@ yargs.build(yargs => {
     .command(users)
     .command(experiments)
     .command(transform)
+    .command(search)
+    .command(hybridSearch)
     .version(MisoClient.version);
 });

package/cli/merge.js

@@ -27,11 +27,12 @@ const run = type => async ({
   server,
   file,
   base,
+  debug,
   ...options
 }) => {
   const mergeFn = await getMergeFn(file);
   const records = await buildBaseRecords(base);
-  const client = new MisoClient({ key, server });
+  const client = new MisoClient({ key, server, debug });
   const mergeStream = client.api[type].mergeStream({ ...options, mergeFn, records });
   const outputStream = new stream.OutputStream({ objectMode: true });
   await stream.pipeline(

package/cli/search.js

@@ -0,0 +1,30 @@
+import { Readable } from 'stream';
+import { stream } from '@miso.ai/server-commons';
+import { MisoClient } from '../src/index.js';
+import { buildForApi, buildForSearch } from './utils.js';
+
+function build(yargs) {
+  return buildForSearch(buildForApi(yargs))
+    .positional('query', {
+      describe: 'Search query',
+    });
+}
+
+async function run({ query, fq, fl, rows, key, server, debug }) {
+  const client = new MisoClient({ key, server, debug });
+  const records = await client.api.search.search({ q: query, fq, fl, rows });
+  const readStream = Readable.from(records);
+  const outputStream = new stream.OutputStream();
+
+  await stream.pipeline(
+    readStream,
+    outputStream,
+  );
+}
+
+export default {
+  command: 'search [query]',
+  description: `Search records by a query.`,
+  builder: build,
+  handler: run,
+};

package/cli/status.js

@@ -10,8 +10,9 @@ const run = type => async ({
   key,
   server,
   taskId,
+  debug,
 }) => {
-  const client = new MisoClient({ key, server });
+  const client = new MisoClient({ key, server, debug });
   if (taskId) {
     runOne(client, type, taskId);
   } else {
@@ -21,7 +22,7 @@ const run = type => async ({
 
 async function runOne(client, type, taskId) {
   try {
-    console.log(await client.api[type].status(taskId));
+    console.log(JSON.stringify(await client.api[type].status(taskId)));
   } catch (err) {
     console.error(err);
     throw err;

package/cli/upload.js

@@ -61,7 +61,7 @@ const run = type => async ({
   loglevel = (debug || progress) ? log.DEBUG : loglevel;
   logFormat = progress ? logger.FORMAT.PROGRESS : logFormat;
 
-  const client = new MisoClient({ key, server });
+  const client = new MisoClient({ key, server, debug });
 
   const uploadStreamObjectMode = lenient;
 

package/cli/utils.js

@@ -1,3 +1,5 @@
+import { yargs as _yargs } from '@miso.ai/server-commons';
+
 export function buildForApi(yargs) {
   return yargs
     .option('key', {
@@ -12,7 +14,7 @@ export function buildForApi(yargs) {
       alias: ['v', 'var'],
       describe: 'Extra URL parameters',
       type: 'array',
-      coerce: yargs.coerceToArray,
+      coerce: _yargs.coerceToArray,
     })
     .option('debug', {
       describe: 'Set log level to debug',
@@ -20,3 +22,35 @@ export function buildForApi(yargs) {
     })
     .demandOption(['key'], 'API key is required.');
 }
+
+export function buildForSearch(yargs) {
+  return yargs
+    .option('fq', {
+      type: 'string',
+      describe: 'Filter query',
+    })
+    .option('fl', {
+      type: 'string',
+      coerce: _yargs.coerceToArray,
+      describe: 'Fields to return',
+    })
+    .option('rows', {
+      alias: ['n'],
+      type: 'number',
+      describe: 'Number of rows to return',
+    })
+    .option('start', {
+      alias: ['s'],
+      type: 'number',
+      describe: 'Start index',
+    });
+}
+
+export function formatError(err) {
+  const { response } = err;
+  if (response) {
+    const { data, status } = response;
+    return { errors: true, status, ...data };
+  }
+  return { errors: true, message: err.message };
+}

package/package.json

@@ -16,11 +16,12 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.5-beta.1",
+    "@miso.ai/server-commons": "0.6.5-beta.11",
     "axios": "^1.6.2",
+    "axios-retry": "^4.5.0",
     "dotenv": "^16.0.1",
     "split2": "^4.1.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.5-beta.1"
+  "version": "0.6.5-beta.11"
 }

package/src/api/ask.js

@@ -0,0 +1,13 @@
+import { Queries } from './base.js';
+
+export default class Ask extends Queries {
+
+  constructor(client) {
+    super(client, 'ask');
+  }
+
+  async search(payload, options = {}) {
+    return this._run('search', payload, options);
+  }
+
+}

package/src/api/base.js

@@ -1,4 +1,3 @@
-import axios from 'axios';
 import { asArray } from '@miso.ai/server-commons';
 import { upload, merge, batchDelete, buildUrl } from './helpers.js';
 import UploadStream from '../stream/upload.js';
@@ -16,7 +15,7 @@ export class Queries {
   async _run(path, payload, options) {
     // TODO: options
     const url = buildUrl(this._client, `${this._group}/${path}`);
-    return (await axios.post(url, payload)).data.data;
+    return (await this._client._axios.post(url, payload)).data.data;
   }
 
 }
@@ -46,13 +45,13 @@ export class Entities extends Writable {
 
   async get(id) {
     const url = buildUrl(this._client, `${this._type}/${id}`);
-    return (await axios.get(url)).data.data;
+    return (await this._client._axios.get(url)).data.data;
   }
 
   async ids({ type } = {}) {
     const options = type ? { params: { type } } : {};
     const url = buildUrl(this._client, `${this._type}/_ids`, options);
-    return (await axios.get(url)).data.data.ids;
+    return (await this._client._axios.get(url)).data.data.ids;
   }
 
   async delete(ids, options = {}) {
@@ -65,7 +64,7 @@ export class Entities extends Writable {
 
   async status(taskId) {
     const url = buildUrl(this._client, `${this._type}/_status/${taskId}`);
-    return (await axios.get(url)).data;
+    return (await this._client._axios.get(url)).data;
   }
 
   statusStream() {

package/src/api/experiments.js

@@ -1,4 +1,3 @@
-import axios from 'axios';
 import { buildUrl } from './helpers.js';
 
 export default class Experiments {
@@ -12,7 +11,7 @@ export default class Experiments {
     const url = buildUrl(this._client, `experiments/${experimentId}/events`);
     // TODO: make content type header global
     const headers = { 'Content-Type': 'application/json' };
-    const response = await axios.post(url, record, { headers });
+    const response = await this._client._axios.post(url, record, { headers });
     // 200 response body does not have .data layer
     return response.data ? response : { data: response };
   }

package/src/api/helpers.js

@@ -1,11 +1,45 @@
 import { asArray, trimObj, computeIfAbsent } from '@miso.ai/server-commons';
-import axios from 'axios';
 import { Buffer } from 'buffer';
 
 export async function upload(client, type, records, options = {}) {
-  const url = buildUrl(client, type, { ...options, async: true });
+  const url = buildUrl(client, type, options);
   const payload = buildUploadPayload(records);
-  return axios.post(url, payload);
+  try {
+    // await to catch errors
+    return await client._axios.post(url, payload);
+  } catch (error) {
+    await recoverValidRecords(client, type, records, options, error.response);
+    throw error;
+  }
+}
+
+async function recoverValidRecords(client, type, records, options, response) {
+  if (!response || response.status !== 422 || !options.recoverValidRecordsOn422) {
+    return;
+  }
+  records = extractRecordsFromUploadPayload(records);
+  // try to collect valid records and resend them, which should pass the validation
+  const { groups = [], unrecognized = [] } = process422ResponseBody(records, response.data); // it takes records too
+  if (groups.length === 0 || groups.length === records.length || unrecognized.length > 0) {
+    // if there are unrecognized messages, it's hard to tell which records are valid
+    return;
+  }
+  const invalidIndices = new Set(groups.map(group => group.index));
+  const validRecords = records.filter((_, index) => !invalidIndices.has(index));
+  if (validRecords.length === 0) {
+    return; // should not be, just in case
+  }
+  const url = buildUrl(client, type, options);
+  const validPayload = buildUploadPayload(validRecords);
+  try {
+    await client._axios.post(url, validPayload);
+  } catch (_) {
+    return; // still fail, never mind...
+  }
+  response.recovered = {
+    records: validRecords.length,
+    bytes: validPayload.length * 2,
+  };
 }
 
 export async function merge(client, type, record, { mergeFn = defaultMerge } = {}) {
@@ -62,10 +96,10 @@ export function shimRecordForMerging(record) {
   return record;
 }
 
-const RE_422_MSG_LINE = /^\s*data\.(\d+)(?:\.(\S+))?\s+is\s+invalid\.\s+(.*)$/;
+const RE_422_MSG_LINE = /^\s*data\.(\d+)(?:\.(\S+))?\s+(?:\(([^)]*)\)\s+)?is\s+invalid\.\s+(.*)$/;
 
 export function process422ResponseBody(payload, { data } = {}) {
-  const records = extractUploadPayload(payload);
+  const records = extractRecordsFromUploadPayload(payload);
   const unrecognized = [];
   const groupsMap = new Map();
   for (const line of data) {
@@ -74,7 +108,7 @@ export function process422ResponseBody(payload, { data } = {}) {
       unrecognized.push(line);
       continue;
     }
-    let [_, index, path, message] = m;
+    let [_, index, path, value, message] = m;
     index = Number(index);
     const { violations } = computeIfAbsent(groupsMap, index, index => ({
       index,
@@ -83,6 +117,7 @@ export function process422ResponseBody(payload, { data } = {}) {
     }));
     violations.push({
       path,
+      value,
       message,
     });
   }
@@ -98,12 +133,7 @@ export function process422ResponseBody(payload, { data } = {}) {
 export async function batchDelete(client, type, ids, options = {}) {
   const url = buildUrl(client, `${type}/_delete`, { ...options, async: true });
   const payload = buildBatchDeletePayload(type, ids);
-  // TODO: organize axios
-  const { data } = await axios.post(url, payload, {
-    headers: {
-      'Content-Type': 'application/json',
-    },
-  });
+  const { data } = await client._axios.post(url, payload);
   return data;
 }
 
@@ -127,10 +157,10 @@ export function buildUrl(client, path, { async, dryRun, params: extraParams } =
 export function buildUploadPayload(records) {
   return typeof records === 'string' ? records :
     Buffer.isBuffer(records) ? records.toString() :
-    { data: Array.isArray(records) ? records : [records] };
+    JSON.stringify({ data: Array.isArray(records) ? records : [records] });
 }
 
-export function extractUploadPayload(records) {
+export function extractRecordsFromUploadPayload(records) {
   if (Buffer.isBuffer(records)) {
     records = records.toString();
   }

package/src/api/index.js

@@ -2,6 +2,7 @@ import Products from './products.js';
 import Users from './users.js';
 import Interactions from './interactions.js';
 import Experiments from './experiments.js';
+import Ask from './ask.js';
 import Search from './search.js';
 import Recommendation from './recommendation.js';
 
@@ -12,6 +13,7 @@ export default class Api {
     this.users = new Users(client);
     this.interactions = new Interactions(client);
     this.experiments = new Experiments(client);
+    this.ask = new Ask(client);
     this.search = new Search(client);
     this.recommendation = new Recommendation(client);
   }

package/src/axios.js

@@ -0,0 +1,47 @@
+import axios from 'axios';
+import axiosRetry from 'axios-retry';
+import version from './version.js';
+
+const DEFAULT_RETRY_OPTIONS = {
+  retries: 5,
+  retryDelay: count => count * 500,
+  retryCondition: (error) => {
+    if (!error.response) {
+      return false;
+    }
+    const { status } = error.response;
+    // Only retry on 5xx or 429 server errors
+    return (status >= 500 && status < 600) || status === 429;
+  },
+};
+
+export function createAxios(options = {}, debug = false) {
+  if (typeof options.get === 'function' && typeof options.post === 'function') {
+    return options; // assume this is an axios instance already
+  }
+  const { retry } = options;
+  const instance = axios.create({
+    headers: {
+      'User-Agent': `MisoNodeSDK/${version}`,
+      'Content-Type': 'application/json',
+    },
+  });
+  axiosRetry(instance, { ...DEFAULT_RETRY_OPTIONS, ...retry });
+  if (debug) {
+    instance.interceptors.request.use(config => {
+      explainAsCurl(config);
+      return config;
+    });
+  }
+  return instance;
+}
+
+function explainAsCurl(config) {
+  // format into a curl command
+  const { method, url, data } = config;
+  let command = `curl -X ${method.toUpperCase()} '${url}'`;
+  if (data) {
+    command += ` -d '${typeof data === 'string' ? data : JSON.stringify(data)}'`;
+  }
+  console.log(`[Explain] ${command}`);
+}

package/src/client.js

@@ -1,12 +1,14 @@
 import version from './version.js';
 import Api from './api/index.js';
+import { createAxios } from './axios.js';
 
 export default class MisoClient {
 
   static version = version;
 
-  constructor(options) {
-    this._options = normalizeOptions(options);
+  constructor(options = {}) {
+    this._options = options = normalizeOptions(options);
+    this._axios = createAxios(options.axios, options.debug); // TODO: pass onRetry() for debug message
     this.version = version;
     this.api = new Api(this);
   }

package/src/stream/api-sink.js

@@ -52,9 +52,13 @@ export default class ApiSink extends sink.BpsSink {
       } else if (status) {
         data = { status, ...data };
       }
+      if (error.response.recovered) {
+        data.recovered = error.response.recovered;
+      }
     }
 
     // keep track of service stats on successful calls
+    // TODO: handle recovered records?
     if (!data.errors) {
       this._serviceStats.track({ records, bytes, took: data.took });
     }

package/src/stream/delete.js

@@ -68,7 +68,7 @@ export default class DeleteStream extends stream.BufferedWriteStream {
     const output = super._output(message, args);
 
     // if upload fails, emit extracted payload at response event
-    if (message.event === 'response' && args.response && args.response.errors && args.payload) {
+    if (message.event === 'response' && args.payload) {
       output.payload = JSON.parse(args.payload);
     }
 

package/src/stream/upload-sink.js

@@ -22,7 +22,7 @@ class UploadSink extends ApiSink {
 
   async _execute(payload) {
     const { type, dryRun, params } = this._options;
-    const { data } = await upload(this._client, type, payload, { dryRun, params });
+    const { data } = await upload(this._client, type, payload, { recoverValidRecordsOn422: true, dryRun, params });
     return data;
   }
 

package/src/stream/upload.js

@@ -65,12 +65,13 @@ export default class UploadStream extends stream.BufferedWriteStream {
 
     // if upload fails, emit extracted payload at response event
     if (message.event === 'response') {
+      // TODO: we can do these near recoverValidRecords()
       const { response, payload } = args;
       if (payload) {
         output.payload = JSON.parse(payload);
-      }
-      if (response && response.status === 422) {
-        output.issues = process422ResponseBody(payload, response);
+        if (response && response.status === 422) {
+          output.issues = process422ResponseBody(payload, response);
+        }
       }
     }
 

package/src/version.js

@@ -1 +1 @@
-export default '0.6.5-beta.1';
+export default '0.6.5-beta.11';