@miso.ai/server-sdk 0.6.4 → 0.6.5-beta.1

package/README.md

@@ -0,0 +1,95 @@
+# Miso Node.js SDK
+
+## Setup
+
+1. Install Node.js.
+
+2. Install the package locally:
+
+```bash
+npm i @miso.ai/server-sdk
+```
+
+Or, install the package globally:
+
+```bash
+npm i -g @miso.ai/server-sdk
+```
+
+Or, use `npx` to run the commands, which will guide the package installation.
+
+3. Put the following settings in your `.env` file:
+
+```env
+MISO_API_KEY=your_api_key
+```
+
+## Usage
+
+### Help message
+
+```bash
+miso --help
+miso products --help
+```
+
+### Get
+
+Get a product by `product_id`:
+
+```bash
+miso products get [id]
+```
+
+### Upload
+
+Given a JSON lines file of products:
+
+```jsonl
+{"product_id": "1", ...}
+{"product_id": "2", ...}
+...
+```
+
+Upload products by piping records into the command:
+
+```bash
+cat records.jsonl | miso products upload
+```
+
+You can dry run:
+
+```bash
+cat records.jsonl | head -20 | miso products upload --dry
+```
+
+When uploading a large amount of records, tt's recommended to display the progress status and pipe errors to a log file:
+
+```bash
+cat records.jsonl | miso products upload -p 2> error.log
+```
+
+The error log is a JSON line file consisting request payloads and response bodies:
+
+```jsonl
+{"response": { (Miso API response) }, "payload": { "data": [ ...(records uploaded) ] }}
+...
+```
+
+You can extract the failed records from the error log to work on them, so you don't need to reprocess the whole data set again.
+
+### Delete
+
+Given a file of product IDs:
+
+```txt
+product_id_1
+product_id_2
+...
+```
+
+Delete products by piping IDs into the command:
+
+```bash
+cat product_ids.txt | miso products delete -p 2> error.log
+```

package/cli/ids.js

@@ -4,18 +4,30 @@ import { MisoClient } from '../src/index.js';
 import diff from './ids-diff.js';
 
 const build = type => yargs => {
-  return yargs
+  yargs = yargs
     .command(diff(type));
+  // only works for products
+  if (type === 'products') {
+    yargs = yargs
+      .option('type', {
+        alias: ['t'],
+        describe: 'Only include record of given type',
+        type: 'string',
+      });
+  }
+  return yargs;
 };
 
 const run = type => async ({
   key,
   server,
+  type: recordType,
 }) => {
   const client = new MisoClient({ key, server });
   let ids;
   try {
-    ids = await client.api[type].ids();
+    const options = recordType ? { type: recordType } : {};
+    ids = await client.api[type].ids(options);
   } catch (err) {
     console.error(err);
     throw err;

package/cli/merge.js

@@ -1,3 +1,6 @@
+import { join } from 'path';
+import { createReadStream } from 'fs';
+import { createGunzip } from 'zlib';
 import split2 from 'split2';
 import { stream } from '@miso.ai/server-commons';
 import { MisoClient } from '../src/index.js';
@@ -7,18 +10,29 @@ function build(yargs) {
     .option('file', {
       alias: ['f'],
       describe: 'File that contains the merge function',
-    });
+    })
+    .option('fetch', {
+      describe: 'Fetch records from server',
+      type: 'boolean',
+      default: true,
+    })
+    .option('base', {
+      alias: ['b'],
+      describe: 'Base record file',
+    })
 }
 
 const run = type => async ({
   key,
   server,
   file,
+  base,
   ...options
 }) => {
-  // TODO: load merge function
+  const mergeFn = await getMergeFn(file);
+  const records = await buildBaseRecords(base);
   const client = new MisoClient({ key, server });
-  const mergeStream = client.api[type].mergeStream(options);
+  const mergeStream = client.api[type].mergeStream({ ...options, mergeFn, records });
   const outputStream = new stream.OutputStream({ objectMode: true });
   await stream.pipeline(
     process.stdin,
@@ -37,3 +51,30 @@ export default function(type) {
     handler: run(type),
   };
 }
+
+async function getMergeFn(file) {
+  if (!file || file === 'default') {
+    return undefined;
+  }
+  try {
+    return (await import(join(process.env.PWD, file))).default;
+  } catch (e) {
+    throw new Error(`Failed to load merge function from ${file}: ${e.message}`);
+  }
+}
+
+async function buildBaseRecords(file) {
+  if (!file) {
+    return undefined;
+  }
+  let readStream = createReadStream(file);
+  if (file.endsWith('.gz')) {
+    readStream = readStream.pipe(createGunzip());
+  }
+  readStream = readStream.pipe(split2()).pipe(stream.parse());
+  const records = [];
+  for await (const record of readStream) {
+    records.push(record);
+  }
+  return records;
+}

package/package.json

@@ -16,11 +16,11 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.4",
+    "@miso.ai/server-commons": "0.6.5-beta.1",
     "axios": "^1.6.2",
     "dotenv": "^16.0.1",
     "split2": "^4.1.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.4"
+  "version": "0.6.5-beta.1"
 }

package/src/api/base.js

@@ -49,8 +49,9 @@ export class Entities extends Writable {
     return (await axios.get(url)).data.data;
   }
 
-  async ids() {
-    const url = buildUrl(this._client, `${this._type}/_ids`);
+  async ids({ type } = {}) {
+    const options = type ? { params: { type } } : {};
+    const url = buildUrl(this._client, `${this._type}/_ids`, options);
     return (await axios.get(url)).data.data.ids;
   }
 

package/src/api/cache.js

@@ -0,0 +1,48 @@
+import { getIdProperty, shimRecordForMerging } from './helpers.js';
+
+export default class RecordCache {
+
+  constructor(client, type, { records = [], fetch = true, ...options } = {}) {
+    this._client = client;
+    this._type = type;
+    this._idProp = getIdProperty(type);
+    this._options = { fetch, ...options };
+
+    let idProp;
+    switch (type) {
+      case 'products':
+        idProp = 'product_id';
+        break;
+      case 'users':
+        idProp = 'user_id';
+        break;
+      default:
+        throw new Error(`Unsupported type: ${type}`);
+    }
+
+    this._cache = new Map();
+
+    for (const record of records) {
+      this._cache.set(record[idProp], record);
+    }
+  }
+
+  async get(id) {
+    if (!this._cache.has(id)) {
+      this._cache.set(id, this._fetch(id)); // don't await
+    }
+    return this._cache.get(id);
+  }
+
+  async _fetch(id) {
+    if (this._options.fetch === false) {
+      return undefined;
+    }
+    try {
+      return shimRecordForMerging(await this._client.api[this._type].get(id));
+    } catch (e) {
+      return undefined;
+    }
+  }
+
+}

package/src/api/helpers.js

@@ -24,22 +24,36 @@ export async function merge(client, type, record, { mergeFn = defaultMerge } = {
   if (!id) {
     throw new Error(`Record missing ${idProp}.`);
   }
-  const base = shimRecordForMerging(await client.api[type].get(id));
+  let base;
+  try {
+    base = shimRecordForMerging(await client.api[type].get(id));
+  } catch (e) {}
   return await mergeFn(base, record);
 }
 
-function defaultMerge(base, patch) {
-  return {
+export function defaultMerge(base, patch) {
+  return trimObj({
     ...base,
     ...patch,
-    custom_attributes: {
-      ...base.custom_attributes,
-      ...patch.custom_attributes,
-    },
-  };
+    custom_attributes: trimObj({
+      ...(base && base.custom_attributes),
+      ...(patch && patch.custom_attributes),
+    }),
+  });
+}
+
+export function getIdProperty(type) {
+  switch (type) {
+    case 'products':
+      return 'product_id';
+    case 'users':
+      return 'user_id';
+    default:
+      throw new Error(`Unsupported type: ${type}`);
+  }
 }
 
-function shimRecordForMerging(record) {
+export function shimRecordForMerging(record) {
   for (const key in record) {
     if (key === 'product_group_id_or_product_id' || key.startsWith('category_path_')) {
       delete record[key];
@@ -104,8 +118,7 @@ export function buildUrl(client, path, { async, dryRun, params: extraParams } =
   }
   if (extraParams) {
     for (const key in extraParams) {
-      // TODO: deal with encodeURIComponent
-      params += `&${key}=${extraParams[key]}`;
+      params += `&${encodeURIComponent(key)}=${encodeURIComponent(extraParams[key])}`;
     }
   }
   return `${server}/v1/${path}${params}`;

package/src/api/index.js

@@ -1,12 +1,9 @@
-import { asArray } from '@miso.ai/server-commons';
-import { merge } from './helpers.js';
 import Products from './products.js';
 import Users from './users.js';
 import Interactions from './interactions.js';
 import Experiments from './experiments.js';
 import Search from './search.js';
 import Recommendation from './recommendation.js';
-import MergeStream from '../stream/merge.js';
 
 export default class Api {
 

package/src/logger/standard.js

@@ -31,6 +31,7 @@ export default class StandardLogStream extends Writable {
     }
     if (!this._debug) {
       delete record.state;
+      delete record.payload;
     }
     (log.isError(level) ? this._err : this._out).write(this._formatter(record) + '\n');
     next();

package/src/stream/merge.js

@@ -1,18 +1,38 @@
-import { stream } from '@miso.ai/server-commons';
-import { merge } from '../api/helpers.js';
+import { Transform } from 'stream';
+import { getIdProperty, defaultMerge } from '../api/helpers.js';
+import RecordCache from '../api/cache.js';
 
-export default class MergeStream extends stream.ParallelTransform {
+export default class MergeStream extends Transform {
 
   constructor(client, type, {
-    mergeFn,
+    mergeFn = defaultMerge,
+    ...options
   } = {}) {
     super({
-      transform: (record) => merge(client, type, record, { mergeFn }),
-      controls: {
-        throttle: 100,
-      },
       objectMode: true,
     });
+    this._client = client;
+    this._type = type;
+    this._idProp = getIdProperty(type);
+    this._mergeFn = mergeFn;
+    this._cache = new RecordCache(client, type, options);
+  }
+
+  async _transform(record, _, next) {
+    const id = record[this._idProp];
+    if (!id) {
+      this._error(new Error(`Record missing ${this._idProp}.`));
+      next();
+      return;
+    }
+    const base = await this._cache.get(id);
+    try {
+      const merged = await this._mergeFn(base, record);
+      merged && this.push(merged);
+    } catch (error) {
+      this._error(error);
+    }
+    next();
   }
 
   _error(error) {

package/src/stream/upload.js

@@ -66,11 +66,11 @@ export default class UploadStream extends stream.BufferedWriteStream {
     // if upload fails, emit extracted payload at response event
     if (message.event === 'response') {
       const { response, payload } = args;
-      if (response && response.errors && payload) {
-        output.payload = JSON.parse(payload); // TODO: call extractUploadPayload
-        if (response.status === 422) {
-          output.issues = process422ResponseBody(payload, response);
-        }
+      if (payload) {
+        output.payload = JSON.parse(payload);
+      }
+      if (response && response.status === 422) {
+        output.issues = process422ResponseBody(payload, response);
       }
     }
 

package/src/version.js

@@ -1 +1 @@
-export default '0.6.4';
+export default '0.6.5-beta.1';