@miso.ai/server-sdk 0.6.6-beta.1 → 0.6.6-beta.11

package/cli/delete.js

@@ -34,6 +34,7 @@ function build(yargs) {
 }
 
 const run = type => async ({
+  env,
   key,
   server,
   param: params,
@@ -49,7 +50,7 @@ const run = type => async ({
   loglevel = (debug || progress) ? log.DEBUG : loglevel;
   logFormat = progress ? logger.FORMAT.PROGRESS : logFormat;
 
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
 
   const deleteStream = client.api[type].deleteStream({
     name,

package/cli/get.js

@@ -4,12 +4,13 @@ import { formatError } from './utils.js';
 const build = yargs => yargs;
 
 const run = type => async ({
+  env,
   key,
   server,
   id,
   debug,
 }) => {
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
   try {
     const entity = await client.api[type].get(id);
     console.log(JSON.stringify(entity));

package/cli/hybrid-search.js

@@ -15,8 +15,8 @@ function build(yargs) {
     });
 }
 
-async function run({ query, fq, fl, rows, answer, key, server, debug }) {
-  const client = new MisoClient({ key, server, debug });
+async function run({ query, fq, fl, rows, answer, env, key, server, debug }) {
+  const client = new MisoClient({ env, key, server, debug });
   const { products } = await client.api.ask.search({ q: query, fq, fl, rows, answer });
   const readStream = Readable.from(products);
   const outputStream = new stream.OutputStream();

package/cli/ids-diff.js

@@ -25,6 +25,7 @@ function build(yargs) {
 };
 
 const run = type => async ({
+  env,
   key,
   server,
   output,
@@ -34,7 +35,7 @@ const run = type => async ({
 }) => {
   output = output || (plus ? 'plus' : minus ? 'minus' : undefined);
 
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
   const misoIds = await client.api[type].ids();
 
   const diffStream = new stream.DiffStream(misoIds, { output });

package/cli/ids.js

@@ -19,12 +19,13 @@ const build = type => yargs => {
 };
 
 const run = type => async ({
+  env,
   key,
   server,
   type: recordType,
   debug,
 }) => {
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
   let ids;
   try {
     const options = recordType ? { type: recordType } : {};

package/cli/index.js

@@ -7,6 +7,7 @@ import upload from './upload.js';
 import del from './delete.js';
 import ids from './ids.js';
 import transform from './transform.js';
+import mergeLocal from './merge-local.js';
 import status from './status.js';
 import get from './get.js';
 import search from './search.js';
@@ -70,6 +71,7 @@ yargs.build(yargs => {
     .command(users)
     .command(experiments)
     .command(transform)
+    .command(mergeLocal)
     .command(search)
     .command(hybridSearch)
     .version(MisoClient.version);

package/cli/merge-local.js

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+import 'dotenv/config';
+import { join } from 'path';
+import { createReadStream } from 'fs';
+import { createGunzip } from 'zlib';
+import { pipeline } from 'stream/promises';
+import { chain } from 'stream-chain';
+import split2 from 'split2';
+import { stream, trimObj } from '@miso.ai/server-commons';
+
+const PWD = process.env.PWD;
+
+process.stdout.on('error', err => err.code == 'EPIPE' && process.exit(0));
+
+function build(yargs) {
+  return yargs
+    .positional('file1', {
+      describe: 'First record file',
+      type: 'string',
+      demandOption: true,
+    })
+    .positional('file2', {
+      describe: 'Second record file',
+      type: 'string',
+    })
+    .option('join-mode', {
+      alias: 'm',
+      describe: 'Join mode',
+      type: 'string',
+      default: 'merge',
+      choices: ['merge', 'rewrite-meta', 'remeta'],
+    })
+    .option('join-function', {
+      alias: 'f',
+      describe: 'Join function',
+      type: 'string',
+    })
+    .option('stdin', {
+      alias: 'i',
+      describe: 'Read the first stream from stdin',
+      type: 'boolean',
+      default: false,
+    });
+}
+
+async function run({ file1, file2, stdin, joinMode, joinFunction }) {
+  if (stdin) {
+    file2 = file1;
+    file1 = undefined;
+  } else if (!file2) {
+    throw new Error('Second file is required');
+  }
+  const stream1 = chain([
+    stdin ? process.stdin : createReadStream(file1),
+    ...(file1 && file1.endsWith('.gz')) ? [createGunzip()] : [],
+    split2(JSON.parse),
+  ]);
+  const stream2 = chain([
+    createReadStream(file2),
+    ...(file2 && file2.endsWith('.gz')) ? [createGunzip()] : [],
+    split2(JSON.parse),
+  ]);
+  await pipeline(
+    stream.joinStreams([stream1, stream2], await getJoinFunction(joinMode, joinFunction)),
+    new stream.OutputStream(),
+  );
+}
+
+async function getJoinFunction(joinMode, joinFunction) {
+  if (joinFunction) {
+    try {
+      const fn = await import(join(PWD, joinFunction));
+      if (!fn.default) {
+        throw new Error('Join function must have a default export');
+      }
+      return fn.default;
+    } catch (e) {
+      console.error(`Failed to load join function ${joinFunction}:`, e);
+      process.exit(1);
+    }
+  }
+  switch (joinMode) {
+    case 'merge':
+      return merge;
+    case 'rewrite-meta':
+    case 'remeta':
+      return rewriteMeta;
+    default:
+      throw new Error(`Unsupported join mode: ${joinMode}`);
+  }
+}
+
+function merge(record1, record2) {
+  assertSameProductId(record1, record2);
+  return trimObj({
+    ...record1,
+    ...record2,
+    custom_attributes: trimObj({
+      ...(record1 && record1.custom_attributes),
+      ...(record2 && record2.custom_attributes),
+    }),
+  });
+}
+
+function rewriteMeta(record1, record2) {
+  assertSameProductId(record1, record2);
+  const { description, html, children } = record1;
+  const { description: _0, html: _1, children: _2, ...rest } = record2;
+  return trimObj({
+    ...rest,
+    description,
+    html,
+    children,
+  });
+}
+
+function assertSameProductId(record1, record2) {
+  if (record1.product_id !== record2.product_id) {
+    throw new Error(`Product IDs do not match: ${record1.product_id} and ${record2.product_id}`);
+  }
+}
+
+export default {
+  command: 'merge-local <file1> [file2]',
+  alias: ['patch-local'],
+  description: 'Apply patch records from file onto input record stream',
+  builder: build,
+  handler: run,
+};

package/cli/merge.js

@@ -23,6 +23,7 @@ function build(yargs) {
 }
 
 const run = type => async ({
+  env,
   key,
   server,
   file,
@@ -32,7 +33,7 @@ const run = type => async ({
 }) => {
   const mergeFn = await getMergeFn(file);
   const records = await buildBaseRecords(base);
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
   const mergeStream = client.api[type].mergeStream({ ...options, mergeFn, records });
   const outputStream = new stream.OutputStream({ objectMode: true });
   await stream.pipeline(

package/cli/search.js

@@ -10,8 +10,8 @@ function build(yargs) {
     });
 }
 
-async function run({ query, fq, fl, rows, key, server, debug }) {
-  const client = new MisoClient({ key, server, debug });
+async function run({ query, fq, fl, rows, env, key, server, debug }) {
+  const client = new MisoClient({ env, key, server, debug });
   const records = await client.api.search.search({ q: query, fq, fl, rows });
   const readStream = Readable.from(records);
   const outputStream = new stream.OutputStream();

package/cli/status.js

@@ -7,12 +7,13 @@ const build = type => yargs => {
 };
 
 const run = type => async ({
+  env,
   key,
   server,
   taskId,
   debug,
 }) => {
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
   if (taskId) {
     runOne(client, type, taskId);
   } else {

package/cli/upload.js

@@ -46,6 +46,7 @@ function build(yargs) {
 }
 
 const run = type => async ({
+  env,
   key,
   server,
   param: params,
@@ -66,7 +67,7 @@ const run = type => async ({
   loglevel = (debug || progress) ? log.DEBUG : loglevel;
   logFormat = progress ? logger.FORMAT.PROGRESS : logFormat;
 
-  const client = new MisoClient({ key, server, debug });
+  const client = new MisoClient({ env, key, server, debug });
 
   const uploadStreamObjectMode = lenient;
 

package/cli/utils.js

@@ -6,6 +6,10 @@ export function buildForApi(yargs) {
       alias: ['k', 'api-key'],
       describe: 'API key',
     })
+    .option('env', {
+      describe: 'Environment',
+      type: 'string',
+    })
     .option('server', {
       alias: ['api-server'],
       describe: 'API server',
@@ -19,8 +23,7 @@ export function buildForApi(yargs) {
     .option('debug', {
       describe: 'Set log level to debug',
       type: 'boolean',
-    })
-    .demandOption(['key'], 'API key is required.');
+    });
 }
 
 export function buildForSearch(yargs) {

package/package.json

@@ -16,12 +16,12 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.6-beta.1",
+    "@miso.ai/server-commons": "0.6.6-beta.11",
     "axios": "^1.6.2",
     "axios-retry": "^4.5.0",
     "dotenv": "^16.0.1",
     "split2": "^4.1.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.6-beta.1"
+  "version": "0.6.6-beta.11"
 }

package/src/api/helpers.js

@@ -37,6 +37,7 @@ async function recoverValidRecords(client, type, records, options, response) {
     return; // still fail, never mind...
   }
   response.recovered = {
+    product_ids: validRecords.map(record => record.product_id),
     records: validRecords.length,
     bytes: validPayload.length * 2,
   };

package/src/client.js

@@ -28,6 +28,7 @@ function normalizeOptions(options) {
   if (typeof options === 'string') {
     options = { key: options };
   }
+  options.key = getApiKeyFromEnv(options.env, options.key);
   if (!options.key || typeof options.key !== 'string') {
     throw new Error(`API key is required.`);
   }
@@ -35,3 +36,10 @@ function normalizeOptions(options) {
 
   return options;
 }
+
+function getApiKeyFromEnv(env, key) {
+  if (!env) {
+    return key;
+  }
+  return process.env[`MISO_${env.toUpperCase()}_API_KEY`] || undefined;
+}

package/src/stream/api-sink.js

@@ -65,7 +65,6 @@ export default class ApiSink extends sink.BpsSink {
     }
 
     // keep track of service stats on successful calls
-    // TODO: handle recovered records?
     if (!data.errors) {
       this._serviceStats.track({ records, bytes, took: data.took });
     }

package/src/version.js

@@ -1 +1 @@
-export default '0.6.6-beta.1';
+export default '0.6.6-beta.11';