@miso.ai/server-sdk 0.6.6-beta.3 → 0.6.6-beta.4

package/cli/merge-local.js

@@ -1,116 +1,127 @@
 #!/usr/bin/env node
 import 'dotenv/config';
-import { promisify } from 'util';
-import { readFile } from 'fs/promises';
-import { gunzip } from 'zlib';
-import { Transform } from 'stream';
+import { join } from 'path';
+import { createReadStream } from 'fs';
+import { createGunzip } from 'zlib';
 import { pipeline } from 'stream/promises';
-import { stream, trimObj } from '@miso.ai/server-commons';
+import { chain } from 'stream-chain';
 import split2 from 'split2';
+import { stream, trimObj } from '@miso.ai/server-commons';
+
+const PWD = process.env.PWD;
 
 process.stdout.on('error', err => err.code == 'EPIPE' && process.exit(0));
 
 function build(yargs) {
   return yargs
-    .positional('file', {
-      describe: 'Patch record file',
+    .positional('file1', {
+      describe: 'First record file',
       type: 'string',
       demandOption: true,
+    })
+    .positional('file2', {
+      describe: 'Second record file',
+      type: 'string',
+    })
+    .option('join-mode', {
+      alias: 'm',
+      describe: 'Join mode',
+      type: 'string',
+      default: 'merge',
+      choices: ['merge', 'rewrite-meta', 'remeta'],
+    })
+    .option('join-function', {
+      alias: 'f',
+      describe: 'Join function',
+      type: 'string',
+    })
+    .option('stdin', {
+      alias: 'i',
+      describe: 'Read the first stream from stdin',
+      type: 'boolean',
+      default: false,
     });
 }
 
-async function run({ file }) {
-  const patches = await readRecordsIntoMap(file);
-  await pipeline(
-    process.stdin,
+async function run({ file1, file2, stdin, joinMode, joinFunction }) {
+  if (stdin) {
+    file2 = file1;
+    file1 = undefined;
+  } else if (!file2) {
+    throw new Error('Second file is required');
+  }
+  const stream1 = chain([
+    stdin ? process.stdin : createReadStream(file1),
+    ...(file1 && file1.endsWith('.gz')) ? [createGunzip()] : [],
+    split2(JSON.parse),
+  ]);
+  const stream2 = chain([
+    createReadStream(file2),
+    ...(file2 && file2.endsWith('.gz')) ? [createGunzip()] : [],
     split2(JSON.parse),
-    new MergeLocalStream({ patches }),
+  ]);
+  await pipeline(
+    stream.joinStreams([stream1, stream2], await getJoinFunction(joinMode, joinFunction)),
     new stream.OutputStream(),
   );
 }
 
-async function readRecordsIntoMap(file) {
-  const map = new Map();
-  const content = await readContent(file);
-  for (let line of content.split('\n')) {
-    line = line.trim();
-    if (!line) {
-      continue;
-    }
+async function getJoinFunction(joinMode, joinFunction) {
+  if (joinFunction) {
     try {
-      const record = JSON.parse(line);
-      const { product_id } = record;
-      if (!product_id) {
-        console.error(`Missing product_id: ${line}`);
-        continue;
+      const fn = await import(join(PWD, joinFunction));
+      if (!fn.default) {
+        throw new Error('Join function must have a default export');
       }
-      map.set(product_id, record);
-    } catch (error) {
-      console.error(`Error parsing JSON line: ${line}`, error);
+      return fn.default;
+    } catch (e) {
+      console.error(`Failed to load join function ${joinFunction}:`, e);
+      process.exit(1);
     }
   }
-  return map;
-}
-
-async function readContent(file) {
-  let fileBuffer = await readFile(file);
-  if (file.endsWith('.gz')) {
-    fileBuffer = await promisify(gunzip)(fileBuffer);
+  switch (joinMode) {
+    case 'merge':
+      return merge;
+    case 'rewrite-meta':
+    case 'remeta':
+      return rewriteMeta;
+    default:
+      throw new Error(`Unsupported join mode: ${joinMode}`);
   }
-  return fileBuffer.toString('utf-8');
 }
 
-class MergeLocalStream extends Transform {
-
-  constructor({
-    patches,
-    ...options
-  } = {}) {
-    super({
-      objectMode: true,
-      ...options,
-    });
-    this._patches = patches;
-    this.on('drain', () => this._handleDrain());
-  }
-
-  _transform(record, _, next) {
-    const patch = this._patches.get(record.product_id);
-    if (patch) {
-      record = this._patch(record, patch);
-    }
-
-    const pause = !this.push(record);
-    if (pause) {
-      this._next = next;
-    } else {
-      next();
-    }
-  }
+function merge(record1, record2) {
+  assertSameProductId(record1, record2);
+  return trimObj({
+    ...record1,
+    ...record2,
+    custom_attributes: trimObj({
+      ...(record1 && record1.custom_attributes),
+      ...(record2 && record2.custom_attributes),
+    }),
+  });
+}
 
-  _patch(record, patch) {
-    patch = trimObj(patch);
-    return {
-      ...record,
-      ...patch,
-      custom_attributes: trimObj({
-        ...(record && record.custom_attributes),
-        ...(patch && patch.custom_attributes),
-      }),
-    };
-  }
+function rewriteMeta(record1, record2) {
+  assertSameProductId(record1, record2);
+  const { description, html, children } = record1;
+  const { description: _0, html: _1, children: _2, ...rest } = record2;
+  return trimObj({
+    ...rest,
+    description,
+    html,
+    children,
+  });
+}
 
-  _handleDrain() {
-    if (this._next) {
-      this._next();
-    }
+function assertSameProductId(record1, record2) {
+  if (record1.product_id !== record2.product_id) {
+    throw new Error(`Product IDs do not match: ${record1.product_id} and ${record2.product_id}`);
   }
-
 }
 
-
 export default {
-  command: 'merge-local <file>',
+  command: 'merge-local <file1> [file2]',
   alias: ['patch-local'],
   description: 'Apply patch records from file onto input record stream',
   builder: build,

package/package.json

@@ -16,12 +16,12 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.6-beta.3",
+    "@miso.ai/server-commons": "0.6.6-beta.4",
     "axios": "^1.6.2",
     "axios-retry": "^4.5.0",
     "dotenv": "^16.0.1",
     "split2": "^4.1.0",
     "yargs": "^17.5.1"
   },
-  "version": "0.6.6-beta.3"
+  "version": "0.6.6-beta.4"
 }

package/src/version.js

@@ -1 +1 @@
-export default '0.6.6-beta.3';
+export default '0.6.6-beta.4';