@engine9-io/input-tools 1.5.3 → 1.6.1

package/file/FileUtilities.js

@@ -5,22 +5,23 @@ const fsp = fs.promises;
 const path = require('node:path');
 const zlib = require('node:zlib');
 const {
-  Readable, Transform, PassThrough,
+  Readable, Transform, PassThrough, Writable,
 } = require('node:stream');
 const { pipeline } = require('node:stream/promises');
 const { stringify } = require('csv');
 
 const debug = require('debug')('FileWorker');
-// const through2 = require('through2');
+
 const csv = require('csv');
-const JSON5 = require('json5');// Useful for parsing extended JSON
+const JSON5 = require('json5');
 const languageEncoding = require('detect-file-encoding-and-language');
 const R2Worker = require('./R2');
 const S3Worker = require('./S3');
 const ParquetWorker = require('./Parquet');
-const { streamPacket } = require('./tools');
 
-const { bool, getTempDir } = require('./tools');
+const {
+  bool, getStringArray, getTempDir, makeStrings, streamPacket,
+} = require('./tools');
 
 function Worker({ accountId }) { this.accountId = accountId; }
 
@@ -543,6 +544,15 @@ Worker.prototype.sample.metadata = {
 
   },
 };
+Worker.prototype.toArray = async function (opts) {
+  const { stream } = await this.fileToObjectStream(opts);
+  return stream.toArray();
+};
+Worker.prototype.toArray.metadata = {
+  options: {
+    filename: {},
+  },
+};
 
 Worker.prototype.write = async function (opts) {
   const { filename, content } = opts;
@@ -774,4 +784,151 @@ Worker.prototype.count.metadata = {
   },
 };
 
+// Get a set of unique entries from a uniqueFunction
+// This could be large
+Worker.prototype.getUniqueSet = async function (options) {
+  const existingFiles = getStringArray(options.filenames);
+  const sample = {};
+
+  let { uniqueFunction } = options;
+  if (!uniqueFunction) {
+    uniqueFunction = ((o) => JSON.stringify(o));
+  }
+  const uniqueSet = new Set();
+  // eslint-disable-next-line no-restricted-syntax, guard-for-in
+  for (const filename of existingFiles) {
+    const { stream: existsStream } = await this.fileToObjectStream({ filename });
+    await pipeline(
+      existsStream,
+      new Transform({
+        objectMode: true,
+        transform(d, enc, cb) {
+          const v = uniqueFunction(makeStrings(d)) || '';
+          if (uniqueSet.size < 3) {
+            sample[v] = d;
+          }
+          uniqueSet.add(v);
+          cb(null, d);
+        },
+      }),
+      new Writable({
+        objectMode: true,
+        write(d, enc, cb) {
+          cb();
+        },
+      }),
+    );
+    debug(`Finished loading ${filename}`);
+  }
+  return { uniqueFunction, uniqueSet, sample };
+};
+
+Worker.prototype.getUniqueStream = async function (options) {
+  const includeDuplicateSourceRecords = bool(options.includeDuplicateSourceRecords, false);
+
+  const { uniqueSet, uniqueFunction, sample } = await this.getUniqueSet({
+    filenames: options.existingFiles,
+    uniqueFunction: options.uniqueFunction,
+  });
+
+  const { stream: inStream } = await this.fileToObjectStream(options);
+  const uniqueStream = inStream.pipe(
+    new Transform({
+      objectMode: true,
+      transform(d, enc, cb) {
+        const v = uniqueFunction(makeStrings(d)) || '';
+
+        if (!v) {
+          // falsey unique function includes
+          // by default
+          cb(null, d);
+        } else if (uniqueSet.has(v)) {
+          // do nothing
+          cb();
+        } else {
+          if (!includeDuplicateSourceRecords) {
+            // add it to the set for the next time
+            uniqueSet.add(v);
+          }
+          cb(null, d);
+        }
+      },
+    }),
+  );
+  return { stream: uniqueStream, sample };
+};
+
+Worker.prototype.getUniqueStream.metadata = {
+  options: {
+    existingFiles: {},
+    uniqueFunction: {},
+    filename: { description: 'Specify a source filename or a stream' },
+    stream: { description: 'Specify a source filename or a stream' },
+    includeDuplicateSourceRecords: {
+      description: 'Sometimes you want the output to include source dupes, sometimes not, default false',
+    },
+  },
+};
+Worker.prototype.getUniqueFile = async function (options) {
+  const { stream, sample } = await this.getUniqueStream(options);
+  const { filename, records } = await this.objectStreamToFile({ stream });
+  return { filename, records, sample };
+};
+
+Worker.prototype.getUniqueFile.metadata = {
+  options: {
+    existingFiles: {},
+    uniqueFunction: {},
+    filename: { description: 'Specify a source filename or a stream' },
+    stream: { description: 'Specify a source filename or a stream' },
+    includeDuplicateSourceRecords: {
+      description: 'Sometimes you want the output to include source dupes, sometimes not, default false',
+    },
+  },
+};
+
+/*
+diff that allows for unordered files, and doesn't store full objects in memory.
+Requires 2 passes of the files,
+but that's a better tradeoff than trying to store huge files in memory
+*/
+Worker.prototype.diff = async function ({
+  fileA, fileB, uniqueFunction: ufOpt, fields, includeDuplicateSourceRecords,
+}) {
+  if (ufOpt && fields) throw new Error('fields and uniqueFunction cannot both be specified');
+  let uniqueFunction = ufOpt;
+  if (!uniqueFunction && fields) {
+    const farr = getStringArray(fields);
+    uniqueFunction = (o) => farr.map((f) => o[f] || '').join('.');
+  }
+
+  const left = await this.getUniqueFile({
+    existingFiles: [fileB],
+    filename: fileA,
+    uniqueFunction,
+    includeDuplicateSourceRecords,
+  });
+  const right = await this.getUniqueFile({
+    existingFiles: [fileA],
+    filename: fileB,
+    uniqueFunction,
+    includeDuplicateSourceRecords,
+  });
+
+  return {
+    left, right,
+  };
+};
+Worker.prototype.diff.metadata = {
+  options: {
+    fileA: {},
+    fileB: {},
+    fields: { description: 'Fields to use for uniqueness -- aka primary key.  Defaults to JSON of line' },
+    uniqueFunction: {},
+    includeDuplicateSourceRecords: {
+      description: 'Sometimes you want the output to include source dupes, sometimes not, default false',
+    },
+  },
+};
+
 module.exports = Worker;

package/file/tools.js

@@ -222,16 +222,38 @@ function bool(x, _defaultVal) {
   const y = x.toLowerCase();
   return !!(y.indexOf('y') + 1) || !!(y.indexOf('t') + 1);
 }
+function getStringArray(s, nonZeroLength) {
+  let a = s || [];
+  if (typeof a === 'number') a = String(a);
+  if (typeof a === 'string') a = [a];
+
+  if (typeof s === 'string') a = s.split(',');
+  a = a.map((x) => x.toString().trim()).filter(Boolean);
+  if (nonZeroLength && a.length === 0) a = [0];
+  return a;
+}
+/*
+  When comparing two objects, some may come from a file (thus strings), and some from
+  a database or elsewhere (not strings), so for deduping make sure to make them all strings
+*/
+function makeStrings(o) {
+  return Object.entries(o).reduce((a, [k, v]) => {
+    a[k] = (typeof v === 'object') ? JSON.stringify(v) : String(v);
+    return a;
+  }, {});
+}
 
 module.exports = {
   bool,
+  downloadFile,
   getTempFilename,
   getTempDir,
-  downloadFile,
   getBatchTransform,
   getDebatchTransform,
   getFile,
   getManifest,
   getPacketFiles,
+  getStringArray,
+  makeStrings,
   streamPacket,
 };

package/index.js

@@ -25,6 +25,8 @@ const {
   getPacketFiles,
   getBatchTransform,
   getDebatchTransform,
+  getStringArray,
+  makeStrings,
 } = require('./file/tools');
 
 const ForEachEntry = require('./ForEachEntry');
@@ -47,17 +49,6 @@ handlebars.registerHelper('json', (d) => JSON.stringify(d));
 
 handlebars.registerHelper('percent', (a, b) => `${((100 * a) / b).toFixed(2)}%`);
 
-function getStringArray(s, nonZeroLength) {
-  let a = s || [];
-  if (typeof a === 'number') a = String(a);
-  if (typeof a === 'string') a = [a];
-
-  if (typeof s === 'string') a = s.split(',');
-  a = a.map((x) => x.toString().trim()).filter(Boolean);
-  if (nonZeroLength && a.length === 0) a = [0];
-  return a;
-}
-
 function isValidDate(d) {
   // we WANT to use isNaN, not the Number.isNaN -- we're checking the date type
   // eslint-disable-next-line no-restricted-globals
@@ -125,17 +116,6 @@ function relativeDate(s, _initialDate) {
   return r;
 }
 
-/*
-  When comparing two objects, some may come from a file (thus strings), and some from
-  a database or elsewhere (not strings), so for deduping make sure to make them all strings
-*/
-function makeStrings(o) {
-  return Object.entries(o).reduce((a, [k, v]) => {
-    a[k] = (typeof v === 'object') ? JSON.stringify(v) : String(v);
-    return a;
-  }, {});
-}
-
 async function list(_path) {
   const directory = await unzipper.Open.file(_path);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.5.3",
+  "version": "1.6.1",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {

package/test/parallelStream.js

@@ -1,48 +0,0 @@
-/* eslint-disable no-console */
-const {
-  setTimeout,
-} = require('node:timers/promises');
-
-const { describe } = require('node:test');
-// const assert = require('node:assert');
-const { Readable } = require('node:stream');
-const { createWriteStream } = require('node:fs');
-const { pipeline } = require('node:stream/promises');
-const ParallelStream = require('../ParallelStream');
-const { getTempFilename } = require('../index');
-
-describe('Should process items in parallel:', async () => {
-  const outputFile = await getTempFilename({});
-  const writeStream = createWriteStream(outputFile);
-
-  const CONCURRENCY = 500;
-  await pipeline(
-    Readable.from(
-      [...Array(1000)].map((v, i) => ({ i })),
-    ),
-
-    new ParallelStream(
-      CONCURRENCY,
-      async (obj, enc, push, done) => {
-        let res;
-
-        try {
-          await setTimeout(Math.random() * 1000);
-          if (Math.random() > 0.7) throw new Error('Random error');
-
-          res = `${obj.id} is complete\n`;
-        } catch (err) {
-          await setTimeout(Math.random() * 2000);// longer timeouts for errors
-          res = `${obj.id} is error, ${err.name}\n`;
-        }
-
-        done(null, obj.id); // _onComplete actually
-
-        return res;
-      },
-    ),
-    writeStream,
-  );
-
-  console.log('Wrote responses to ', outputFile);
-});