@engine9-io/input-tools 1.9.4 → 1.9.7

package/ForEachEntry.js

@@ -24,7 +24,7 @@ class ForEachEntry {
     this.fileUtilities = new FileUtilities({ accountId });
   }
 
-  getOutputStream({ name, postfix = '.timeline.csv', validatorFunction = () => true }) {
+  getOutputStream({ name, filename, postfix = '.timeline.csv', validatorFunction = () => true }) {
     this.outputStreams = this.outputStreams || {};
     if (this.outputStreams[name]?.items) return this.outputStreams[name].items;
 
@@ -33,12 +33,14 @@ class ForEachEntry {
     };
 
     return this.outputStreams[name].mutex.runExclusive(async () => {
+      let f = filename || (await getTempFilename({ postfix }));
+
       const fileInfo = {
-        filename: await getTempFilename({ postfix }),
+        filename: f,
         records: 0
       };
 
-      debug(`Output file requested, writing output to to: ${fileInfo.filename}`);
+      debug(`Output file requested ${name}, writing output to to: ${fileInfo.filename}`);
       const outputStream = new ValidatingReadable(
         {
           objectMode: true

package/file/FileUtilities.js

@@ -206,6 +206,17 @@ Worker.prototype.xlsxToObjectStream = async function (options) {
   return { stream };
 };
 
+Worker.prototype.getFormat = async function (options) {
+  const { sourcePostfix, filename, format: formatOverride } = options;
+  let postfix = sourcePostfix || filename.toLowerCase().split('.').pop();
+
+  if (postfix === 'gz') {
+    postfix = filename.toLowerCase().split('.');
+    postfix = postfix[postfix.length - 2];
+  }
+  return formatOverride || postfix;
+};
+
 /*
 Commonly used method to transform a file into a stream of objects.
 */
@@ -635,6 +646,8 @@ Worker.prototype.write = async function (opts) {
       content
     });
   } else {
+    const directory = path.dirname(filename);
+    await fsp.mkdir(directory, { recursive: true });
     await fsp.writeFile(filename, content);
   }
   return { success: true, filename };
@@ -861,7 +874,14 @@ Worker.prototype.move = async function ({ filename, target, remove = true }) {
   }
   await fsp.mkdir(path.dirname(target), { recursive: true });
   if (remove) {
-    await fsp.rename(filename, target);
+    try {
+      await fsp.rename(filename, target);
+    } catch (e) {
+      //it may be a filesystem issue moving between items
+      debug(e);
+      await fsp.copyFile(filename, target);
+      await fsp.unlink(filename);
+    }
   } else {
     await fsp.copyFile(filename, target);
   }
@@ -953,6 +973,34 @@ Worker.prototype.head.metadata = {
   }
 };
 
+Worker.prototype.columns = async function (options) {
+  const head = await this.head(options);
+  if (head.length == 0) {
+    return {
+      records: 0,
+      likelyHeaderLines: 0,
+      columns: []
+    };
+  }
+
+  let likelyHeaderLines = 1;
+  const columns = Object.keys(head[0]);
+  let s = columns.join(',');
+  if (s.match(/[()@#%!]/)) {
+    likelyHeaderLines = 0;
+  }
+  return {
+    likelyHeaderLines,
+    columns
+  };
+};
+
+Worker.prototype.columns.metadata = {
+  options: {
+    filename: { required: true }
+  }
+};
+
 Worker.prototype.count = async function (options) {
   const { stream } = await this.fileToObjectStream(options);
   const sample = [];
@@ -1084,17 +1132,14 @@ diff that allows for unordered files, and doesn't store full objects in memory.
 Requires 2 passes of the files,
 but that's a better tradeoff than trying to store huge files in memory
 */
-Worker.prototype.diff = async function ({
-  fileA,
-  fileB,
-  uniqueFunction: ufOpt,
-  fields,
-  includeDuplicateSourceRecords
-}) {
-  if (ufOpt && fields) throw new Error('fields and uniqueFunction cannot both be specified');
+Worker.prototype.diff = async function (options) {
+  const { fileA, fileB, uniqueFunction: ufOpt, columns, includeDuplicateSourceRecords } = options;
+  if (options.fields) throw new Error('fields is deprecated, use columns');
+
+  if (ufOpt && columns) throw new Error('fields and uniqueFunction cannot both be specified');
   let uniqueFunction = ufOpt;
-  if (!uniqueFunction && fields) {
-    const farr = getStringArray(fields);
+  if (!uniqueFunction && columns) {
+    const farr = getStringArray(columns);
     uniqueFunction = (o) => farr.map((f) => o[f] || '').join('.');
   }
 
@@ -1120,7 +1165,7 @@ Worker.prototype.diff.metadata = {
   options: {
     fileA: {},
     fileB: {},
-    fields: { description: 'Fields to use for uniqueness -- aka primary key.  Defaults to JSON of line' },
+    columns: { description: 'Columns to use for uniqueness -- aka primary key.  Defaults to JSON of line' },
     uniqueFunction: {},
     includeDuplicateSourceRecords: {
       description: 'Sometimes you want the output to include source dupes, sometimes not, default false'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.9.4",
+  "version": "1.9.7",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {

package/test/file.js

@@ -1,18 +1,24 @@
-const {
-  it,
-} = require('node:test');
+const { it } = require('node:test');
 const assert = require('node:assert');
 const debug = require('debug')('files');
 
 const { FileUtilities } = require('../index');
 
 it('Should list a directory', async () => {
-  const futil=new FileUtilities({accountId:'test'});
-  let files=await futil.list({directory:'.'});
-  assert(files.length,"Should have some files");
+  const futil = new FileUtilities({ accountId: 'test' });
+  let files = await futil.list({ directory: '.' });
+  assert(files.length, 'Should have some files');
   debug(files);
-  let startTest=await futil.list({directory:'.',start:'2040-01-01'});
-  assert(startTest.length===0,"Should NOT have any files before future start date");
-  let endTest=await futil.list({directory:'.',end:'1900-01-01'});
-  assert(endTest.length===0,"Should NOT have any files before past end date");
+  let startTest = await futil.list({ directory: '.', start: '2040-01-01' });
+  assert(startTest.length === 0, 'Should NOT have any files before future start date');
+  let endTest = await futil.list({ directory: '.', end: '1900-01-01' });
+  assert(endTest.length === 0, 'Should NOT have any files before past end date');
+});
+
+it('Should be able to analyze CSV files with and without header lines', async () => {
+  const futil = new FileUtilities({ accountId: 'test' });
+  const f1 = await futil.columns({ filename: __dirname + '/sample/fileWithHead.csv' });
+  assert.equal(f1.likelyHeaderLines, 1, 'Number of header lines should be 1');
+  const f2 = await futil.columns({ filename: __dirname + '/sample/fileWithoutHead.csv' });
+  assert.equal(f2.likelyHeaderLines, 0, 'Number of header lines should be 1');
 });

package/test/processing/forEachResume.js

@@ -0,0 +1,56 @@
+const { describe, it } = require('node:test');
+const assert = require('node:assert');
+const debug = require('debug')('test/forEach');
+
+const { ForEachEntry } = require('../../index');
+
+describe('Test Person File For Each', async () => {
+  it('forEachPerson Should loop through 1000 sample people', async () => {
+    let counter = 0;
+    const forEach = new ForEachEntry();
+    const result = await forEach.process({
+      packet: 'test/sample/1000_message.packet.zip',
+      batchSize: 50,
+      bindings: {
+        timelineOutputFileStream: {
+          path: 'output.timeline',
+          options: {
+            entry_type: 'ENTRY_OPTION'
+          }
+        },
+        sampleOutputFileStream: {
+          path: 'output.stream'
+        }
+      },
+      async transform(props) {
+        const { batch, timelineOutputFileStream, sampleOutputFileStream } = props;
+
+        batch.forEach((p) => {
+          if (Math.random() > 0.9) {
+            sampleOutputFileStream.push({
+              // for testing we don't need real person_ids
+              person_id: p.person_id || Math.floor(Math.random() * 1000000),
+              email: p.email,
+              entry_type: 'SAMPLE_OUTPUT'
+            });
+          }
+          timelineOutputFileStream.push({
+            // for testing we don't need real person_ids
+            person_id: p.person_id || Math.floor(Math.random() * 1000000),
+            email: p.email,
+            entry_type: 'EMAIL_DELIVERED'
+          });
+        });
+
+        batch.forEach(() => {
+          counter += 1;
+        });
+      }
+    });
+    assert(result.outputFiles?.timelineOutputFileStream?.[0]?.records);
+    assert(result.outputFiles?.sampleOutputFileStream?.[0]?.records);
+    debug(result);
+    assert.equal(counter, 1000, `Expected to loop through 1000 people, actual:${counter}`);
+  });
+  debug('Completed tests');
+});

package/test/sample/fileWithHead.csv

@@ -0,0 +1,3 @@
+first,last,email,phone
+Bob,Smith,test@test.com,(703) 555-5555
+Jane,Doe,test2@test.com,(703) 555-5432

package/test/sample/fileWithoutHead.csv

@@ -0,0 +1,2 @@
+Bob,Smith,test@test.com,(703) 555-5555
+Jane,Doe,test2@test.com,(703) 555-5432

package/timelineTypes.js

@@ -26,6 +26,16 @@ const TRANSACTION_REFUND = 15;
 const SEGMENT_PERSON_ADD = 16;
 const SEGMENT_PERSON_REMOVE = 17;
 
+// unknown generic conversion on a message
+const MESSAGE_CONVERSION = 20;
+// advocacy conversion on a message
+const MESSAGE_CONVERSION_ADVOCACY = 21;
+// unknown transaction conversion on a message
+const MESSAGE_CONVERSION_TRANSACTION = 22;
+
+const MESSAGE_DELIVERY_FAILURE_SHOULD_RETRY = 25;
+const MESSAGE_DELIVERY_FAILURE_SHOULD_NOT_RETRY = 26;
+
 const SMS_SEND = 30;
 const SMS_DELIVERED = 31;
 const SMS_CLICK = 33;
@@ -54,16 +64,6 @@ const FORM_SUBMIT = 60;
 const FORM_PETITION = 61;
 const FORM_PETITION_CONTACT_TARGET = 62;
 
-// unknown generic conversion on a message
-const MESSAGE_CONVERSION = 63;
-// advocacy conversion on a message
-const MESSAGE_CONVERSION_ADVOCACY = 64;
-// unknown transaction conversion on a message
-const MESSAGE_CONVERSION_TRANSACTION = 65;
-
-const MESSAGE_DELIVERY_FAILURE_SHOULD_RETRY = 66;
-const MESSAGE_DELIVERY_FAILURE_SHOULD_NOT_RETRY = 66;
-
 const FORM_ADVOCACY = 66;
 const FORM_SURVEY = 67;