@engine9-io/input-tools 1.6.9 → 1.7.1

package/ForEachEntry.js

@@ -1,6 +1,6 @@
 const fs = require('node:fs');
 
-const { Writable } = require('node:stream');
+const { Transform, Writable } = require('node:stream');
 const { pipeline } = require('node:stream/promises');
 const { throttle } = require('throttle-debounce');
 const parallelTransform = require('parallel-transform');
@@ -23,30 +23,32 @@ const {
 
 class ForEachEntry {
   constructor({ accountId } = {}) {
-    this.timelineOutputMutex = new Mutex();
     this.fileUtilities = new FileUtilities({ accountId });
   }
 
-  getTimelineOutputStream() {
-    return this.timelineOutputMutex.runExclusive(async () => {
-      if (this.outputStream) return this.outputStream;
-      const timelineFile = await getTempFilename({ postfix: '.timeline.csv' });
-      debug(`Timeline output requested, writing timeline file to: ${timelineFile}`);
-      const timelineOutputStream = new ValidatingReadable({
+  getOutputStream({ name, postfix = '.timeline.csv', validatorFunction = () => true }) {
+    this.outputStreams = this.outputStreams || {};
+    if (this.outputStreams[name]?.items) return this.outputStreams[name].items;
+
+    this.outputStreams[name] = this.outputStreams[name] || {
+      mutex: new Mutex(),
+    };
+
+    return this.outputStreams[name].mutex.runExclusive(async () => {
+      const fileInfo = {
+        filename: await getTempFilename({ postfix }),
+        records: 0,
+      };
+
+      debug(`Output file requested, writing output to to: ${fileInfo.filename}`);
+      const outputStream = new ValidatingReadable({
         objectMode: true,
-      }, (data) => {
-        if (!data) return true;
-        if (typeof data !== 'object') throw new Error('Invalid timeline data push, must be an object');
-        // Is this necessary?
-        if (!data.person_id) throw new Error('Invalid timeline data push, must have a person_id, even if 0');
-        if (!data.ts) data.ts = new Date().toISOString();
-        return true;
-      });
+      }, validatorFunction);
       // eslint-disable-next-line no-underscore-dangle
-      timelineOutputStream._read = () => {};
+      outputStream._read = () => {};
 
-      const writeStream = fs.createWriteStream(timelineFile);
-      const finishWritingTimelinePromise = new Promise((resolve, reject) => {
+      const writeStream = fs.createWriteStream(fileInfo.filename);
+      const finishWritingOutputPromise = new Promise((resolve, reject) => {
         writeStream.on('finish', () => {
           resolve();
         }).on('error', (err) => {
@@ -54,16 +56,24 @@ class ForEachEntry {
         });
       });
 
-      timelineOutputStream
+      this.outputStreams[name].items = {
+        stream: outputStream,
+        promises: [finishWritingOutputPromise],
+        files: [fileInfo],
+      };
+
+      outputStream
+        .pipe(new Transform({
+          objectMode: true,
+          transform(o, enc, cb) {
+            fileInfo.records += 1;
+            cb();
+          },
+        }))
         .pipe(csv.stringify({ header: true }))
         .pipe(writeStream);
 
-      this.outputStream = {
-        stream: timelineOutputStream,
-        promises: [finishWritingTimelinePromise],
-        files: [timelineFile],
-      };
-      return this.outputStream;
+      return this.outputStreams[name].items;
     });
   }
 
@@ -90,7 +100,7 @@ class ForEachEntry {
     let records = 0;
     let batches = 0;
 
-    let timelineFiles = [];
+    const outputFiles = {};
 
     const transformArguments = {};
     // An array of promises that must be completed, such as writing to disk
@@ -105,11 +115,31 @@ class ForEachEntry {
       const binding = bindings[bindingName];
       if (!binding.path) throw new Error(`Invalid binding: path is required for binding ${bindingName}`);
       if (binding.path === 'output.timeline') {
-        const { stream: streamImpl, promises, files } = await this.getTimelineOutputStream({});
+        const { stream: streamImpl, promises, files } = await this.getOutputStream({
+          name: bindingName,
+          postfix: '.timeline.csv',
+          validatorFunction: (data) => {
+            if (!data) return true;
+            if (typeof data !== 'object') throw new Error('Invalid timeline data push, must be an object');
+            // Is this necessary?
+            if (!data.person_id) throw new Error('Invalid timeline data push, must have a person_id, even if 0');
+            if (!data.ts) data.ts = new Date().toISOString();
+            return true;
+          },
+        });
+        newStreams.push(streamImpl);
+        transformArguments[bindingName] = streamImpl;
+        bindingPromises = bindingPromises.concat(promises || []);
+        outputFiles[bindingName] = files;
+      } else if (binding.path === 'output.stream') {
+        const { stream: streamImpl, promises, files } = await this.getOutputStream({
+          name: bindingName,
+          postfix: '.output.csv',
+        });
         newStreams.push(streamImpl);
         transformArguments[bindingName] = streamImpl;
         bindingPromises = bindingPromises.concat(promises || []);
-        timelineFiles = timelineFiles.concat(files);
+        outputFiles[bindingName] = files;
       } else if (binding.path === 'file') {
         transformArguments[bindingName] = await getFile(binding);
       } else if (binding.path === 'handlebars') {
@@ -152,7 +182,7 @@ class ForEachEntry {
     newStreams.forEach((s) => s.push(null));
     await Promise.all(bindingPromises);
 
-    return { timelineFiles };
+    return { outputFiles };
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.6.9",
+  "version": "1.7.1",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {

package/test/packet/forEach.js

@@ -10,41 +10,53 @@ describe('Test Person Packet For Each', async () => {
   it('forEachPerson Should loop through 1000 sample people', async () => {
     let counter = 0;
     const forEach = new ForEachEntry();
-    await forEach.process(
+    const result = await forEach.process(
       {
         packet: 'test/sample/1000_message.packet.zip',
         batchSize: 50,
         bindings: {
-          timelineOutputStream: {
+          timelineOutputFileStream: {
             path: 'output.timeline',
             options: {
-              entry_type: 'SAMPLE',
+              entry_type: 'ENTRY_OPTION',
             },
           },
+          sampleOutputFileStream: {
+            path: 'output.stream',
+          },
         },
         async transform(props) {
           const {
             batch,
-            timelineOutputStream,
+            timelineOutputFileStream,
+            sampleOutputFileStream,
           } = props;
-          if (timelineOutputStream) {
-            batch.forEach((p) => {
-              timelineOutputStream.push(
-                {
-                  // for testing we don't need real person_ids
-                  person_id: p.person_id || Math.floor(Math.random() * 1000000),
-                  email: p.email,
-                  entry_type: 'EMAIL_DELIVERED',
-                },
-              );
-            });
-          } else {
-            throw new Error(`output.timeline did not put a timelineOutputStream into the bindings:${Object.keys(props)}`);
-          }
+
+          batch.forEach((p) => {
+            if (Math.random() > 0.9) {
+              sampleOutputFileStream.push({
+                // for testing we don't need real person_ids
+                person_id: p.person_id || Math.floor(Math.random() * 1000000),
+                email: p.email,
+                entry_type: 'SAMPLE_OUTPUT',
+              });
+            }
+            timelineOutputFileStream.push(
+              {
+                // for testing we don't need real person_ids
+                person_id: p.person_id || Math.floor(Math.random() * 1000000),
+                email: p.email,
+                entry_type: 'EMAIL_DELIVERED',
+              },
+            );
+          });
+
           batch.forEach(() => { counter += 1; });
         },
       },
     );
+    assert(result.outputFiles?.timelineOutputFileStream?.[0]?.records);
+    assert(result.outputFiles?.sampleOutputFileStream?.[0]?.records);
     assert.equal(counter, 1000, `Expected to loop through 1000 people, actual:${counter}`);
   });
   debug('Completed tests');

package/test/packet/timelineStream.js

@@ -1,26 +0,0 @@
-const {
-  describe, it,
-} = require('node:test');
-const fs = require('node:fs');
-const assert = require('node:assert');
-
-const { getTimelineOutputStream } = require('../../index');
-
-describe('TimelineOutputStream', async () => {
-  it('timeline: It should save items to a csv file', async () => {
-    const {
-      stream: timelineStream, promises,
-      files,
-    } = await getTimelineOutputStream({});
-    timelineStream.push({ foo: 'bar' });
-    // finish the input stream
-    timelineStream.push(null);
-    await promises[0];
-    const content = fs.readFileSync(files[0]).toString().split('\n').map((d) => d.trim())
-      .filter(Boolean);
-
-    const s = 'uuid,entry_type,person_id,reference_id';
-    assert.equal(content[0].slice(0, s.length), s, "Beginning of first line doesn't match expected timeline csv header");
-    assert.equal(content.length, 2, `There are ${content.length}, not 2 lines in the CSV file`);
-  });
-});