@engine9-io/input-tools 1.9.2 → 1.9.4

package/ForEachEntry.js

@@ -17,9 +17,7 @@ const handlebars = require('handlebars');
 const ValidatingReadable = require('./ValidatingReadable');
 const FileUtilities = require('./file/FileUtilities');
 
-const {
-  getTempFilename, getBatchTransform, getFile, streamPacket,
-} = require('./file/tools');
+const { getTempFilename, getBatchTransform, getFile, streamPacket } = require('./file/tools');
 
 class ForEachEntry {
   constructor({ accountId } = {}) {
@@ -31,45 +29,52 @@ class ForEachEntry {
     if (this.outputStreams[name]?.items) return this.outputStreams[name].items;
 
     this.outputStreams[name] = this.outputStreams[name] || {
-      mutex: new Mutex(),
+      mutex: new Mutex()
     };
 
     return this.outputStreams[name].mutex.runExclusive(async () => {
       const fileInfo = {
         filename: await getTempFilename({ postfix }),
-        records: 0,
+        records: 0
       };
 
       debug(`Output file requested, writing output to to: ${fileInfo.filename}`);
-      const outputStream = new ValidatingReadable({
-        objectMode: true,
-      }, validatorFunction);
-      // eslint-disable-next-line no-underscore-dangle
+      const outputStream = new ValidatingReadable(
+        {
+          objectMode: true
+        },
+        validatorFunction
+      );
+
       outputStream._read = () => {};
 
       const writeStream = fs.createWriteStream(fileInfo.filename);
       const finishWritingOutputPromise = new Promise((resolve, reject) => {
-        writeStream.on('finish', () => {
-          resolve();
-        }).on('error', (err) => {
-          reject(err);
-        });
+        writeStream
+          .on('finish', () => {
+            resolve();
+          })
+          .on('error', (err) => {
+            reject(err);
+          });
       });
 
       this.outputStreams[name].items = {
         stream: outputStream,
         promises: [finishWritingOutputPromise],
-        files: [fileInfo],
+        files: [fileInfo]
       };
 
       outputStream
-        .pipe(new Transform({
-          objectMode: true,
-          transform(o, enc, cb) {
-            fileInfo.records += 1;
-            cb(null, o);
-          },
-        }))
+        .pipe(
+          new Transform({
+            objectMode: true,
+            transform(o, enc, cb) {
+              fileInfo.records += 1;
+              cb(null, o);
+            }
+          })
+        )
         .pipe(csv.stringify({ header: true }))
         .pipe(writeStream);
 
@@ -80,10 +85,11 @@ class ForEachEntry {
   async process({
     packet,
     filename,
+    progress,
     transform: userTransform,
     batchSize = 500,
     concurrency = 10,
-    bindings = {},
+    bindings = {}
   }) {
     let inStream = null;
 
@@ -97,6 +103,22 @@ class ForEachEntry {
     if (typeof userTransform !== 'function') throw new Error('async transform function is required');
     if (userTransform.length > 1) throw new Error('transform should be an async function that accepts one argument');
 
+    let progressThrottle = () => {};
+    if (typeof progress === 'function') {
+      const startTime = new Date().getTime();
+      progressThrottle = throttle(
+        2000,
+        function ({ records, batches }) {
+          let message = `Processed ${records} across ${batches} batches,${(
+            (records * 60 * 1000) /
+            (new Date().getTime() - startTime)
+          ).toFixed(1)} records/minute`;
+          progress({ records, message });
+        },
+        { noLeading: false, noTrailing: false }
+      );
+    }
+
     let records = 0;
     let batches = 0;
 
@@ -110,72 +132,80 @@ class ForEachEntry {
     const newStreams = [];
 
     const bindingNames = Object.keys(bindings);
-    // eslint-disable-next-line no-await-in-loop
-    await Promise.all(bindingNames.map(async (bindingName) => {
-      const binding = bindings[bindingName];
-      if (!binding.path) throw new Error(`Invalid binding: path is required for binding ${bindingName}`);
-      if (binding.path === 'output.timeline') {
-        const { stream: streamImpl, promises, files } = await this.getOutputStream({
-          name: bindingName,
-          postfix: binding.options?.postfix || '.timeline.csv',
-          validatorFunction: (data) => {
-            if (!data) return true;
-            if (typeof data !== 'object') throw new Error('Invalid timeline data push, must be an object');
-            // Is this necessary?
-            if (!data.person_id) throw new Error('Invalid timeline data push, must have a person_id, even if 0');
-            if (!data.ts) data.ts = new Date().toISOString();
-            return true;
-          },
-        });
-        newStreams.push(streamImpl);
-        transformArguments[bindingName] = streamImpl;
-        bindingPromises = bindingPromises.concat(promises || []);
-        outputFiles[bindingName] = files;
-      } else if (binding.path === 'output.stream') {
-        const { stream: streamImpl, promises, files } = await this.getOutputStream({
-          name: bindingName,
-          postfix: binding.options?.postfix || '.timeline.csv',
-        });
-        newStreams.push(streamImpl);
-        transformArguments[bindingName] = streamImpl;
-        bindingPromises = bindingPromises.concat(promises || []);
-        outputFiles[bindingName] = files;
-      } else if (binding.path === 'file') {
-        transformArguments[bindingName] = await getFile(binding);
-      } else if (binding.path === 'handlebars') {
-        transformArguments[bindingName] = handlebars;
-      } else {
-        throw new Error(`Unsupported binding path for binding ${bindingName}: ${binding.path}`);
-      }
-    }));
+
+    await Promise.all(
+      bindingNames.map(async (bindingName) => {
+        const binding = bindings[bindingName];
+        if (!binding.path) throw new Error(`Invalid binding: path is required for binding ${bindingName}`);
+        if (binding.path === 'output.timeline') {
+          const {
+            stream: streamImpl,
+            promises,
+            files
+          } = await this.getOutputStream({
+            name: bindingName,
+            postfix: binding.options?.postfix || '.timeline.csv',
+            validatorFunction: (data) => {
+              if (!data) return true;
+              if (typeof data !== 'object') throw new Error('Invalid timeline data push, must be an object');
+              // Is this necessary?
+              if (!data.person_id) throw new Error('Invalid timeline data push, must have a person_id, even if 0');
+              if (!data.ts) data.ts = new Date().toISOString();
+              return true;
+            }
+          });
+          newStreams.push(streamImpl);
+          transformArguments[bindingName] = streamImpl;
+          bindingPromises = bindingPromises.concat(promises || []);
+          outputFiles[bindingName] = files;
+        } else if (binding.path === 'output.stream') {
+          const {
+            stream: streamImpl,
+            promises,
+            files
+          } = await this.getOutputStream({
+            name: bindingName,
+            postfix: binding.options?.postfix || '.timeline.csv'
+          });
+          newStreams.push(streamImpl);
+          transformArguments[bindingName] = streamImpl;
+          bindingPromises = bindingPromises.concat(promises || []);
+          outputFiles[bindingName] = files;
+        } else if (binding.path === 'file') {
+          transformArguments[bindingName] = await getFile(binding);
+        } else if (binding.path === 'handlebars') {
+          transformArguments[bindingName] = handlebars;
+        } else {
+          throw new Error(`Unsupported binding path for binding ${bindingName}: ${binding.path}`);
+        }
+      })
+    );
     await pipeline(
       inStream,
       csv.parse({
         relax: true,
         skip_empty_lines: true,
         max_limit_on_data_read: 10000000,
-        columns: true,
+        columns: true
       }),
       getBatchTransform({ batchSize }).transform,
-      parallelTransform(
-        concurrency,
-        (batch, cb) => {
-          userTransform({ ...transformArguments, batch })
-            .then((d) => cb(null, d))
-            .catch(cb);
-        },
-
-      ),
+      parallelTransform(concurrency, (batch, cb) => {
+        userTransform({ ...transformArguments, batch })
+          .then((d) => {
+            batches += 1;
+            records += batch?.length || 0;
+            progressThrottle({ records, batches });
+            debugThrottle(`Processed ${batches} batches for a total of ${records} outbound records`);
+            cb(null, d);
+          })
+          .catch(cb);
+      }),
       new Writable({
         objectMode: true,
         write(batch, enc, cb) {
-          batches += 1;
-          records += batch?.length || 0;
-
-          debugThrottle(`Processed ${batches} batches for a total of ${records} outbound records`);
           cb();
-        },
-      }),
+        }
+      })
     );
     debug('Completed all batches');
 

package/file/FileUtilities.js

@@ -890,7 +890,7 @@ Worker.prototype.stat = async function ({ filename }) {
 
   if (filename.slice(-8) === '.parquet') {
     const pq = new ParquetWorker(this);
-    output.schema = await pq.schema({ filename });
+    output.schema = (await pq.schema({ filename }))?.schema;
     output.records = (await pq.meta({ filename }))?.records;
   }
 

package/file/tools.js

@@ -351,6 +351,15 @@ function appendPostfix(filename, postfix) {
   return filenameParts.slice(0, -1).concat(targetFile).join('/');
 }
 
+function parseJSON5(o, defaultVal) {
+  if (o) {
+    if (typeof o === 'object') return o;
+    if (typeof o === 'string') return JSON5.parse(o);
+    throw new Error(`Could not parse object:${o}`);
+  }
+  return defaultVal || o;
+}
+
 module.exports = {
   appendPostfix,
   bool,
@@ -364,6 +373,7 @@ module.exports = {
   getPacketFiles,
   getStringArray,
   makeStrings,
+  parseJSON5,
   relativeDate,
   streamPacket,
   writeTempFile

package/index.js

@@ -15,18 +15,19 @@ const FileUtilities = require('./file/FileUtilities');
 const {
   appendPostfix,
   bool,
-  getManifest,
+  getBatchTransform,
+  getDebatchTransform,
   getFile,
+  getManifest,
+  getPacketFiles,
+  getStringArray,
   downloadFile,
   getTempFilename,
   getTempDir,
   isValidDate,
+  parseJSON5,
   relativeDate,
   streamPacket,
-  getPacketFiles,
-  getBatchTransform,
-  getDebatchTransform,
-  getStringArray,
   makeStrings,
   writeTempFile
 } = require('./file/tools');
@@ -319,6 +320,79 @@ function getEntryType(o, defaults = {}) {
   return etype;
 }
 
+function getDateRangeArray(startDate, endDate) {
+  const start = new Date(startDate);
+  const end = new Date(endDate);
+  const result = [];
+  const msInDay = 24 * 60 * 60 * 1000;
+
+  function addDays(date, days) {
+    const d = new Date(date);
+    d.setDate(d.getDate() + days);
+    return d;
+  }
+  function addMonths(date, months) {
+    const d = new Date(date);
+    d.setMonth(d.getMonth() + months);
+    return d;
+  }
+  function addYears(date, years) {
+    const d = new Date(date);
+    d.setFullYear(d.getFullYear() + years);
+    return d;
+  }
+
+  const diffDays = Math.floor((end - start) / msInDay);
+  const diffMonths = (end.getFullYear() - start.getFullYear()) * 12 + (end.getMonth() - start.getMonth());
+  const diffYears = end.getFullYear() - start.getFullYear();
+
+  let current = new Date(start);
+
+  let stepFn;
+  if (diffDays < 10) {
+    stepFn = (date) => addDays(date, 1);
+  } else if (diffDays < 32) {
+    stepFn = (date) => addDays(date, 3);
+  } else if (diffMonths < 4) {
+    stepFn = (date) => addDays(date, 7);
+  } else if (diffYears < 2) {
+    stepFn = (date) => addMonths(date, 1);
+  } else if (diffYears < 4) {
+    stepFn = (date) => addMonths(date, 3);
+  } else {
+    stepFn = (date) => addYears(date, 1);
+  }
+
+  while (current <= end) {
+    result.push(new Date(current));
+    const next = stepFn(current);
+    if (next > end) break;
+    current = next;
+  }
+  // Ensure the last date is exactly the end date
+  if (result.length === 0 || result[result.length - 1].getTime() !== end.getTime()) {
+    result.push(new Date(end));
+  }
+  return result;
+}
+
+class ObjectError extends Error {
+  constructor(data) {
+    if (typeof data === 'string') {
+      // normal behavior
+      super(data);
+    } else if (typeof data === 'object') {
+      super(data.message);
+      Object.keys(data).forEach((k) => {
+        this[k] = data[k];
+      });
+      this.status = data.status;
+    } else {
+      super('(No error message)');
+    }
+  }
+}
+
 module.exports = {
   appendPostfix,
   bool,
@@ -329,6 +403,7 @@ module.exports = {
   ForEachEntry,
   FileUtilities,
   getBatchTransform,
+  getDateRangeArray,
   getDebatchTransform,
   getEntryType,
   getEntryTypeId,
@@ -346,6 +421,8 @@ module.exports = {
   handlebars,
   isValidDate,
   makeStrings,
+  ObjectError,
+  parseJSON5,
   relativeDate,
   streamPacket,
   TIMELINE_ENTRY_TYPES,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.9.2",
+  "version": "1.9.4",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {

package/test/processing/bigDataMessage.js

@@ -1,10 +1,8 @@
-const {
-  describe, it,
-} = require('node:test');
+const { describe, it } = require('node:test');
 const assert = require('node:assert');
 const debug = require('debug')('test:big-data');
 const { setTimeout } = require('node:timers/promises');
-const { v7: uuidv7 } = require('uuid');
+//const { v7: uuidv7 } = require('uuid');
 
 const { ForEachEntry } = require('../../index');
 
@@ -14,46 +12,40 @@ describe('big-data message: forEachPerson', async () => {
     let counter = 0;
     const forEach = new ForEachEntry();
 
-    const output = await forEach.process(
-      {
-        // packet: '../1000000_person_message.packet.zip',
-        filename: '../1000000_fake_people.csv',
-        batchSize: 10000,
-        concurrency: 1000,
-        bindings: {
-          timelineOutputStream: { path: 'output.timeline' },
-          message: { path: 'file', filename: '../1000000_person_message.packet/message/message.json5' },
-          handlebars: { path: 'handlebars' },
-        },
-        async transform({
-          batch,
-          message,
-          handlebars,
-          timelineOutputStream,
-        }) {
-          const id = uuidv7();
-          debug(`Processing batch of ${batch.length} - ${id}`);
-          if (!message?.content?.text) throw new Error(`Sample message has no content.text:${JSON.stringify(message)}`);
-          const template = handlebars.compile(message.content.text);
-          batch.forEach((person) => {
-            messageContent.push(template(person));
-          });
-          batch.forEach((p) => {
-            const o = {
-              person_id: p.person_id,
-              email: p.email,
-              entry_type: 'EMAIL_DELIVERED',
-            };
-            counter += 1;
-            if (counter % 10000 === 0) debug(`*** Processed ${counter} items, last person_id=${p.person_id}`, o);
-            timelineOutputStream.push(o);
-          });
-          // debug(`Processed batch of size ${batch.length}`);
-          await setTimeout(Math.random() * 3000);
-          debug(`Completed processing ${id}`);
-        },
+    const output = await forEach.process({
+      // packet: '../1000000_person_message.packet.zip',
+      filename: '../1000000_person_message.packet/person/1000000_fake_people.csv',
+      batchSize: 10000,
+      concurrency: 1000,
+      progress: debug,
+      bindings: {
+        timelineOutputStream: { path: 'output.timeline' },
+        message: { path: 'file', filename: '../1000000_person_message.packet/message/message.json5' },
+        handlebars: { path: 'handlebars' }
       },
-    );
+      async transform({ batch, message, handlebars, timelineOutputStream }) {
+        //const id = uuidv7();
+        //debug(`Processing batch of ${batch.length} - ${id}`);
+        if (!message?.content?.text) throw new Error(`Sample message has no content.text:${JSON.stringify(message)}`);
+        const template = handlebars.compile(message.content.text);
+        batch.forEach((person) => {
+          messageContent.push(template(person));
+        });
+        batch.forEach((p) => {
+          const o = {
+            person_id: p.person_id,
+            email: p.email,
+            entry_type: 'EMAIL_DELIVERED'
+          };
+          counter += 1;
+          //if (counter % 10000 === 0) debug(`*** Processed ${counter} items, last person_id=${p.person_id}`, o);
+          timelineOutputStream.push(o);
+        });
+        // debug(`Processed batch of size ${batch.length}`);
+        await setTimeout(Math.random() * 3000);
+        //debug(`Completed processing ${id}`);
+      }
+    });
     debug(output);
 
     assert.equal(counter, 1000000, `Expected to loop through 1000000 people, actual:${counter}`);

package/timelineTypes.js

@@ -31,6 +31,7 @@ const SMS_DELIVERED = 31;
 const SMS_CLICK = 33;
 const SMS_UNSUBSCRIBE = 34;
 const SMS_BOUNCE = 37;
+const SMS_SPAM = 38;
 const SMS_REPLY = 39;
 
 const EMAIL_SEND = 40;
@@ -112,6 +113,7 @@ const TIMELINE_ENTRY_TYPES = {
   SMS_CLICK,
   SMS_UNSUBSCRIBE,
   SMS_BOUNCE,
+  SMS_SPAM,
   SMS_REPLY,
 
   // email interactions