@engine9-io/input-tools 1.7.8 → 1.7.9

package/file/tools.js

@@ -14,16 +14,9 @@ const unzipper = require('unzipper');
 
 const dayjs = require('dayjs');
 
-const {
-  S3Client,
-  HeadObjectCommand,
-  GetObjectCommand,
-} = require('@aws-sdk/client-s3');
+const { S3Client, HeadObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3');
 
-
-const {
-  v7: uuidv7,
-} = require('uuid');
+const { v7: uuidv7 } = require('uuid');
 
 async function getTempDir({ accountId = 'engine9' }) {
   const dir = [os.tmpdir(), accountId, new Date().toISOString().substring(0, 10)].join(path.sep);
@@ -52,7 +45,10 @@ async function getTempFilename(options) {
     }
 
     // make a distinct directory, so we don't overwrite the file
-    dir = `${dir}/${new Date().toISOString().slice(0, -6).replace(/[^0-9]/g, '_')}`;
+    dir = `${dir}/${new Date()
+      .toISOString()
+      .slice(0, -6)
+      .replace(/[^0-9]/g, '_')}`;
 
     const newDir = await mkdirp(dir);
 
@@ -97,8 +93,8 @@ async function getPacketFiles({ packet }) {
         const info = await s3Client.send(
           new HeadObjectCommand({
             Bucket,
-            Key,
-          }),
+            Key
+          })
         );
         size = info.ContentLength;
         progress(`Retrieving file of size ${size / (1024 * 1024)} MB`);
@@ -107,13 +103,14 @@ async function getPacketFiles({ packet }) {
 
       stream(offset, length) {
         const ptStream = new PassThrough();
-        s3Client.send(
-          new GetObjectCommand({
-            Bucket,
-            Key,
-            Range: `bytes=${offset}-${length ?? ''}`,
-          }),
-        )
+        s3Client
+          .send(
+            new GetObjectCommand({
+              Bucket,
+              Key,
+              Range: `bytes=${offset}-${length ?? ''}`
+            })
+          )
           .then((response) => {
             response.Body.pipe(ptStream);
           })
@@ -122,7 +119,7 @@ async function getPacketFiles({ packet }) {
           });
 
         return ptStream;
-      },
+      }
     });
 
     return directory;
@@ -131,7 +128,6 @@ async function getPacketFiles({ packet }) {
   return directory;
 }
 
-
 async function getManifest({ packet }) {
   if (!packet) throw new Error('no packet option specififed');
   const { files } = await getPacketFiles({ packet });
@@ -156,8 +152,8 @@ function getBatchTransform({ batchSize = 100 }) {
       flush(cb) {
         if (this.buffer?.length > 0) this.push(this.buffer);
         cb();
-      },
-    }),
+      }
+    })
   };
 }
 function getDebatchTransform() {
@@ -167,8 +163,8 @@ function getDebatchTransform() {
       transform(chunk, encoding, cb) {
         chunk.forEach((c) => this.push(c));
         cb();
-      },
-    }),
+      }
+    })
   };
 }
 
@@ -218,7 +214,8 @@ async function downloadFile({ packet, type = 'person' }) {
   const filename = await getTempFilename({ targetFilename: filePath.split('/').pop() });
 
   return new Promise((resolve, reject) => {
-    fileStream.pipe(fs.createWriteStream(filename))
+    fileStream
+      .pipe(fs.createWriteStream(filename))
       .on('error', reject)
       .on('finish', () => {
         resolve({ filename });
@@ -228,12 +225,12 @@ async function downloadFile({ packet, type = 'person' }) {
 
 function isValidDate(d) {
   // we WANT to use isNaN, not the Number.isNaN -- we're checking the date type
-  // eslint-disable-next-line no-restricted-globals
+
   return d instanceof Date && !isNaN(d);
 }
 
 function bool(x, _defaultVal) {
-  const defaultVal = (_defaultVal === undefined) ? false : _defaultVal;
+  const defaultVal = _defaultVal === undefined ? false : _defaultVal;
   if (x === undefined || x === null || x === '') return defaultVal;
   if (typeof x !== 'string') return !!x;
   if (x === '1') return true; // 0 will return false, but '1' is true
@@ -255,7 +252,7 @@ function relativeDate(s, _initialDate) {
   if (!s || s === 'none') return null;
   if (typeof s.getMonth === 'function') return s;
   // We actually want a double equals here to test strings as well
-  // eslint-disable-next-line eqeqeq
+
   if (parseInt(s, 10) == s) {
     const r = new Date(parseInt(s, 10));
     if (!isValidDate(r)) throw new Error(`Invalid integer date:${s}`);
@@ -274,15 +271,31 @@ function relativeDate(s, _initialDate) {
     let period = null;
     switch (r[3]) {
       case 'Y':
-      case 'y': period = 'years'; break;
-
-      case 'M': period = 'months'; break;
-      case 'w': period = 'weeks'; break;
-      case 'd': period = 'days'; break;
-      case 'h': period = 'hours'; break;
-      case 'm': period = 'minutes'; break;
-      case 's': period = 'seconds'; break;
-      default: period = 'minutes'; break;
+      case 'y':
+        period = 'years';
+        break;
+
+      case 'M':
+        period = 'months';
+        break;
+      case 'w':
+        period = 'weeks';
+        break;
+      case 'd':
+        period = 'days';
+        break;
+      case 'h':
+        period = 'hours';
+        break;
+      case 'm':
+        period = 'minutes';
+        break;
+      case 's':
+        period = 'seconds';
+        break;
+      default:
+        period = 'minutes';
+        break;
     }
 
     let d = dayjs(initialDate);
@@ -317,12 +330,29 @@ function relativeDate(s, _initialDate) {
 */
 function makeStrings(o) {
   return Object.entries(o).reduce((a, [k, v]) => {
-    a[k] = (typeof v === 'object') ? JSON.stringify(v) : String(v);
+    a[k] = typeof v === 'object' ? JSON.stringify(v) : String(v);
     return a;
   }, {});
 }
+function appendPostfix(filename, postfix) {
+  const filenameParts = filename.split('/');
+  const fileParts = filenameParts
+    .slice(-1)[0]
+    .split('.')
+    .filter(Boolean)
+    .filter((d) => d !== postfix);
+
+  let targetFile = null;
+  if (fileParts.slice(-1)[0] === 'gz') {
+    targetFile = fileParts.slice(0, -2).concat(postfix).concat(fileParts.slice(-2)).join('.');
+  } else {
+    targetFile = fileParts.slice(0, -1).concat(postfix).concat(fileParts.slice(-1)).join('.');
+  }
+  return filenameParts.slice(0, -1).concat(targetFile).join('/');
+}
 
 module.exports = {
+  appendPostfix,
   bool,
   downloadFile,
   getTempFilename,
@@ -336,5 +366,5 @@ module.exports = {
   makeStrings,
   relativeDate,
   streamPacket,
-  writeTempFile,
+  writeTempFile
 };

package/index.js

@@ -6,15 +6,14 @@ const dayjs = require('dayjs');
 const debug = require('debug')('@engine9/input-tools');
 
 const unzipper = require('unzipper');
-const {
-  v4: uuidv4, v5: uuidv5, v7: uuidv7, validate: uuidIsValid,
-} = require('uuid');
+const { v4: uuidv4, v5: uuidv5, v7: uuidv7, validate: uuidIsValid } = require('uuid');
 const archiver = require('archiver');
 const handlebars = require('handlebars');
 
 const FileUtilities = require('./file/FileUtilities');
 
 const {
+  appendPostfix,
   bool,
   getManifest,
   getFile,
@@ -29,7 +28,7 @@ const {
   getDebatchTransform,
   getStringArray,
   makeStrings,
-  writeTempFile,
+  writeTempFile
 } = require('./file/tools');
 
 const ForEachEntry = require('./ForEachEntry');
@@ -45,7 +44,7 @@ function getFormattedDate(dateObject, format = 'MMM DD,YYYY') {
 
 handlebars.registerHelper('date', (d, f) => {
   let format;
-  if (typeof f === 'string')format = f;
+  if (typeof f === 'string') format = f;
   return getFormattedDate(d, format);
 });
 handlebars.registerHelper('json', (d) => JSON.stringify(d));
@@ -60,11 +59,7 @@ async function list(_path) {
   const directory = await unzipper.Open.file(_path);
 
   return new Promise((resolve, reject) => {
-    directory.files[0]
-      .stream()
-      .pipe(fs.createWriteStream('firstFile'))
-      .on('error', reject)
-      .on('finish', resolve);
+    directory.files[0].stream().pipe(fs.createWriteStream('firstFile')).on('error', reject).on('finish', resolve);
   });
 }
 
@@ -74,11 +69,7 @@ async function extract(_path, _file) {
   const file = directory.files.find((d) => d.path === _file);
   const tempFilename = await getTempFilename({ source: _file });
   return new Promise((resolve, reject) => {
-    file
-      .stream()
-      .pipe(fs.createWriteStream(tempFilename))
-      .on('error', reject)
-      .on('finish', resolve);
+    file.stream().pipe(fs.createWriteStream(tempFilename)).on('error', reject).on('finish', resolve);
   });
 }
 
@@ -87,7 +78,7 @@ function appendFiles(existingFiles, _newFiles, options) {
   if (newFiles.length === 0) return;
   let { type, dateCreated } = options || {};
   if (!type) type = 'unknown';
-  if (!dateCreated)dateCreated = new Date().toISOString();
+  if (!dateCreated) dateCreated = new Date().toISOString();
   let arr = newFiles;
   if (!Array.isArray(newFiles)) arr = [arr];
 
@@ -96,7 +87,7 @@ function appendFiles(existingFiles, _newFiles, options) {
       type,
       originalFilename: '',
       isNew: true,
-      dateCreated,
+      dateCreated
     };
 
     if (typeof p === 'string') {
@@ -121,7 +112,7 @@ async function create(options) {
     messageFiles = [], // file with contents of message, used for delivery
     personFiles = [], // files with data on people
     timelineFiles = [], // activity entry
-    statisticsFiles = [], // files with aggregate statistics
+    statisticsFiles = [] // files with aggregate statistics
   } = options;
   if (options.peopleFiles) throw new Error('Unknown option: peopleFiles, did you mean personFiles?');
 
@@ -132,21 +123,21 @@ async function create(options) {
   appendFiles(files, timelineFiles, { type: 'timeline', dateCreated });
   appendFiles(files, statisticsFiles, { type: 'statistics', dateCreated });
 
-  const zipFilename = target || await getTempFilename({ postfix: '.packet.zip' });
+  const zipFilename = target || (await getTempFilename({ postfix: '.packet.zip' }));
 
   const manifest = {
     accountId,
     source: {
-      pluginId,
+      pluginId
     },
     dateCreated,
-    files,
+    files
   };
 
   // create a file to stream archive data to.
   const output = fs.createWriteStream(zipFilename);
   const archive = archiver('zip', {
-    zlib: { level: 9 }, // Sets the compression level.
+    zlib: { level: 9 } // Sets the compression level.
   });
   return new Promise((resolve, reject) => {
     debug(`Setting up write stream to ${zipFilename}`);
@@ -157,7 +148,7 @@ async function create(options) {
       debug(zipFilename);
       return resolve({
         filename: zipFilename,
-        bytes: archive.pointer(),
+        bytes: archive.pointer()
       });
     });
 
@@ -196,7 +187,6 @@ function intToByteArray(_v) {
   const byteArray = [0, 0, 0, 0, 0, 0, 0, 0];
   let v = _v;
   for (let index = 0; index < byteArray.length; index += 1) {
-    // eslint-disable-next-line no-bitwise
     const byte = v & 0xff;
     byteArray[index] = byte;
     v = (v - byte) / 256;
@@ -226,23 +216,26 @@ function getInputUUID(a, b) {
   return uuidv5(`${pluginId}:${rid}`, '3d0e5d99-6ba9-4fab-9bb2-c32304d3df8e');
 }
 
-function getUUIDv7(date, inputUuid) { /* optional date and input UUID */
+function getUUIDv7(date, inputUuid) {
+  /* optional date and input UUID */
   const uuid = inputUuid || uuidv7();
   const bytes = Buffer.from(uuid.replace(/-/g, ''), 'hex');
   if (date !== undefined) {
     const d = new Date(date);
     // isNaN behaves differently than Number.isNaN -- we're actually going for the
     // attempted conversion here
-    // eslint-disable-next-line no-restricted-globals
+
     if (isNaN(d)) throw new Error(`getUUIDv7 got an invalid date:${date || '<blank>'}`);
     const dateBytes = intToByteArray(d.getTime()).reverse();
-    dateBytes.slice(2, 8).forEach((b, i) => { bytes[i] = b; });
+    dateBytes.slice(2, 8).forEach((b, i) => {
+      bytes[i] = b;
+    });
   }
   return uuidv4({ random: bytes });
 }
 /* Returns a date from a given uuid (assumed to be a v7, otherwise the results are ... weird */
 function getUUIDTimestamp(uuid) {
-  const ts = parseInt((`${uuid}`).replace(/-/g, '').slice(0, 12), 16);
+  const ts = parseInt(`${uuid}`.replace(/-/g, '').slice(0, 12), 16);
   return new Date(ts);
 }
 
@@ -266,7 +259,8 @@ function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
 
   if (o.remote_entry_id) {
     // get a temp ID
-    if (!o.input_id) throw new Error('Error generating timeline entry uuid -- remote_entry_id specified, but no input_id');
+    if (!o.input_id)
+      throw new Error('Error generating timeline entry uuid -- remote_entry_id specified, but no input_id');
     const uuid = uuidv5(o.remote_entry_id, o.input_id);
     // Change out the ts to match the v7 sorting.
     // But because outside specified remote_entry_uuid
@@ -274,14 +268,13 @@ function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
     return getUUIDv7(o.ts, uuid);
   }
 
-  const missing = requiredTimelineEntryFields
-    .filter((d) => o[d] === undefined);// 0 could be an entry type value
+  const missing = requiredTimelineEntryFields.filter((d) => o[d] === undefined); // 0 could be an entry type value
 
   if (missing.length > 0) throw new Error(`Missing required fields to append an entry_id:${missing.join(',')}`);
   const ts = new Date(o.ts);
   // isNaN behaves differently than Number.isNaN -- we're actually going for the
   // attempted conversion here
-  // eslint-disable-next-line no-restricted-globals
+
   if (isNaN(ts)) throw new Error(`getTimelineEntryUUID got an invalid date:${o.ts || '<blank>'}`);
   const idString = `${ts.toISOString()}-${o.person_id}-${o.entry_type_id}-${o.source_code_id || 0}`;
 
@@ -308,6 +301,7 @@ function getEntryTypeId(o, { defaults = {} } = {}) {
 }
 
 module.exports = {
+  appendPostfix,
   bool,
   create,
   list,
@@ -339,5 +333,5 @@ module.exports = {
   uuidIsValid,
   uuidv4,
   uuidv5,
-  uuidv7,
+  uuidv7
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.7.8",
+  "version": "1.7.9",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {

package/test/processing/forEach.js

@@ -0,0 +1,55 @@
+const { describe, it } = require('node:test');
+const assert = require('node:assert');
+const debug = require('debug')('test/forEach');
+
+const { ForEachEntry } = require('../../index');
+
+describe('Test Person File For Each', async () => {
+  it('forEachPerson Should loop through 1000 sample people', async () => {
+    let counter = 0;
+    const forEach = new ForEachEntry();
+    const result = await forEach.process({
+      packet: 'test/sample/1000_message.packet.zip',
+      batchSize: 50,
+      bindings: {
+        timelineOutputFileStream: {
+          path: 'output.timeline',
+          options: {
+            entry_type: 'ENTRY_OPTION'
+          }
+        },
+        sampleOutputFileStream: {
+          path: 'output.stream'
+        }
+      },
+      async transform(props) {
+        const { batch, timelineOutputFileStream, sampleOutputFileStream } = props;
+
+        batch.forEach((p) => {
+          if (Math.random() > 0.9) {
+            sampleOutputFileStream.push({
+              // for testing we don't need real person_ids
+              person_id: p.person_id || Math.floor(Math.random() * 1000000),
+              email: p.email,
+              entry_type: 'SAMPLE_OUTPUT'
+            });
+          }
+          timelineOutputFileStream.push({
+            // for testing we don't need real person_ids
+            person_id: p.person_id || Math.floor(Math.random() * 1000000),
+            email: p.email,
+            entry_type: 'EMAIL_DELIVERED'
+          });
+        });
+
+        batch.forEach(() => {
+          counter += 1;
+        });
+      }
+    });
+    assert(result.outputFiles?.timelineOutputFileStream?.[0]?.records);
+    assert(result.outputFiles?.sampleOutputFileStream?.[0]?.records);
+    assert.equal(counter, 1000, `Expected to loop through 1000 people, actual:${counter}`);
+  });
+  debug('Completed tests');
+});

package/test/packet/forEach.js

@@ -1,63 +0,0 @@
-const {
-  describe, it,
-} = require('node:test');
-const assert = require('node:assert');
-const debug = require('debug')('test/forEach');
-
-const { ForEachEntry } = require('../../index');
-
-describe('Test Person Packet For Each', async () => {
-  it('forEachPerson Should loop through 1000 sample people', async () => {
-    let counter = 0;
-    const forEach = new ForEachEntry();
-    const result = await forEach.process(
-      {
-        packet: 'test/sample/1000_message.packet.zip',
-        batchSize: 50,
-        bindings: {
-          timelineOutputFileStream: {
-            path: 'output.timeline',
-            options: {
-              entry_type: 'ENTRY_OPTION',
-            },
-          },
-          sampleOutputFileStream: {
-            path: 'output.stream',
-          },
-        },
-        async transform(props) {
-          const {
-            batch,
-            timelineOutputFileStream,
-            sampleOutputFileStream,
-          } = props;
-
-          batch.forEach((p) => {
-            if (Math.random() > 0.9) {
-              sampleOutputFileStream.push({
-                // for testing we don't need real person_ids
-                person_id: p.person_id || Math.floor(Math.random() * 1000000),
-                email: p.email,
-                entry_type: 'SAMPLE_OUTPUT',
-              });
-            }
-            timelineOutputFileStream.push(
-              {
-                // for testing we don't need real person_ids
-                person_id: p.person_id || Math.floor(Math.random() * 1000000),
-                email: p.email,
-                entry_type: 'EMAIL_DELIVERED',
-              },
-            );
-          });
-
-          batch.forEach(() => { counter += 1; });
-        },
-      },
-    );
-    assert(result.outputFiles?.timelineOutputFileStream?.[0]?.records);
-    assert(result.outputFiles?.sampleOutputFileStream?.[0]?.records);
-    assert.equal(counter, 1000, `Expected to loop through 1000 people, actual:${counter}`);
-  });
-  debug('Completed tests');
-});