@engine9-io/input-tools 1.9.10 → 2.0.0

package/index.js

@@ -1,17 +1,17 @@
-const fs = require('node:fs');
-
-const path = require('node:path');
-const dayjs = require('dayjs');
-
-const debug = require('debug')('@engine9/input-tools');
-
-const unzipper = require('unzipper');
-const { v4: uuidv4, v5: uuidv5, v7: uuidv7, validate: uuidIsValid } = require('uuid');
-const archiver = require('archiver');
-const handlebars = require('handlebars');
-
-const FileUtilities = require('./file/FileUtilities');
-
+import fs from 'node:fs';
+import path from 'node:path';
+import dayjs from 'dayjs';
+import debug$0 from 'debug';
+import unzipper from 'unzipper';
+import uuid from 'uuid';
+import archiver from 'archiver';
+import handlebars from 'handlebars';
+import FileUtilities from './file/FileUtilities.js';
+import tools from './file/tools.js';
+import ForEachEntry from './ForEachEntry.js';
+import { TIMELINE_ENTRY_TYPES } from './timelineTypes.js';
+const debug = debug$0('@engine9/input-tools');
+const { v4: uuidv4, v5: uuidv5, v7: uuidv7, validate: uuidIsValid } = uuid;
 const {
   appendPostfix,
   bool,
@@ -30,40 +30,28 @@ const {
   streamPacket,
   makeStrings,
   writeTempFile
-} = require('./file/tools');
-
-const ForEachEntry = require('./ForEachEntry');
-
-const { TIMELINE_ENTRY_TYPES } = require('./timelineTypes');
-
+} = tools;
 function getFormattedDate(dateObject, format = 'MMM DD,YYYY') {
   let d = dateObject;
   if (d === 'now') d = new Date();
   if (d) return dayjs(d).format(format);
   return '';
 }
-
 handlebars.registerHelper('date', (d, f) => {
   let format;
   if (typeof f === 'string') format = f;
   return getFormattedDate(d, format);
 });
 handlebars.registerHelper('json', (d) => JSON.stringify(d));
-
 handlebars.registerHelper('uuid', () => uuidv7());
-
 handlebars.registerHelper('percent', (a, b) => `${((100 * a) / b).toFixed(2)}%`);
-
 handlebars.registerHelper('or', (a, b, c) => a || b || c);
-
 async function list(_path) {
   const directory = await unzipper.Open.file(_path);
-
   return new Promise((resolve, reject) => {
     directory.files[0].stream().pipe(fs.createWriteStream('firstFile')).on('error', reject).on('finish', resolve);
   });
 }
-
 async function extract(_path, _file) {
   const directory = await unzipper.Open(_path);
   // return directory.files.map((f) => f.path);
@@ -73,7 +61,6 @@ async function extract(_path, _file) {
     file.stream().pipe(fs.createWriteStream(tempFilename)).on('error', reject).on('finish', resolve);
   });
 }
-
 function appendFiles(existingFiles, _newFiles, options) {
   const newFiles = getStringArray(_newFiles);
   if (newFiles.length === 0) return;
@@ -82,7 +69,6 @@ function appendFiles(existingFiles, _newFiles, options) {
   if (!dateCreated) dateCreated = new Date().toISOString();
   let arr = newFiles;
   if (!Array.isArray(newFiles)) arr = [arr];
-
   arr.forEach((p) => {
     const item = {
       type,
@@ -90,13 +76,11 @@ function appendFiles(existingFiles, _newFiles, options) {
       isNew: true,
       dateCreated
     };
-
     if (typeof p === 'string') {
       item.originalFilename = path.resolve(process.cwd(), p);
     } else {
       item.originalFilename = path.resolve(process.cwd(), item.originalFilename);
     }
-
     const file = item.originalFilename.split(path.sep).pop();
     item.path = `${type}/${file}`;
     const existingFile = existingFiles.find((f) => f.path === item.path);
@@ -104,7 +88,6 @@ function appendFiles(existingFiles, _newFiles, options) {
     existingFiles.push(item);
   });
 }
-
 async function create(options) {
   const {
     accountId = 'engine9',
@@ -116,16 +99,13 @@ async function create(options) {
     statisticsFiles = [] // files with aggregate statistics
   } = options;
   if (options.peopleFiles) throw new Error('Unknown option: peopleFiles, did you mean personFiles?');
-
   const files = [];
   const dateCreated = new Date().toISOString();
   appendFiles(files, messageFiles, { type: 'message', dateCreated });
   appendFiles(files, personFiles, { type: 'person', dateCreated });
   appendFiles(files, timelineFiles, { type: 'timeline', dateCreated });
   appendFiles(files, statisticsFiles, { type: 'statistics', dateCreated });
-
   const zipFilename = target || (await getTempFilename({ postfix: '.packet.zip' }));
-
   const manifest = {
     accountId,
     source: {
@@ -134,7 +114,6 @@ async function create(options) {
     dateCreated,
     files
   };
-
   // create a file to stream archive data to.
   const output = fs.createWriteStream(zipFilename);
   const archive = archiver('zip', {
@@ -152,37 +131,30 @@ async function create(options) {
         bytes: archive.pointer()
       });
     });
-
     // This event is fired when the data source is drained no matter what was the data source.
     // It is not part of this library but rather from the NodeJS Stream API.
     // @see: https://nodejs.org/api/stream.html#stream_event_end
     output.on('end', () => {
       // debug('end event -- Data has been drained');
     });
-
     // warnings could be file not founds, etc, but we error even on those
     archive.on('warning', (err) => {
       reject(err);
     });
-
     // good practice to catch this error explicitly
     archive.on('error', (err) => {
       reject(err);
     });
-
     archive.pipe(output);
-
     files.forEach(({ path: name, originalFilename }) => archive.file(originalFilename, { name }));
     files.forEach((f) => {
       delete f.originalFilename;
       delete f.isNew;
     });
-
     archive.append(Buffer.from(JSON.stringify(manifest, null, 4), 'utf8'), { name: 'manifest.json' });
     archive.finalize();
   });
 }
-
 function intToByteArray(_v) {
   // we want to represent the input as a 8-bytes array
   const byteArray = [0, 0, 0, 0, 0, 0, 0, 0];
@@ -192,14 +164,12 @@ function intToByteArray(_v) {
     byteArray[index] = byte;
     v = (v - byte) / 256;
   }
-
   return byteArray;
 }
 function getPluginUUID(uniqueNamespaceLikeDomainName, valueWithinNamespace) {
   // Random custom namespace for plugins -- not secure, just a namespace:
   return uuidv5(`${uniqueNamespaceLikeDomainName}::${valueWithinNamespace}`, 'f9e1024d-21ac-473c-bac6-64796dd771dd');
 }
-
 function getInputUUID(a, b) {
   let pluginId = a;
   let remoteInputId = b;
@@ -207,7 +177,6 @@ function getInputUUID(a, b) {
     pluginId = a.pluginId;
     remoteInputId = a.remoteInputId;
   }
-
   if (!pluginId) throw new Error('getInputUUID: Cowardly rejecting a blank plugin_id');
   if (!uuidIsValid(pluginId)) throw new Error(`Invalid pluginId:${pluginId}, should be a UUID`);
   const rid = (remoteInputId || '').trim();
@@ -216,7 +185,6 @@ function getInputUUID(a, b) {
   // 3d0e5d99-6ba9-4fab-9bb2-c32304d3df8e
   return uuidv5(`${pluginId}:${rid}`, '3d0e5d99-6ba9-4fab-9bb2-c32304d3df8e');
 }
-
 const timestampMatch = /^\d{13}$/;
 function dateFromString(s) {
   if (typeof s === 'number') return new Date(s);
@@ -225,7 +193,6 @@ function dateFromString(s) {
   }
   return new Date(s);
 }
-
 function getUUIDv7(date, inputUuid) {
   /* optional date and input UUID */
   const uuid = inputUuid || uuidv7();
@@ -234,7 +201,6 @@ function getUUIDv7(date, inputUuid) {
     const d = dateFromString(date);
     // isNaN behaves differently than Number.isNaN -- we're actually going for the
     // attempted conversion here
-
     if (isNaN(d)) throw new Error(`getUUIDv7 got an invalid date:${date || '<blank>'}`);
     const dateBytes = intToByteArray(d.getTime()).reverse();
     dateBytes.slice(2, 8).forEach((b, i) => {
@@ -248,13 +214,12 @@ function getUUIDTimestamp(uuid) {
   const ts = parseInt(`${uuid}`.replace(/-/g, '').slice(0, 12), 16);
   return new Date(ts);
 }
-
 function getEntryTypeId(o, { defaults = {} } = {}) {
   let id = o.entry_type_id || defaults.entry_type_id;
   if (id) return id;
   const etype = o.entry_type || defaults.entry_type;
   if (!etype) {
-    throw new Error('No entry_type, nor entry_type_id specified, specify a defaultEntryType');
+    throw new Error('No entry_type, nor entry_type_id specified, specify one to generate a timeline suitable ID');
   }
   id = TIMELINE_ENTRY_TYPES[etype];
   if (id === undefined) throw new Error(`Invalid entry_type: ${etype}`);
@@ -263,32 +228,27 @@ function getEntryTypeId(o, { defaults = {} } = {}) {
 function getEntryType(o, defaults = {}) {
   let etype = o.entry_type || defaults.entry_type;
   if (etype) return etype;
-
   const id = o.entry_type_id || defaults.entry_type_id;
-
   etype = TIMELINE_ENTRY_TYPES[id];
   if (etype === undefined) throw new Error(`Invalid entry_type: ${etype}`);
   return etype;
 }
-
 const requiredTimelineEntryFields = ['ts', 'entry_type_id', 'input_id', 'person_id'];
-
 function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
   const o = { ...defaults, ...inputObject };
   /*
-      Outside systems CAN specify a unique UUID as remote_entry_uuid,
-      which will be used for updates, etc.
-      If not, it will be generated using whatever info we have
-    */
+        Outside systems CAN specify a unique UUID as remote_entry_uuid,
+        which will be used for updates, etc.
+        If not, it will be generated using whatever info we have
+      */
   if (o.remote_entry_uuid) {
     if (!uuidIsValid(o.remote_entry_uuid)) throw new Error('Invalid remote_entry_uuid, it must be a UUID');
     return o.remote_entry_uuid;
   }
   /*
-        Outside systems CAN specify a unique remote_entry_id
-        If not, it will be generated using whatever info we have
-      */
-
+          Outside systems CAN specify a unique remote_entry_id
+          If not, it will be generated using whatever info we have
+        */
   if (o.remote_entry_id) {
     // get a temp ID
     if (!o.input_id)
@@ -299,18 +259,14 @@ function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
     // may not match this standard, uuid sorting isn't guaranteed
     return getUUIDv7(o.ts, uuid);
   }
-  const entry_type_id = getEntryTypeId(o);
-
+  o.entry_type_id = getEntryTypeId(o);
   const missing = requiredTimelineEntryFields.filter((d) => o[d] === undefined); // 0 could be an entry type value
-
   if (missing.length > 0) throw new Error(`Missing required fields to append an entry_id:${missing.join(',')}`);
   const ts = new Date(o.ts);
   // isNaN behaves differently than Number.isNaN -- we're actually going for the
   // attempted conversion here
-
   if (isNaN(ts)) throw new Error(`getTimelineEntryUUID got an invalid date:${o.ts || '<blank>'}`);
-  const idString = `${ts.toISOString()}-${o.person_id}-${entry_type_id}-${o.source_code_id || 0}`;
-
+  const idString = `${ts.toISOString()}-${o.person_id}-${o.entry_type_id}-${o.source_code_id || 0}`;
   if (!uuidIsValid(o.input_id)) {
     throw new Error(`Invalid input_id:'${o.input_id}', type ${typeof o.input_id} -- should be a uuid`);
   }
@@ -321,13 +277,11 @@ function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
   // may not match this standard, uuid sorting isn't guaranteed
   return getUUIDv7(ts, uuid);
 }
-
 function getDateRangeArray(startDate, endDate) {
   const start = new Date(startDate);
   const end = new Date(endDate);
   const result = [];
   const msInDay = 24 * 60 * 60 * 1000;
-
   function addDays(date, days) {
     const d = new Date(date);
     d.setDate(d.getDate() + days);
@@ -343,13 +297,10 @@ function getDateRangeArray(startDate, endDate) {
     d.setFullYear(d.getFullYear() + years);
     return d;
   }
-
   const diffDays = Math.floor((end - start) / msInDay);
   const diffMonths = (end.getFullYear() - start.getFullYear()) * 12 + (end.getMonth() - start.getMonth());
   const diffYears = end.getFullYear() - start.getFullYear();
-
   let current = new Date(start);
-
   let stepFn;
   if (diffDays < 10) {
     stepFn = (date) => addDays(date, 1);
@@ -364,7 +315,6 @@ function getDateRangeArray(startDate, endDate) {
   } else {
     stepFn = (date) => addYears(date, 1);
   }
-
   while (current <= end) {
     result.push(new Date(current));
     const next = stepFn(current);
@@ -377,7 +327,6 @@ function getDateRangeArray(startDate, endDate) {
   }
   return result;
 }
-
 class ObjectError extends Error {
   constructor(data) {
     if (typeof data === 'string') {
@@ -394,8 +343,44 @@ class ObjectError extends Error {
     }
   }
 }
-
-module.exports = {
+export { appendPostfix };
+export { bool };
+export { create };
+export { list };
+export { downloadFile };
+export { extract };
+export { ForEachEntry };
+export { FileUtilities };
+export { getBatchTransform };
+export { getDateRangeArray };
+export { getDebatchTransform };
+export { getEntryType };
+export { getEntryTypeId };
+export { getFile };
+export { getManifest };
+export { getStringArray };
+export { getTempDir };
+export { getTempFilename };
+export { getTimelineEntryUUID };
+export { getPacketFiles };
+export { getPluginUUID };
+export { getInputUUID };
+export { getUUIDv7 };
+export { getUUIDTimestamp };
+export { handlebars };
+export { isValidDate };
+export { makeStrings };
+export { ObjectError };
+export { parseJSON5 };
+export { relativeDate };
+export { streamPacket };
+export { TIMELINE_ENTRY_TYPES };
+export { writeTempFile };
+export { uuidIsValid };
+export { uuidv4 };
+export { uuidv5 };
+export { uuidv7 };
+export default {
   appendPostfix,
   bool,
   create,

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.9.10",
+  "version": "2.0.0",
+  "type": "module",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {

package/test/cli.js

@@ -1,10 +1,9 @@
-const argv = require('yargs/yargs')(process.argv.slice(2)).parse();
-const methods = require('../index');
-
+import yargs from 'yargs/yargs';
+import methods from '../index.js';
+const argv = yargs(process.argv.slice(2)).parse();
 async function run() {
   if (typeof methods[argv._[0]] !== 'function') throw new Error(`${argv._[0]} is not a function`);
   const output = await methods[argv._[0]](argv);
-  // eslint-disable-next-line no-console
   console.log(output);
 }
 run();

package/test/file.js

@@ -1,9 +1,9 @@
-const { it } = require('node:test');
-const assert = require('node:assert');
-const debug = require('debug')('files');
-
-const { FileUtilities } = require('../index');
-
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import { FileUtilities } from '../index.js';
+const { it } = nodetest;
+const debug = debug$0('files');
 it('Should list a directory', async () => {
   const futil = new FileUtilities({ accountId: 'test' });
   let files = await futil.list({ directory: '.' });
@@ -14,7 +14,6 @@ it('Should list a directory', async () => {
   let endTest = await futil.list({ directory: '.', end: '1900-01-01' });
   assert(endTest.length === 0, 'Should NOT have any files before past end date');
 });
-
 it('Should be able to analyze CSV files with and without header lines', async () => {
   const futil = new FileUtilities({ accountId: 'test' });
   const f1 = await futil.columns({ filename: __dirname + '/sample/fileWithHead.csv' });

package/test/processing/bigDataMessage.js

@@ -1,17 +1,16 @@
-const { describe, it } = require('node:test');
-const assert = require('node:assert');
-const debug = require('debug')('test:big-data');
-const { setTimeout } = require('node:timers/promises');
-//const { v7: uuidv7 } = require('uuid');
-
-const { ForEachEntry } = require('../../index');
-
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import promises from 'node:timers/promises';
+import { ForEachEntry } from '../../index.js';
+const { describe, it } = nodetest;
+const debug = debug$0('test:big-data');
+const { setTimeout } = promises;
 describe('big-data message: forEachPerson', async () => {
   it('message: forEachPerson should loop through 1000000 sample people', async () => {
     const messageContent = [];
     let counter = 0;
     const forEach = new ForEachEntry();
-
     const output = await forEach.process({
       // packet: '../1000000_person_message.packet.zip',
       filename: '../1000000_person_message.packet/person/1000000_fake_people.csv',
@@ -47,7 +46,6 @@ describe('big-data message: forEachPerson', async () => {
       }
     });
     debug(output);
-
     assert.equal(counter, 1000000, `Expected to loop through 1000000 people, actual:${counter}`);
   });
   debug('Completed all tests');

package/test/processing/forEach.js

@@ -1,9 +1,9 @@
-const { describe, it } = require('node:test');
-const assert = require('node:assert');
-const debug = require('debug')('test/forEach');
-
-const { ForEachEntry } = require('../../index');
-
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import { ForEachEntry } from '../../index.js';
+const { describe, it } = nodetest;
+const debug = debug$0('test/forEach');
 describe('Test Person File For Each', async () => {
   it('forEachPerson Should loop through 1000 sample people', async () => {
     let counter = 0;
@@ -24,7 +24,6 @@ describe('Test Person File For Each', async () => {
       },
       async transform(props) {
         const { batch, timelineOutputFileStream, sampleOutputFileStream } = props;
-
         batch.forEach((p) => {
           if (Math.random() > 0.9) {
             sampleOutputFileStream.push({
@@ -41,7 +40,6 @@ describe('Test Person File For Each', async () => {
             entry_type: 'EMAIL_DELIVERED'
           });
         });
-
         batch.forEach(() => {
           counter += 1;
         });

package/test/processing/forEachResume.js

@@ -1,9 +1,9 @@
-const { describe, it } = require('node:test');
-const assert = require('node:assert');
-const debug = require('debug')('test/forEach');
-
-const { ForEachEntry } = require('../../index');
-
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import { ForEachEntry } from '../../index.js';
+const { describe, it } = nodetest;
+const debug = debug$0('test/forEach');
 describe('Test Person File For Each', async () => {
   it('forEachPerson Should loop through 1000 sample people', async () => {
     let counter = 0;
@@ -24,7 +24,6 @@ describe('Test Person File For Each', async () => {
       },
       async transform(props) {
         const { batch, timelineOutputFileStream, sampleOutputFileStream } = props;
-
         batch.forEach((p) => {
           if (Math.random() > 0.9) {
             sampleOutputFileStream.push({
@@ -41,7 +40,6 @@ describe('Test Person File For Each', async () => {
             entry_type: 'EMAIL_DELIVERED'
           });
         });
-
         batch.forEach(() => {
           counter += 1;
         });

package/test/processing/message.js

@@ -1,47 +1,39 @@
-const {
-  describe, it,
-} = require('node:test');
-const assert = require('node:assert');
-const debug = require('debug')('message');
-
-const { forEachPerson } = require('../../index');
-
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import index from '../../index.js';
+const { describe, it } = nodetest;
+const debug = debug$0('message');
+const { forEachPerson } = index;
 describe('Test Person Packet Message For Each', async () => {
   it('message: forEachPerson should loop through 1000 sample people', async () => {
     const messageContent = [];
     let counter = 0;
-    const results = await forEachPerson(
-      {
-        packet: 'test/sample/1000_message.packet.zip',
-        batchSize: 50,
-        bindings: {
-          timelineOutputStream: { type: 'output.timeline' },
-          message: { type: 'packet.message' },
-          handlebars: { type: 'handlebars' },
-        },
-        async transform({
-          batch,
-          message,
-          handlebars,
-          timelineOutputStream,
-        }) {
-          const template = handlebars.compile(message.content.text);
-          batch.forEach((person) => {
-            messageContent.push(template(person));
-          });
-          batch.forEach(() => { counter += 1; });
-          batch.forEach((p) => {
-            timelineOutputStream.push(
-              {
-                person_id: p.person_id,
-                email: p.email,
-                entry_type: 'EMAIL_DELIVERED',
-              },
-            );
-          });
-        },
+    const results = await forEachPerson({
+      packet: 'test/sample/1000_message.packet.zip',
+      batchSize: 50,
+      bindings: {
+        timelineOutputStream: { type: 'output.timeline' },
+        message: { type: 'packet.message' },
+        handlebars: { type: 'handlebars' }
       },
-    );
+      async transform({ batch, message, handlebars, timelineOutputStream }) {
+        const template = handlebars.compile(message.content.text);
+        batch.forEach((person) => {
+          messageContent.push(template(person));
+        });
+        batch.forEach(() => {
+          counter += 1;
+        });
+        batch.forEach((p) => {
+          timelineOutputStream.push({
+            person_id: p.person_id,
+            email: p.email,
+            entry_type: 'EMAIL_DELIVERED'
+          });
+        });
+      }
+    });
     debug(results);
     assert.equal(counter, 1000, `Expected to loop through 1000 people, actual:${counter}`);
   });

package/test/processing/zip.js

@@ -1,15 +1,14 @@
-const { describe, it } = require('node:test');
-const assert = require('node:assert');
-const debug = require('debug');
-
-const { create, getManifest } = require('../../index');
-
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug from 'debug';
+import { create, getManifest } from '../../index.js';
+const { describe, it } = nodetest;
 describe('Test Person Packet Creator', async () => {
   const pfile = './test/sample/message/5_fake_people.csv';
   it(`should create a zip file from directory ${process.cwd()} with path ${pfile}`, async () => {
     const out = await create({
       personFiles: [pfile],
-      messageFiles: 'test/sample/message/message.json5',
+      messageFiles: 'test/sample/message/message.json5'
     });
     debug('Successfully created:', out);
     return out;

package/test/uuid.js

@@ -1,25 +1,20 @@
-const {
-  it,
-} = require('node:test');
-const assert = require('node:assert');
-const debug = require('debug')('message');
-
-const { getUUIDv7, getUUIDTimestamp } = require('../index');
-
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import { getUUIDv7, getUUIDTimestamp } from '../index.js';
+const { it } = nodetest;
+const debug = debug$0('message');
 it('Should correctly calculate UUIDS given various inputs', async () => {
   const now = new Date();
-
   const uuid = getUUIDv7();
   const ts = getUUIDTimestamp(uuid);
   debug(uuid, ts, now);
   const diff = now.getTime() - ts.getTime();
   assert(diff < 1000 && diff >= 0, 'More than a second between newly created');
-
   const uuid2 = getUUIDv7(now);
   const ts2 = getUUIDTimestamp(uuid2);
   const diff2 = now.getTime() - ts2.getTime();
   assert(diff2 === 0, 'Timestamps should match');
-
   const uuid3 = getUUIDv7(now);
   assert(uuid2 !== uuid3, 'UUIDs should be unique match');
 });