@engine9-io/input-tools 1.3.6

package/index.js

@@ -0,0 +1,327 @@
+const fs = require('node:fs');
+
+const path = require('node:path');
+
+const debug = require('debug')('@engine9/input-tools');
+
+const unzipper = require('unzipper');
+const {
+  v4: uuidv4, v5: uuidv5, v7: uuidv7, validate: uuidIsValid,
+} = require('uuid');
+const archiver = require('archiver');
+const FileUtilities = require('./file/FileUtilities');
+
+const {
+  bool,
+  getManifest,
+  getFile,
+  downloadFile,
+  getTempFilename,
+  streamPacket,
+  getPacketFiles,
+  getBatchTransform,
+  getDebatchTransform,
+} = require('./file/tools');
+
+const ForEachEntry = require('./ForEachEntry');
+
+const { TIMELINE_ENTRY_TYPES } = require('./timelineTypes');
+
+function getStringArray(s, nonZeroLength) {
+  let a = s || [];
+  if (typeof a === 'number') a = String(a);
+  if (typeof a === 'string') a = [a];
+
+  if (typeof s === 'string') a = s.split(',');
+  a = a.map((x) => x.toString().trim()).filter(Boolean);
+  if (nonZeroLength && a.length === 0) a = [0];
+  return a;
+}
+
+/*
+  When comparing two objects, some may come from a file (thus strings), and some from
+  a database or elsewhere (not strings), so for deduping make sure to make them all strings
+*/
+function makeStrings(o) {
+  return Object.entries(o).reduce((a, [k, v]) => {
+    a[k] = (typeof v === 'object') ? JSON.stringify(v) : String(v);
+    return a;
+  }, {});
+}
+
+async function list(_path) {
+  const directory = await unzipper.Open.file(_path);
+
+  return new Promise((resolve, reject) => {
+    directory.files[0]
+      .stream()
+      .pipe(fs.createWriteStream('firstFile'))
+      .on('error', reject)
+      .on('finish', resolve);
+  });
+}
+
+async function extract(_path, _file) {
+  const directory = await unzipper.Open(_path);
+  // return directory.files.map((f) => f.path);
+  const file = directory.files.find((d) => d.path === _file);
+  const tempFilename = await getTempFilename({ source: _file });
+  return new Promise((resolve, reject) => {
+    file
+      .stream()
+      .pipe(fs.createWriteStream(tempFilename))
+      .on('error', reject)
+      .on('finish', resolve);
+  });
+}
+
+function appendFiles(existingFiles, _newFiles, options) {
+  const newFiles = getStringArray(_newFiles);
+  if (newFiles.length === 0) return;
+  let { type, dateCreated } = options || {};
+  if (!type) type = 'unknown';
+  if (!dateCreated)dateCreated = new Date().toISOString();
+  let arr = newFiles;
+  if (!Array.isArray(newFiles)) arr = [arr];
+
+  arr.forEach((p) => {
+    const item = {
+      type,
+      originalFilename: '',
+      isNew: true,
+      dateCreated,
+    };
+
+    if (typeof p === 'string') {
+      item.originalFilename = path.resolve(process.cwd(), p);
+    } else {
+      item.originalFilename = path.resolve(process.cwd(), item.originalFilename);
+    }
+
+    const file = item.originalFilename.split(path.sep).pop();
+    item.path = `${type}/${file}`;
+    const existingFile = existingFiles.find((f) => f.path === item.path);
+    if (existingFile) throw new Error('Error adding files, duplicate path found for path:', +item.path);
+    existingFiles.push(item);
+  });
+}
+
+async function create(options) {
+  const {
+    accountId = 'engine9',
+    pluginId = '',
+    target = '', // target filename, creates one if not specified
+    messageFiles = [], // file with contents of message, used for delivery
+    personFiles = [], // files with data on people
+    timelineFiles = [], // activity entry
+    statisticsFiles = [], // files with aggregate statistics
+  } = options;
+  if (options.peopleFiles) throw new Error('Unknown option: peopleFiles, did you mean personFiles?');
+
+  const files = [];
+  const dateCreated = new Date().toISOString();
+  appendFiles(files, messageFiles, { type: 'message', dateCreated });
+  appendFiles(files, personFiles, { type: 'person', dateCreated });
+  appendFiles(files, timelineFiles, { type: 'timeline', dateCreated });
+  appendFiles(files, statisticsFiles, { type: 'statistics', dateCreated });
+
+  const zipFilename = target || await getTempFilename({ postfix: '.packet.zip' });
+
+  const manifest = {
+    accountId,
+    source: {
+      pluginId,
+    },
+    dateCreated,
+    files,
+  };
+
+  // create a file to stream archive data to.
+  const output = fs.createWriteStream(zipFilename);
+  const archive = archiver('zip', {
+    zlib: { level: 9 }, // Sets the compression level.
+  });
+  return new Promise((resolve, reject) => {
+    debug(`Setting up write stream to ${zipFilename}`);
+    // listen for all archive data to be written
+    // 'close' event is fired only when a file descriptor is involved
+    output.on('close', () => {
+      debug('archiver has been finalized and the output file descriptor has closed, calling success');
+      debug(zipFilename);
+      return resolve({
+        filename: zipFilename,
+        bytes: archive.pointer(),
+      });
+    });
+
+    // This event is fired when the data source is drained no matter what was the data source.
+    // It is not part of this library but rather from the NodeJS Stream API.
+    // @see: https://nodejs.org/api/stream.html#stream_event_end
+    output.on('end', () => {
+      // debug('end event -- Data has been drained');
+    });
+
+    // warnings could be file not founds, etc, but we error even on those
+    archive.on('warning', (err) => {
+      reject(err);
+    });
+
+    // good practice to catch this error explicitly
+    archive.on('error', (err) => {
+      reject(err);
+    });
+
+    archive.pipe(output);
+
+    files.forEach(({ path: name, originalFilename }) => archive.file(originalFilename, { name }));
+    files.forEach((f) => {
+      delete f.originalFilename;
+      delete f.isNew;
+    });
+
+    archive.append(Buffer.from(JSON.stringify(manifest, null, 4), 'utf8'), { name: 'manifest.json' });
+    archive.finalize();
+  });
+}
+
+function intToByteArray(_v) {
+  // we want to represent the input as a 8-bytes array
+  const byteArray = [0, 0, 0, 0, 0, 0, 0, 0];
+  let v = _v;
+  for (let index = 0; index < byteArray.length; index += 1) {
+    // eslint-disable-next-line no-bitwise
+    const byte = v & 0xff;
+    byteArray[index] = byte;
+    v = (v - byte) / 256;
+  }
+
+  return byteArray;
+}
+function getPluginUUID(uniqueNamespaceLikeDomainName, valueWithinNamespace) {
+  // Random custom namespace for plugins -- not secure, just a namespace:
+  return uuidv5(`${uniqueNamespaceLikeDomainName}::${valueWithinNamespace}`, 'f9e1024d-21ac-473c-bac6-64796dd771dd');
+}
+
+function getInputUUID(a, b) {
+  let pluginId = a;
+  let remoteInputId = b;
+  if (typeof a === 'object') {
+    pluginId = a.pluginId;
+    remoteInputId = a.remoteInputId;
+  }
+
+  if (!pluginId) throw new Error('getInputUUID: Cowardly rejecting a blank plugin_id');
+  if (!uuidIsValid(pluginId)) throw new Error(`Invalid pluginId:${pluginId}, should be a UUID`);
+  const rid = (remoteInputId || '').trim();
+  if (!rid) throw new Error('getInputUUID: Cowardly rejecting a blank remote_input_id, set a default');
+  // Random custom namespace for inputs -- not secure, just a namespace:
+  // 3d0e5d99-6ba9-4fab-9bb2-c32304d3df8e
+  return uuidv5(`${pluginId}:${rid}`, '3d0e5d99-6ba9-4fab-9bb2-c32304d3df8e');
+}
+
+function getUUIDv7(date, inputUuid) { /* optional date and input UUID */
+  const uuid = inputUuid || uuidv7();
+  const bytes = Buffer.from(uuid.replace(/-/g, ''), 'hex');
+  if (date !== undefined) {
+    const d = new Date(date);
+    // isNaN behaves differently than Number.isNaN -- we're actually going for the
+    // attempted conversion here
+    // eslint-disable-next-line no-restricted-globals
+    if (isNaN(d)) throw new Error(`getUUIDv7 got an invalid date:${date || '<blank>'}`);
+    const dateBytes = intToByteArray(d.getTime()).reverse();
+    dateBytes.slice(2, 8).forEach((b, i) => { bytes[i] = b; });
+  }
+  return uuidv4({ random: bytes });
+}
+/* Returns a date from a given uuid (assumed to be a v7, otherwise the results are ... weird */
+function getUUIDTimestamp(uuid) {
+  const ts = parseInt((`${uuid}`).replace(/-/g, '').slice(0, 12), 16);
+  return new Date(ts);
+}
+
+const requiredTimelineEntryFields = ['ts', 'entry_type_id', 'input_id', 'person_id'];
+
+function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
+  const o = { ...defaults, ...inputObject };
+  /*
+      Outside systems CAN specify a unique UUID as remote_entry_uuid,
+      which will be used for updates, etc.
+      If not, it will be generated using whatever info we have
+    */
+  if (o.remote_entry_uuid) {
+    if (!uuidIsValid(o.remote_entry_uuid)) throw new Error('Invalid remote_entry_uuid, it must be a UUID');
+    return o.remote_entry_uuid;
+  }
+  /*
+        Outside systems CAN specify a unique remote_entry_id
+        If not, it will be generated using whatever info we have
+      */
+
+  if (o.remote_entry_id) {
+    // get a temp ID
+    if (!o.input_id) throw new Error('Error generating timeline entry uuid -- remote_entry_id specified, but no input_id');
+    const uuid = uuidv5(o.remote_entry_id, o.input_id);
+    // Change out the ts to match the v7 sorting.
+    // But because outside specified remote_entry_uuid
+    // may not match this standard, uuid sorting isn't guaranteed
+    return getUUIDv7(o.ts, uuid);
+  }
+
+  const missing = requiredTimelineEntryFields
+    .filter((d) => o[d] === undefined);// 0 could be an entry type value
+
+  if (missing.length > 0) throw new Error(`Missing required fields to append an entry_id:${missing.join(',')}`);
+  const ts = new Date(o.ts);
+  // isNaN behaves differently than Number.isNaN -- we're actually going for the
+  // attempted conversion here
+  // eslint-disable-next-line no-restricted-globals
+  if (isNaN(ts)) throw new Error(`getTimelineEntryUUID got an invalid date:${o.ts || '<blank>'}`);
+  const idString = `${ts.toISOString()}-${o.person_id}-${o.entry_type_id}-${o.source_code_id || 0}`;
+  // get a temp ID
+  const uuid = uuidv5(idString, o.input_id);
+  // Change out the ts to match the v7 sorting.
+  // But because outside specified remote_entry_uuid
+  // may not match this standard, uuid sorting isn't guaranteed
+  return getUUIDv7(ts, uuid);
+}
+function getEntryTypeId(o, { defaults = {} } = {}) {
+  let id = o.entry_type_id || defaults.entry_type_id;
+  if (id) return id;
+  const etype = o.entry_type || defaults.entry_type;
+  if (!etype) {
+    throw new Error('No entry_type, nor entry_type_id specified, specify a defaultEntryType');
+  }
+  id = TIMELINE_ENTRY_TYPES[etype];
+  if (id === undefined) throw new Error(`Invalid entry_type: ${etype}`);
+  return id;
+}
+
+module.exports = {
+  bool,
+  create,
+  list,
+  extract,
+  streamPacket,
+  getBatchTransform,
+  getDebatchTransform,
+  getManifest,
+  getFile,
+  downloadFile,
+  getStringArray,
+  getTempFilename,
+  getTimelineEntryUUID,
+  getPacketFiles,
+  getPluginUUID,
+  getInputUUID,
+  getUUIDv7,
+  getUUIDTimestamp,
+  uuidIsValid,
+  uuidv4,
+  uuidv5,
+  uuidv7,
+  makeStrings,
+  ForEachEntry,
+  FileUtilities,
+  TIMELINE_ENTRY_TYPES,
+  getEntryTypeId,
+};

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@engine9-io/input-tools",
+  "version": "1.3.6",
+  "description": "Tools for dealing with Engine9 inputs",
+  "main": "index.js",
+  "scripts": {
+    "test-big-data": "export DEBUG=*;node --test --test-name-pattern=big-data",
+    "test": "export DEBUG=*;node --test --test-skip-pattern=big-data"
+  },
+  "author": "Engine9",
+  "license": "GPL-3.0-or-later",
+  "devDependencies": {
+    "eslint": "^8.57.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-jsonc": "^2.15.1"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.723.0",
+    "@dsnp/parquetjs": "^1.8.6",
+    "archiver": "^7.0.1",
+    "async-mutex": "^0.5.0",
+    "csv": "^6.3.11",
+    "debug": "^4.3.4",
+    "detect-file-encoding-and-language": "^2.4.0",
+    "googleapis": "^148.0.0",
+    "handlebars": "^4.7.8",
+    "json5": "^2.2.3",
+    "mime-type": "^5.0.3",
+    "mkdirp": "^3.0.1",
+    "parallel-transform": "^1.2.0",
+    "throttle-debounce": "^5.0.2",
+    "unzipper": "^0.12.1",
+    "uuid": "^11.1.0",
+    "yargs": "^17.7.2"
+  },
+  "directories": {
+    "test": "test"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/engine9-io/input-tools.git"
+  },
+  "keywords": [
+    "Engine9",
+    "CRM",
+    "CDP"
+  ],
+  "bugs": {
+    "url": "https://github.com/engine9-io/input-tools/issues"
+  },
+  "homepage": "https://github.com/engine9-io/input-tools#readme"
+}

package/test/cli.js

@@ -0,0 +1,10 @@
+const argv = require('yargs/yargs')(process.argv.slice(2)).parse();
+const methods = require('../index');
+
+async function run() {
+  if (typeof methods[argv._[0]] !== 'function') throw new Error(`${argv._[0]} is not a function`);
+  const output = await methods[argv._[0]](argv);
+  // eslint-disable-next-line no-console
+  console.log(output);
+}
+run();

package/test/packet/bigDataMessage.js

@@ -0,0 +1,62 @@
+const {
+  describe, it,
+} = require('node:test');
+const assert = require('node:assert');
+const debug = require('debug')('test:big-data');
+const { setTimeout } = require('node:timers/promises');
+const { v7: uuidv7 } = require('uuid');
+
+const { ForEachEntry } = require('../../index');
+
+describe('big-data message: forEachPerson', async () => {
+  it('message: forEachPerson should loop through 1000000 sample people', async () => {
+    const messageContent = [];
+    let counter = 0;
+    const forEach = new ForEachEntry();
+
+    const output = await forEach.process(
+      {
+        // packet: '../1000000_person_message.packet.zip',
+        filename: '../1000000_fake_people.csv',
+        batchSize: 10000,
+        concurrency: 1000,
+        bindings: {
+          timelineOutputStream: { path: 'output.timeline' },
+          message: { path: 'file', filename: '../1000000_person_message.packet/message/message.json5' },
+          handlebars: { path: 'handlebars' },
+        },
+        async transform({
+          batch,
+          message,
+          handlebars,
+          timelineOutputStream,
+        }) {
+          const id = uuidv7();
+          debug(`Processing batch of ${batch.length} - ${id}`);
+          if (!message?.content?.text) throw new Error(`Sample message has no content.text:${JSON.stringify(message)}`);
+          const template = handlebars.compile(message.content.text);
+          batch.forEach((person) => {
+            messageContent.push(template(person));
+          });
+          batch.forEach((p) => {
+            const o = {
+              person_id: p.person_id,
+              email: p.email,
+              entry_type: 'EMAIL_DELIVERED',
+            };
+            counter += 1;
+            if (counter % 10000 === 0) debug(`*** Processed ${counter} items, last person_id=${p.person_id}`, o);
+            timelineOutputStream.push(o);
+          });
+          // debug(`Processed batch of size ${batch.length}`);
+          await setTimeout(Math.random() * 3000);
+          debug(`Completed processing ${id}`);
+        },
+      },
+    );
+    debug(output);
+
+    assert.equal(counter, 1000000, `Expected to loop through 1000000 people, actual:${counter}`);
+  });
+  debug('Completed all tests');
+});

package/test/packet/forEach.js

@@ -0,0 +1,51 @@
+const {
+  describe, it,
+} = require('node:test');
+const assert = require('node:assert');
+const debug = require('debug')('test/forEach');
+
+const { ForEachEntry } = require('../../index');
+
+describe('Test Person Packet For Each', async () => {
+  it('forEachPerson Should loop through 1000 sample people', async () => {
+    let counter = 0;
+    const forEach = new ForEachEntry();
+    await forEach.process(
+      {
+        packet: 'test/sample/1000_message.packet.zip',
+        batchSize: 50,
+        bindings: {
+          timelineOutputStream: {
+            path: 'output.timeline',
+            options: {
+              entry_type: 'SAMPLE',
+            },
+          },
+        },
+        async transform(props) {
+          const {
+            batch,
+            timelineOutputStream,
+          } = props;
+          if (timelineOutputStream) {
+            batch.forEach((p) => {
+              timelineOutputStream.push(
+                {
+                  // for testing we don't need real person_ids
+                  person_id: p.person_id || Math.floor(Math.random() * 1000000),
+                  email: p.email,
+                  entry_type: 'EMAIL_DELIVERED',
+                },
+              );
+            });
+          } else {
+            throw new Error(`output.timeline did not put a timelineOutputStream into the bindings:${Object.keys(props)}`);
+          }
+          batch.forEach(() => { counter += 1; });
+        },
+      },
+    );
+    assert.equal(counter, 1000, `Expected to loop through 1000 people, actual:${counter}`);
+  });
+  debug('Completed tests');
+});

package/test/packet/message.js

@@ -0,0 +1,48 @@
+const {
+  describe, it,
+} = require('node:test');
+const assert = require('node:assert');
+const debug = require('debug')('message');
+
+const { forEachPerson } = require('../../index');
+
+describe('Test Person Packet Message For Each', async () => {
+  it('message: forEachPerson should loop through 1000 sample people', async () => {
+    const messageContent = [];
+    let counter = 0;
+    const results = await forEachPerson(
+      {
+        packet: 'test/sample/1000_message.packet.zip',
+        batchSize: 50,
+        bindings: {
+          timelineOutputStream: { type: 'packet.output.timeline' },
+          message: { type: 'packet.message' },
+          handlebars: { type: 'handlebars' },
+        },
+        async transform({
+          batch,
+          message,
+          handlebars,
+          timelineOutputStream,
+        }) {
+          const template = handlebars.compile(message.content.text);
+          batch.forEach((person) => {
+            messageContent.push(template(person));
+          });
+          batch.forEach(() => { counter += 1; });
+          batch.forEach((p) => {
+            timelineOutputStream.push(
+              {
+                person_id: p.person_id,
+                email: p.email,
+                entry_type: 'EMAIL_DELIVERED',
+              },
+            );
+          });
+        },
+      },
+    );
+    debug(results);
+    assert.equal(counter, 1000, `Expected to loop through 1000 people, actual:${counter}`);
+  });
+});

package/test/packet/timelineStream.js

@@ -0,0 +1,26 @@
+const {
+  describe, it,
+} = require('node:test');
+const fs = require('node:fs');
+const assert = require('node:assert');
+
+const { getTimelineOutputStream } = require('../../index');
+
+describe('TimelineOutputStream', async () => {
+  it('timeline: It should save items to a csv file', async () => {
+    const {
+      stream: timelineStream, promises,
+      files,
+    } = await getTimelineOutputStream({});
+    timelineStream.push({ foo: 'bar' });
+    // finish the input stream
+    timelineStream.push(null);
+    await promises[0];
+    const content = fs.readFileSync(files[0]).toString().split('\n').map((d) => d.trim())
+      .filter(Boolean);
+
+    const s = 'uuid,entry_type,person_id,reference_id';
+    assert.equal(content[0].slice(0, s.length), s, "Beginning of first line doesn't match expected timeline csv header");
+    assert.equal(content.length, 2, `There are ${content.length}, not 2 lines in the CSV file`);
+  });
+});

package/test/packet/zip.js

@@ -0,0 +1,22 @@
+const { describe, it } = require('node:test');
+const assert = require('node:assert');
+const debug = require('debug');
+
+const { create, getManifest } = require('../../index');
+
+describe('Test Person Packet Creator', async () => {
+  const pfile = './test/sample/message/5_fake_people.csv';
+  it(`should create a zip file from directory ${process.cwd()} with path ${pfile}`, async () => {
+    const out = await create({
+      personFiles: [pfile],
+      messageFiles: 'test/sample/message/message.json5',
+    });
+    debug('Successfully created:', out);
+    return out;
+  });
+  it('should retrieve a manifest', async () => {
+    const manifest = await getManifest({ packet: './test/sample/5_message.packet.zip' });
+    assert.equal(typeof manifest, 'object', 'Manifest is not an object');
+    assert.equal(manifest.accountId, 'engine9', 'Manifest does not have an accountId=engine9');
+  });
+});

package/test/parallelStream.js

@@ -0,0 +1,48 @@
+/* eslint-disable no-console */
+const {
+  setTimeout,
+} = require('node:timers/promises');
+
+const { describe } = require('node:test');
+// const assert = require('node:assert');
+const { Readable } = require('node:stream');
+const { createWriteStream } = require('node:fs');
+const { pipeline } = require('node:stream/promises');
+const ParallelStream = require('../ParallelStream');
+const { getTempFilename } = require('../index');
+
+describe('Should process items in parallel:', async () => {
+  const outputFile = await getTempFilename({});
+  const writeStream = createWriteStream(outputFile);
+
+  const CONCURRENCY = 500;
+  await pipeline(
+    Readable.from(
+      [...Array(1000)].map((v, i) => ({ i })),
+    ),
+
+    new ParallelStream(
+      CONCURRENCY,
+      async (obj, enc, push, done) => {
+        let res;
+
+        try {
+          await setTimeout(Math.random() * 1000);
+          if (Math.random() > 0.7) throw new Error('Random error');
+
+          res = `${obj.id} is complete\n`;
+        } catch (err) {
+          await setTimeout(Math.random() * 2000);// longer timeouts for errors
+          res = `${obj.id} is error, ${err.name}\n`;
+        }
+
+        done(null, obj.id); // _onComplete actually
+
+        return res;
+      },
+    ),
+    writeStream,
+  );
+
+  console.log('Wrote responses to ', outputFile);
+});