@engine9/input-tools 2.0.7

package/test/file.js

@@ -0,0 +1,23 @@
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import { FileUtilities } from '../index.js';
+const { it } = nodetest;
+const debug = debug$0('files');
+it('Should list a directory', async () => {
+  const futil = new FileUtilities({ accountId: 'test' });
+  let files = await futil.list({ directory: '.' });
+  assert(files.length, 'Should have some files');
+  debug(files);
+  let startTest = await futil.list({ directory: '.', start: '2040-01-01' });
+  assert(startTest.length === 0, 'Should NOT have any files before future start date');
+  let endTest = await futil.list({ directory: '.', end: '1900-01-01' });
+  assert(endTest.length === 0, 'Should NOT have any files before past end date');
+});
+it('Should be able to analyze CSV files with and without header lines', async () => {
+  const futil = new FileUtilities({ accountId: 'test' });
+  const f1 = await futil.columns({ filename: import.meta.dirname + '/sample/fileWithHead.csv' });
+  assert.equal(f1.likelyHeaderLines, 1, 'Number of header lines should be 1');
+  const f2 = await futil.columns({ filename: import.meta.dirname + '/sample/fileWithoutHead.csv' });
+  assert.equal(f2.likelyHeaderLines, 0, 'Number of header lines should be 1');
+});

package/test/processing/bigDataMessage.js

@@ -0,0 +1,52 @@
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import promises from 'node:timers/promises';
+import { ForEachEntry } from '../../index.js';
+const { describe, it } = nodetest;
+const debug = debug$0('test:big-data');
+const { setTimeout } = promises;
+describe('big-data message: forEachPerson', async () => {
+  it('message: forEachPerson should loop through 1000000 sample people', async () => {
+    const messageContent = [];
+    let counter = 0;
+    const forEach = new ForEachEntry();
+    const output = await forEach.process({
+      // packet: '../1000000_person_message.packet.zip',
+      filename: '../1000000_person_message.packet/person/1000000_fake_people.csv',
+      batchSize: 10000,
+      concurrency: 1000,
+      progress: debug,
+      bindings: {
+        timelineOutputStream: { path: 'output.timeline' },
+        message: { path: 'file', filename: '../1000000_person_message.packet/message/message.json5' },
+        handlebars: { path: 'handlebars' }
+      },
+      async transform({ batch, message, handlebars, timelineOutputStream }) {
+        //const id = uuidv7();
+        //debug(`Processing batch of ${batch.length} - ${id}`);
+        if (!message?.content?.text) throw new Error(`Sample message has no content.text:${JSON.stringify(message)}`);
+        const template = handlebars.compile(message.content.text);
+        batch.forEach((person) => {
+          messageContent.push(template(person));
+        });
+        batch.forEach((p) => {
+          const o = {
+            person_id: p.person_id,
+            email: p.email,
+            entry_type: 'EMAIL_DELIVERED'
+          };
+          counter += 1;
+          //if (counter % 10000 === 0) debug(`*** Processed ${counter} items, last person_id=${p.person_id}`, o);
+          timelineOutputStream.push(o);
+        });
+        // debug(`Processed batch of size ${batch.length}`);
+        await setTimeout(Math.random() * 3000);
+        //debug(`Completed processing ${id}`);
+      }
+    });
+    debug(output);
+    assert.equal(counter, 1000000, `Expected to loop through 1000000 people, actual:${counter}`);
+  });
+  debug('Completed all tests');
+});

package/test/processing/forEach.js

@@ -0,0 +1,53 @@
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import { ForEachEntry } from '../../index.js';
+const { describe, it } = nodetest;
+const debug = debug$0('test/forEach');
+describe('Test Person File For Each', async () => {
+  it('forEachPerson Should loop through 1000 sample people', async () => {
+    let counter = 0;
+    const forEach = new ForEachEntry();
+    const result = await forEach.process({
+      packet: 'test/sample/1000_message.packet.zip',
+      batchSize: 50,
+      bindings: {
+        timelineOutputFileStream: {
+          path: 'output.timeline',
+          options: {
+            entry_type: 'ENTRY_OPTION'
+          }
+        },
+        sampleOutputFileStream: {
+          path: 'output.stream'
+        }
+      },
+      async transform(props) {
+        const { batch, timelineOutputFileStream, sampleOutputFileStream } = props;
+        batch.forEach((p) => {
+          if (Math.random() > 0.9) {
+            sampleOutputFileStream.push({
+              // for testing we don't need real person_ids
+              person_id: p.person_id || Math.floor(Math.random() * 1000000),
+              email: p.email,
+              entry_type: 'SAMPLE_OUTPUT'
+            });
+          }
+          timelineOutputFileStream.push({
+            // for testing we don't need real person_ids
+            person_id: p.person_id || Math.floor(Math.random() * 1000000),
+            email: p.email,
+            entry_type: 'EMAIL_DELIVERED'
+          });
+        });
+        batch.forEach(() => {
+          counter += 1;
+        });
+      }
+    });
+    assert(result.outputFiles?.timelineOutputFileStream?.[0]?.records);
+    assert(result.outputFiles?.sampleOutputFileStream?.[0]?.records);
+    assert.equal(counter, 1000, `Expected to loop through 1000 people, actual:${counter}`);
+  });
+  debug('Completed tests');
+});

package/test/processing/forEachResume.js

@@ -0,0 +1,54 @@
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import { ForEachEntry } from '../../index.js';
+const { describe, it } = nodetest;
+const debug = debug$0('test/forEach');
+describe('Test Person File For Each', async () => {
+  it('forEachPerson Should loop through 1000 sample people', async () => {
+    let counter = 0;
+    const forEach = new ForEachEntry();
+    const result = await forEach.process({
+      packet: 'test/sample/1000_message.packet.zip',
+      batchSize: 50,
+      bindings: {
+        timelineOutputFileStream: {
+          path: 'output.timeline',
+          options: {
+            entry_type: 'ENTRY_OPTION'
+          }
+        },
+        sampleOutputFileStream: {
+          path: 'output.stream'
+        }
+      },
+      async transform(props) {
+        const { batch, timelineOutputFileStream, sampleOutputFileStream } = props;
+        batch.forEach((p) => {
+          if (Math.random() > 0.9) {
+            sampleOutputFileStream.push({
+              // for testing we don't need real person_ids
+              person_id: p.person_id || Math.floor(Math.random() * 1000000),
+              email: p.email,
+              entry_type: 'SAMPLE_OUTPUT'
+            });
+          }
+          timelineOutputFileStream.push({
+            // for testing we don't need real person_ids
+            person_id: p.person_id || Math.floor(Math.random() * 1000000),
+            email: p.email,
+            entry_type: 'EMAIL_DELIVERED'
+          });
+        });
+        batch.forEach(() => {
+          counter += 1;
+        });
+      }
+    });
+    assert(result.outputFiles?.timelineOutputFileStream?.[0]?.records);
+    assert(result.outputFiles?.sampleOutputFileStream?.[0]?.records);
+    debug(result);
+    assert.equal(counter, 1000, `Expected to loop through 1000 people, actual:${counter}`);
+  });
+  debug('Completed tests');
+});

package/test/processing/message.js

@@ -0,0 +1,40 @@
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug$0 from 'debug';
+import index from '../../index.js';
+const { describe, it } = nodetest;
+const debug = debug$0('message');
+const { forEachPerson } = index;
+describe('Test Person Packet Message For Each', async () => {
+  it('message: forEachPerson should loop through 1000 sample people', async () => {
+    const messageContent = [];
+    let counter = 0;
+    const results = await forEachPerson({
+      packet: 'test/sample/1000_message.packet.zip',
+      batchSize: 50,
+      bindings: {
+        timelineOutputStream: { type: 'output.timeline' },
+        message: { type: 'packet.message' },
+        handlebars: { type: 'handlebars' }
+      },
+      async transform({ batch, message, handlebars, timelineOutputStream }) {
+        const template = handlebars.compile(message.content.text);
+        batch.forEach((person) => {
+          messageContent.push(template(person));
+        });
+        batch.forEach(() => {
+          counter += 1;
+        });
+        batch.forEach((p) => {
+          timelineOutputStream.push({
+            person_id: p.person_id,
+            email: p.email,
+            entry_type: 'EMAIL_DELIVERED'
+          });
+        });
+      }
+    });
+    debug(results);
+    assert.equal(counter, 1000, `Expected to loop through 1000 people, actual:${counter}`);
+  });
+});

package/test/processing/zip.js

@@ -0,0 +1,21 @@
+import nodetest from 'node:test';
+import assert from 'node:assert';
+import * as debug from 'debug';
+import { create, getManifest } from '../../index.js';
+const { describe, it } = nodetest;
+describe('Test Person Packet Creator', async () => {
+  const pfile = './test/sample/message/5_fake_people.csv';
+  it(`should create a zip file from directory ${process.cwd()} with path ${pfile}`, async () => {
+    const out = await create({
+      personFiles: [pfile],
+      messageFiles: 'test/sample/message/message.json5'
+    });
+    debug('Successfully created:', out);
+    return out;
+  });
+  it('should retrieve a manifest', async () => {
+    const manifest = await getManifest({ packet: './test/sample/5_message.packet.zip' });
+    assert.equal(typeof manifest, 'object', 'Manifest is not an object');
+    assert.equal(manifest.accountId, 'engine9', 'Manifest does not have an accountId=engine9');
+  });
+});

package/test/sample/fileWithHead.csv

@@ -0,0 +1,3 @@
+first,last,email,phone
+Bob,Smith,test@test.com,(703) 555-5555
+Jane,Doe,test2@test.com,(703) 555-5432

package/test/sample/fileWithoutHead.csv

@@ -0,0 +1,2 @@
+Bob,Smith,test@test.com,(703) 555-5555
+Jane,Doe,test2@test.com,(703) 555-5432