@engine9-io/input-tools 1.6.0 → 1.6.3

package/file/FileUtilities.js

@@ -824,6 +824,8 @@ Worker.prototype.getUniqueSet = async function (options) {
 };
 
 Worker.prototype.getUniqueStream = async function (options) {
+  const includeDuplicateSourceRecords = bool(options.includeDuplicateSourceRecords, false);
+
   const { uniqueSet, uniqueFunction, sample } = await this.getUniqueSet({
     filenames: options.existingFiles,
     uniqueFunction: options.uniqueFunction,
@@ -844,7 +846,10 @@ Worker.prototype.getUniqueStream = async function (options) {
           // do nothing
           cb();
         } else {
-          uniqueSet.add(v);
+          if (!includeDuplicateSourceRecords) {
+            // add it to the set for the next time
+            uniqueSet.add(v);
+          }
           cb(null, d);
         }
       },
@@ -857,8 +862,11 @@ Worker.prototype.getUniqueStream.metadata = {
   options: {
     existingFiles: {},
     uniqueFunction: {},
-    filename: {},
-    stream: {},
+    filename: { description: 'Specify a source filename or a stream' },
+    stream: { description: 'Specify a source filename or a stream' },
+    includeDuplicateSourceRecords: {
+      description: 'Sometimes you want the output to include source dupes, sometimes not, default false',
+    },
   },
 };
 Worker.prototype.getUniqueFile = async function (options) {
@@ -871,8 +879,11 @@ Worker.prototype.getUniqueFile.metadata = {
   options: {
     existingFiles: {},
     uniqueFunction: {},
-    filename: {},
-    stream: {},
+    filename: { description: 'Specify a source filename or a stream' },
+    stream: { description: 'Specify a source filename or a stream' },
+    includeDuplicateSourceRecords: {
+      description: 'Sometimes you want the output to include source dupes, sometimes not, default false',
+    },
   },
 };
 
@@ -882,7 +893,7 @@ Requires 2 passes of the files,
 but that's a better tradeoff than trying to store huge files in memory
 */
 Worker.prototype.diff = async function ({
-  fileA, fileB, uniqueFunction: ufOpt, fields,
+  fileA, fileB, uniqueFunction: ufOpt, fields, includeDuplicateSourceRecords,
 }) {
   if (ufOpt && fields) throw new Error('fields and uniqueFunction cannot both be specified');
   let uniqueFunction = ufOpt;
@@ -891,17 +902,19 @@ Worker.prototype.diff = async function ({
     uniqueFunction = (o) => farr.map((f) => o[f] || '').join('.');
   }
 
+  const left = await this.getUniqueFile({
+    existingFiles: [fileB],
+    filename: fileA,
+    uniqueFunction,
+    includeDuplicateSourceRecords,
+  });
   const right = await this.getUniqueFile({
     existingFiles: [fileA],
     filename: fileB,
     uniqueFunction,
+    includeDuplicateSourceRecords,
   });
 
-  const left = await this.getUniqueFile({
-    existingFiles: [fileB],
-    filename: fileA,
-    uniqueFunction,
-  });
   return {
     left, right,
   };
@@ -912,6 +925,9 @@ Worker.prototype.diff.metadata = {
     fileB: {},
     fields: { description: 'Fields to use for uniqueness -- aka primary key.  Defaults to JSON of line' },
     uniqueFunction: {},
+    includeDuplicateSourceRecords: {
+      description: 'Sometimes you want the output to include source dupes, sometimes not, default false',
+    },
   },
 };
 

package/file/S3.js

@@ -187,7 +187,7 @@ Worker.prototype.write.metadata = {
   },
 };
 
-Worker.prototype.list = async function ({ directory }) {
+Worker.prototype.list = async function ({ directory, raw }) {
   if (!directory) throw new Error('directory is required');
   let dir = directory;
   while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
@@ -200,14 +200,17 @@ Worker.prototype.list = async function ({ directory }) {
   });
 
   const { Contents: files, CommonPrefixes } = await s3Client.send(command);
+  if (raw) return files;
   // debug('Prefixes:', { CommonPrefixes });
   const output = [].concat((CommonPrefixes || []).map((f) => ({
     name: f.Prefix.slice(Prefix.length + 1, -1),
     type: 'directory',
   })))
-    .concat((files || []).map(({ Key }) => ({
+    .concat((files || []).map(({ Key, Size, LastModified }) => ({
       name: Key.slice(Prefix.length + 1),
       type: 'file',
+      size: Size,
+      modifiedAt: new Date(LastModified).toISOString(),
     })));
 
   return output;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.6.0",
+  "version": "1.6.3",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {