@engine9-io/input-tools 1.9.8 → 1.9.10

package/file/FileUtilities.js

@@ -734,7 +734,7 @@ Worker.prototype.listAll = async function ({ directory, start: s, end: e }) {
   if (e) end = relativeDate(e).getTime();
   if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
     const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    return worker.listAll({ directory });
+    return worker.listAll({ directory, start, end });
   }
   const a = await fsp.readdir(directory, { recursive: true });
 

package/file/S3.js

@@ -11,7 +11,7 @@ const {
   PutObjectCommand,
   ListObjectsV2Command
 } = require('@aws-sdk/client-s3');
-const { getTempFilename } = require('./tools');
+const { getTempFilename, relativeDate } = require('./tools');
 
 function Worker() {
   this.prefix = 's3';
@@ -261,9 +261,12 @@ Worker.prototype.list.metadata = {
   }
 };
 /* List everything with the prefix */
-Worker.prototype.listAll = async function ({ directory }) {
+Worker.prototype.listAll = async function (options) {
+  const { directory } = options;
   if (!directory) throw new Error('directory is required');
   let dir = directory;
+  const start = options.start && relativeDate(options.start);
+  const end = options.end && relativeDate(options.end);
   while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
   const { Bucket, Key } = getParts(dir);
   const s3Client = this.getClient();
@@ -281,7 +284,16 @@ Worker.prototype.listAll = async function ({ directory }) {
     debug(`Sending List command with prefix ${Prefix} with ContinuationToken ${ContinuationToken}`);
 
     const result = await s3Client.send(command);
-    const newFiles = result.Contents?.map((d) => `${this.prefix}://${Bucket}/${d.Key}`) || [];
+    const newFiles =
+      result.Contents?.filter(({ LastModified }) => {
+        if (start && new Date(LastModified) < start) {
+          return false;
+        } else if (end && new Date(LastModified) > end) {
+          return false;
+        } else {
+          return true;
+        }
+      })?.map((d) => `${this.prefix}://${Bucket}/${d.Key}`) || [];
     debug(`Retrieved ${newFiles.length} new files, total ${files.length},sample ${newFiles.slice(0, 3).join(',')}`);
     files.push(...newFiles);
     ContinuationToken = result.NextContinuationToken;

package/index.js

@@ -249,6 +249,28 @@ function getUUIDTimestamp(uuid) {
   return new Date(ts);
 }
 
+function getEntryTypeId(o, { defaults = {} } = {}) {
+  let id = o.entry_type_id || defaults.entry_type_id;
+  if (id) return id;
+  const etype = o.entry_type || defaults.entry_type;
+  if (!etype) {
+    throw new Error('No entry_type, nor entry_type_id specified, specify a defaultEntryType');
+  }
+  id = TIMELINE_ENTRY_TYPES[etype];
+  if (id === undefined) throw new Error(`Invalid entry_type: ${etype}`);
+  return id;
+}
+function getEntryType(o, defaults = {}) {
+  let etype = o.entry_type || defaults.entry_type;
+  if (etype) return etype;
+
+  const id = o.entry_type_id || defaults.entry_type_id;
+
+  etype = TIMELINE_ENTRY_TYPES[id];
+  if (etype === undefined) throw new Error(`Invalid entry_type: ${etype}`);
+  return etype;
+}
+
 const requiredTimelineEntryFields = ['ts', 'entry_type_id', 'input_id', 'person_id'];
 
 function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
@@ -277,6 +299,7 @@ function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
     // may not match this standard, uuid sorting isn't guaranteed
     return getUUIDv7(o.ts, uuid);
   }
+  const entry_type_id = getEntryTypeId(o);
 
   const missing = requiredTimelineEntryFields.filter((d) => o[d] === undefined); // 0 could be an entry type value
 
@@ -286,7 +309,7 @@ function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
   // attempted conversion here
 
   if (isNaN(ts)) throw new Error(`getTimelineEntryUUID got an invalid date:${o.ts || '<blank>'}`);
-  const idString = `${ts.toISOString()}-${o.person_id}-${o.entry_type_id}-${o.source_code_id || 0}`;
+  const idString = `${ts.toISOString()}-${o.person_id}-${entry_type_id}-${o.source_code_id || 0}`;
 
   if (!uuidIsValid(o.input_id)) {
     throw new Error(`Invalid input_id:'${o.input_id}', type ${typeof o.input_id} -- should be a uuid`);
@@ -298,27 +321,6 @@ function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
   // may not match this standard, uuid sorting isn't guaranteed
   return getUUIDv7(ts, uuid);
 }
-function getEntryTypeId(o, { defaults = {} } = {}) {
-  let id = o.entry_type_id || defaults.entry_type_id;
-  if (id) return id;
-  const etype = o.entry_type || defaults.entry_type;
-  if (!etype) {
-    throw new Error('No entry_type, nor entry_type_id specified, specify a defaultEntryType');
-  }
-  id = TIMELINE_ENTRY_TYPES[etype];
-  if (id === undefined) throw new Error(`Invalid entry_type: ${etype}`);
-  return id;
-}
-function getEntryType(o, defaults = {}) {
-  let etype = o.entry_type || defaults.entry_type;
-  if (etype) return etype;
-
-  const id = o.entry_type_id || defaults.entry_type_id;
-
-  etype = TIMELINE_ENTRY_TYPES[id];
-  if (etype === undefined) throw new Error(`Invalid entry_type: ${etype}`);
-  return etype;
-}
 
 function getDateRangeArray(startDate, endDate) {
   const start = new Date(startDate);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.9.8",
+  "version": "1.9.10",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {