@engine9-io/input-tools 1.9.7 → 1.9.9

package/file/FileUtilities.js

@@ -734,7 +734,7 @@ Worker.prototype.listAll = async function ({ directory, start: s, end: e }) {
   if (e) end = relativeDate(e).getTime();
   if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
     const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    return worker.listAll({ directory });
+    return worker.listAll({ directory, start, end });
   }
   const a = await fsp.readdir(directory, { recursive: true });
 
@@ -769,9 +769,12 @@ Worker.prototype.listAll = async function ({ directory, start: s, end: e }) {
 };
 Worker.prototype.listAll.metadata = {
   options: {
-    directory: { required: true }
+    directory: { required: true },
+    start: {},
+    end: {}
   }
 };
+
 Worker.prototype.moveAll = async function (options) {
   const { directory, targetDirectory } = options;
   if (!directory) throw new Error('directory is required');
@@ -821,6 +824,23 @@ Worker.prototype.empty.metadata = {
   }
 };
 
+Worker.prototype.removeAll = async function (options) {
+  const filenames = await this.listAll(options);
+
+  const pLimit = await import('p-limit');
+
+  const limitedMethod = pLimit.default(10);
+
+  return Promise.all(filenames.map((filename) => limitedMethod(async () => this.remove({ filename }))));
+};
+Worker.prototype.removeAll.metadata = {
+  options: {
+    directory: { required: true },
+    start: {},
+    end: {}
+  }
+};
+
 Worker.prototype.remove = async function ({ filename }) {
   if (!filename) throw new Error('filename is required');
   if (typeof filename !== 'string') throw new Error(`filename isn't a string:${JSON.stringify(filename)}`);
@@ -878,7 +898,7 @@ Worker.prototype.move = async function ({ filename, target, remove = true }) {
       await fsp.rename(filename, target);
     } catch (e) {
       //it may be a filesystem issue moving between items
-      debug(e);
+      debug('Assuming this is a filesystem crosslink error, ignoring ', e.getMessage());
       await fsp.copyFile(filename, target);
       await fsp.unlink(filename);
     }

package/file/S3.js

@@ -11,7 +11,7 @@ const {
   PutObjectCommand,
   ListObjectsV2Command
 } = require('@aws-sdk/client-s3');
-const { getTempFilename } = require('./tools');
+const { getTempFilename, relativeDate } = require('./tools');
 
 function Worker() {
   this.prefix = 's3';
@@ -261,9 +261,12 @@ Worker.prototype.list.metadata = {
   }
 };
 /* List everything with the prefix */
-Worker.prototype.listAll = async function ({ directory }) {
+Worker.prototype.listAll = async function (options) {
+  const { directory } = options;
   if (!directory) throw new Error('directory is required');
   let dir = directory;
+  const start = options.start && relativeDate(options.start);
+  const end = options.end && relativeDate(options.end);
   while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
   const { Bucket, Key } = getParts(dir);
   const s3Client = this.getClient();
@@ -281,7 +284,16 @@ Worker.prototype.listAll = async function ({ directory }) {
     debug(`Sending List command with prefix ${Prefix} with ContinuationToken ${ContinuationToken}`);
 
     const result = await s3Client.send(command);
-    const newFiles = result.Contents?.map((d) => `${this.prefix}://${Bucket}/${d.Key}`) || [];
+    const newFiles =
+      result.Contents?.filter(({ LastModified }) => {
+        if (start && new Date(LastModified) < start) {
+          return false;
+        } else if (end && new Date(LastModified) > end) {
+          return false;
+        } else {
+          return true;
+        }
+      })?.map((d) => `${this.prefix}://${Bucket}/${d.Key}`) || [];
     debug(`Retrieved ${newFiles.length} new files, total ${files.length},sample ${newFiles.slice(0, 3).join(',')}`);
     files.push(...newFiles);
     ContinuationToken = result.NextContinuationToken;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.9.7",
+  "version": "1.9.9",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {