@engine9-io/input-tools 1.4.0 → 1.4.2

package/file/FileUtilities.js

@@ -489,9 +489,9 @@ Worker.prototype.stream = async function (
   } else if (filename) {
     if (filename.startsWith('engine9-accounts/')) {
       filename = `${process.env.ENGINE9_ACCOUNT_DIR}/${filename.slice('engine9-accounts/'.length)}`;
-      debug(`Prepending file with ${process.env.ENGINE9_ACCOUNT_DIR}, filename=${filename}`);
+      // debug(`Prepending file with ${process.env.ENGINE9_ACCOUNT_DIR}, filename=${filename}`);
     } else {
-      debug(`Not prepending filename:${filename}`);
+      // debug(`Not prepending filename:${filename}`);
     }
     let encoding; let stream;
     if (filename.slice(-8) === '.parquet') {
@@ -649,9 +649,16 @@ Worker.prototype.empty.metadata = {
 };
 
 Worker.prototype.move = async function ({ filename, target }) {
-  if (!target) throw new Error('directory is required');
+  if (!target) throw new Error('target is required');
   if (target.indexOf('s3://') === 0) {
     const s3Worker = new S3Worker(this);
+
+    if (filename.indexOf('s3://') === 0) {
+      // We need to copy and delete
+      const output = await s3Worker.copy({ filename, target });
+      await s3Worker.remove({ filename });
+      return output;
+    }
     const parts = target.split('/');
     return s3Worker.put({ filename, directory: parts.slice(0, -1).join('/'), file: parts.slice(-1)[0] });
   }

package/file/S3.js

@@ -1,9 +1,11 @@
-const debug = require('debug')('S3Worker');
+const debug = require('debug')('@engine9-io/input/S3');
 const fs = require('node:fs');
 // eslint-disable-next-line import/no-unresolved
 const { mimeType: mime } = require('mime-type/with-db');
 const {
   S3Client,
+  CopyObjectCommand,
+  DeleteObjectCommand,
   GetObjectCommand,
   HeadObjectCommand,
   GetObjectAttributesCommand, PutObjectCommand,
@@ -44,7 +46,7 @@ Worker.prototype.getMetadata.metadata = {
 };
 
 Worker.prototype.stream = async function ({ filename }) {
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const { Bucket, Key } = getParts(filename);
   const command = new GetObjectCommand({ Bucket, Key });
   try {
@@ -62,6 +64,41 @@ Worker.prototype.stream.metadata = {
   },
 };
 
+Worker.prototype.copy = async function ({ filename, target }) {
+  if (!filename.startsWith('s3://')) throw new Error('Cowardly not copying a file not from s3 -- use put instead');
+  const s3Client = this.getClient();
+  const { Bucket, Key } = getParts(target);
+
+  debug(`Copying ${filename} to ${JSON.stringify({ Bucket, Key })}}`);
+
+  const command = new CopyObjectCommand({
+    CopySource: filename.slice(4), // remove the s3:/
+    Bucket,
+    Key,
+  });
+
+  return s3Client.send(command);
+};
+
+Worker.prototype.copy.metadata = {
+  options: {
+    filename: {},
+    target: {},
+  },
+};
+
+Worker.prototype.remove = async function ({ filename }) {
+  const s3Client = this.getClient();
+  const { Bucket, Key } = getParts(filename);
+  const command = new DeleteObjectCommand({ Bucket, Key });
+  return s3Client.send(command);
+};
+Worker.prototype.remove.metadata = {
+  options: {
+    filename: {},
+  },
+};
+
 Worker.prototype.download = async function ({ filename }) {
   const file = filename.split('/').pop();
   const localPath = await getTempFilename({ targetFilename: file });
@@ -182,7 +219,7 @@ Worker.prototype.listAll = async function ({ directory }) {
   let dir = directory;
   while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
   const { Bucket, Key: Prefix } = getParts(dir);
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const files = [];
   let ContinuationToken = null;
   do {
@@ -192,7 +229,7 @@ Worker.prototype.listAll = async function ({ directory }) {
       ContinuationToken,
       // Delimiter: '/',
     });
-    debug(`Sending command with ContinuationToken ${ContinuationToken}`);
+    debug(`Sending List command with prefix ${Prefix} with ContinuationToken ${ContinuationToken}`);
     // eslint-disable-next-line no-await-in-loop
     const result = await s3Client.send(command);
     const newFiles = (result.Contents?.map((d) => `s3://${Bucket}/${d.Key}`) || []);

package/index.js

@@ -364,6 +364,10 @@ function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
   // eslint-disable-next-line no-restricted-globals
   if (isNaN(ts)) throw new Error(`getTimelineEntryUUID got an invalid date:${o.ts || '<blank>'}`);
   const idString = `${ts.toISOString()}-${o.person_id}-${o.entry_type_id}-${o.source_code_id || 0}`;
+
+  if (!uuidIsValid(o.input_id)) {
+    throw new Error(`Invalid input_id:'${o.input_id}', type ${typeof o.input_id} -- should be a uuid`);
+  }
   // get a temp ID
   const uuid = uuidv5(idString, o.input_id);
   // Change out the ts to match the v7 sorting.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.4.0",
+  "version": "1.4.2",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {