@engine9-io/input-tools 1.4.1 → 1.5.0

package/file/FileUtilities.js

@@ -15,6 +15,7 @@ const debug = require('debug')('FileWorker');
 const csv = require('csv');
 const JSON5 = require('json5');// Useful for parsing extended JSON
 const languageEncoding = require('detect-file-encoding-and-language');
+const R2Worker = require('./R2');
 const S3Worker = require('./S3');
 const ParquetWorker = require('./Parquet');
 const { streamPacket } = require('./tools');
@@ -401,7 +402,7 @@ Worker.prototype.objectStreamToFile = async function (options) {
 Worker.prototype.transform = async function (options) {
   const worker = this;
 
-  const filename = worker.getFilename(options);
+  const { filename } = options;
 
   debug(`Transforming ${filename}`);
 
@@ -489,19 +490,23 @@ Worker.prototype.stream = async function (
   } else if (filename) {
     if (filename.startsWith('engine9-accounts/')) {
       filename = `${process.env.ENGINE9_ACCOUNT_DIR}/${filename.slice('engine9-accounts/'.length)}`;
-      debug(`Prepending file with ${process.env.ENGINE9_ACCOUNT_DIR}, filename=${filename}`);
+      // debug(`Prepending file with ${process.env.ENGINE9_ACCOUNT_DIR}, filename=${filename}`);
     } else {
-      debug(`Not prepending filename:${filename}`);
+      // debug(`Not prepending filename:${filename}`);
     }
     let encoding; let stream;
     if (filename.slice(-8) === '.parquet') {
       const pq = new ParquetWorker(this);
       stream = (await pq.stream({ filename, columns, limit })).stream;
       encoding = 'object';
-    } else if (filename.indexOf('s3://') === 0) {
+    } else if (filename.startsWith('s3://')) {
       const s3Worker = new S3Worker(this);
       stream = (await s3Worker.stream({ filename, columns, limit })).stream;
       encoding = 'UTF-8';
+    } else if (filename.startsWith('r2://')) {
+      const r2Worker = new R2Worker(this);
+      stream = (await r2Worker.stream({ filename, columns, limit })).stream;
+      encoding = 'UTF-8';
     } else {
       // Check if the file exists, and fast fail if not
       // Otherwise the stream hangs out as a handle
@@ -541,13 +546,13 @@ Worker.prototype.sample.metadata = {
 
 Worker.prototype.write = async function (opts) {
   const { filename, content } = opts;
-  if (filename.indexOf('s3://') === 0) {
-    const s3Worker = new S3Worker(this);
+  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
     const parts = filename.split('/');
     const directory = parts.slice(0, -1).join('/');
     const file = parts.slice(-1)[0];
     // debug(JSON.stringify({ parts, directory, file }));
-    await s3Worker.write({
+    await worker.write({
       directory,
       file,
       content,
@@ -559,7 +564,7 @@ Worker.prototype.write = async function (opts) {
 };
 Worker.prototype.write.metadata = {
   options: {
-    filename: { description: 'Location to write content to, can be local or s3://' },
+    filename: { description: 'Location to write content to, can be local or s3:// or r2://' },
     content: {},
   },
 };
@@ -596,9 +601,9 @@ Worker.prototype.json.metadata = {
 
 Worker.prototype.list = async function ({ directory }) {
   if (!directory) throw new Error('directory is required');
-  if (directory.indexOf('s3://') === 0) {
-    const s3Worker = new S3Worker(this);
-    return s3Worker.list({ directory });
+  if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
+    const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
+    return worker.list({ directory });
   }
   const a = await fsp.readdir(directory, { withFileTypes: true });
   return a.map((f) => ({
@@ -614,9 +619,9 @@ Worker.prototype.list.metadata = {
 
 Worker.prototype.listAll = async function ({ directory }) {
   if (!directory) throw new Error('directory is required');
-  if (directory.indexOf('s3://') === 0) {
-    const s3Worker = new S3Worker(this);
-    return s3Worker.listAll({ directory });
+  if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
+    const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
+    return worker.listAll({ directory });
   }
   const a = await fsp.readdir(directory, { recursive: true });
 
@@ -630,9 +635,9 @@ Worker.prototype.listAll.metadata = {
 
 Worker.prototype.empty = async function ({ directory }) {
   if (!directory) throw new Error('directory is required');
-  if (directory.indexOf('s3://') === 0) {
+  if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
     // currently not emptying S3 this way -- dangerous
-    throw new Error('Cannot empty an s3:// directory');
+    throw new Error('Cannot empty an s3:// or r2:// directory');
   }
   const removed = [];
   // eslint-disable-next-line no-restricted-syntax
@@ -649,11 +654,23 @@ Worker.prototype.empty.metadata = {
 };
 
 Worker.prototype.move = async function ({ filename, target }) {
-  if (!target) throw new Error('directory is required');
-  if (target.indexOf('s3://') === 0) {
-    const s3Worker = new S3Worker(this);
+  if (!target) throw new Error('target is required');
+  if (target.startsWith('s3://') || target.startsWith('r2://')) {
+    if ((target.startsWith('s3://') && filename.startsWith('r2://'))
+      || (target.startsWith('r2://') && filename.startsWith('s3://'))) {
+      throw new Error('Cowardly not copying between services');
+    }
+
+    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
+
+    if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+      // We need to copy and delete
+      const output = await worker.copy({ filename, target });
+      await worker.remove({ filename });
+      return output;
+    }
     const parts = target.split('/');
-    return s3Worker.put({ filename, directory: parts.slice(0, -1).join('/'), file: parts.slice(-1)[0] });
+    return worker.put({ filename, directory: parts.slice(0, -1).join('/'), file: parts.slice(-1)[0] });
   }
   await fsp.mkdir(path.dirname(target), { recursive: true });
   await fsp.rename(filename, target);
@@ -668,9 +685,9 @@ Worker.prototype.move.metadata = {
 
 Worker.prototype.stat = async function ({ filename }) {
   if (!filename) throw new Error('filename is required');
-  if (filename.indexOf('s3://') === 0) {
-    const s3Worker = new S3Worker(this);
-    return s3Worker.stat({ filename });
+  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
+    return worker.stat({ filename });
   }
   const {
     ctime,
@@ -693,6 +710,20 @@ Worker.prototype.stat.metadata = {
   },
 };
 
+Worker.prototype.download = async function ({ filename }) {
+  if (!filename) throw new Error('filename is required');
+  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
+    return worker.download({ filename });
+  }
+  throw new Error('Cannot download a local file');
+};
+Worker.prototype.download.metadata = {
+  options: {
+    filename: {},
+  },
+};
+
 Worker.prototype.head = async function (options) {
   const { stream } = await this.fileToObjectStream(options);
   const chunks = [];

package/file/R2.js

@@ -0,0 +1,36 @@
+const util = require('node:util');
+const {
+  S3Client,
+} = require('@aws-sdk/client-s3');
+const S3 = require('./S3');
+
+function R2(worker) {
+  S3.call(this, worker);
+}
+util.inherits(R2, S3);
+
+R2.prototype.getClient = function () {
+  const missing = ['CLOUDFLARE_R2_ACCOUNT_ID', 'CLOUDFLARE_R2_ACCESS_KEY_ID', 'CLOUDFLARE_R2_SECRET_ACCESS_KEY']
+    .filter((r) => !process.env[r]);
+  if (missing.length > 0) throw new Error(`Missing environment variables for Cloudflare access:${missing.join(',')}`);
+  const ACCOUNT_ID = process.env.CLOUDFLARE_R2_ACCOUNT_ID;
+  const ACCESS_KEY_ID = process.env.CLOUDFLARE_R2_ACCESS_KEY_ID;
+  const SECRET_ACCESS_KEY = process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY;
+
+  if (!this.client) {
+    this.client = new S3Client({
+      // R2 does not strictly require a region, but the SDK expects one. 'auto' works fine.
+      region: 'auto',
+      endpoint: `https://${ACCOUNT_ID}.r2.cloudflarestorage.com`,
+      credentials: {
+        accessKeyId: ACCESS_KEY_ID,
+        secretAccessKey: SECRET_ACCESS_KEY,
+      },
+      forcePathStyle: true, // Important for R2 compatibility
+
+    });
+  }
+  return this.client;
+};
+
+module.exports = R2;

package/file/S3.js

@@ -1,9 +1,11 @@
-const debug = require('debug')('S3Worker');
+const debug = require('debug')('@engine9-io/input/S3');
 const fs = require('node:fs');
 // eslint-disable-next-line import/no-unresolved
 const { mimeType: mime } = require('mime-type/with-db');
 const {
   S3Client,
+  CopyObjectCommand,
+  DeleteObjectCommand,
   GetObjectCommand,
   HeadObjectCommand,
   GetObjectAttributesCommand, PutObjectCommand,
@@ -14,7 +16,10 @@ const { getTempFilename } = require('./tools');
 function Worker() {}
 
 function getParts(filename) {
-  if (!filename || filename.indexOf('s3://') !== 0) throw new Error(`Invalid filename for s3:${filename}`);
+  if (!filename) throw new Error(`Invalid filename: ${filename}`);
+  if (!filename.startsWith('r2://') && !filename.startsWith('s3://')) {
+    throw new Error(`Invalid filename, must start with r2:// or s3://: ${filename}`);
+  }
   const parts = filename.split('/');
   const Bucket = parts[2];
   const Key = parts.slice(3).join('/');
@@ -44,7 +49,7 @@ Worker.prototype.getMetadata.metadata = {
 };
 
 Worker.prototype.stream = async function ({ filename }) {
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const { Bucket, Key } = getParts(filename);
   const command = new GetObjectCommand({ Bucket, Key });
   try {
@@ -62,10 +67,45 @@ Worker.prototype.stream.metadata = {
   },
 };
 
+Worker.prototype.copy = async function ({ filename, target }) {
+  if (!filename.startsWith('s3://')) throw new Error('Cowardly not copying a file not from s3 -- use put instead');
+  const s3Client = this.getClient();
+  const { Bucket, Key } = getParts(target);
+
+  debug(`Copying ${filename} to ${JSON.stringify({ Bucket, Key })}}`);
+
+  const command = new CopyObjectCommand({
+    CopySource: filename.slice(4), // remove the s3:/
+    Bucket,
+    Key,
+  });
+
+  return s3Client.send(command);
+};
+
+Worker.prototype.copy.metadata = {
+  options: {
+    filename: {},
+    target: {},
+  },
+};
+
+Worker.prototype.remove = async function ({ filename }) {
+  const s3Client = this.getClient();
+  const { Bucket, Key } = getParts(filename);
+  const command = new DeleteObjectCommand({ Bucket, Key });
+  return s3Client.send(command);
+};
+Worker.prototype.remove.metadata = {
+  options: {
+    filename: {},
+  },
+};
+
 Worker.prototype.download = async function ({ filename }) {
   const file = filename.split('/').pop();
   const localPath = await getTempFilename({ targetFilename: file });
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const { Bucket, Key } = getParts(filename);
   const command = new GetObjectCommand({ Bucket, Key });
   debug(`Downloading ${file} to ${localPath}`);
@@ -102,7 +142,7 @@ Worker.prototype.put = async function (options) {
   const ContentType = mime.lookup(file);
 
   debug(`Putting ${filename} to ${JSON.stringify({ Bucket, Key, ContentType })}}`);
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
 
   const command = new PutObjectCommand({
     Bucket, Key, Body, ContentType,
@@ -129,7 +169,7 @@ Worker.prototype.write = async function (options) {
   const Body = content;
 
   debug(`Writing content of length ${content.length} to ${JSON.stringify({ Bucket, Key })}}`);
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const ContentType = mime.lookup(file);
 
   const command = new PutObjectCommand({
@@ -151,7 +191,7 @@ Worker.prototype.list = async function ({ directory }) {
   let dir = directory;
   while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
   const { Bucket, Key: Prefix } = getParts(dir);
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const command = new ListObjectsV2Command({
     Bucket,
     Prefix: `${Prefix}/`,
@@ -182,7 +222,7 @@ Worker.prototype.listAll = async function ({ directory }) {
   let dir = directory;
   while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
   const { Bucket, Key: Prefix } = getParts(dir);
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const files = [];
   let ContinuationToken = null;
   do {
@@ -211,7 +251,7 @@ Worker.prototype.listAll.metadata = {
 Worker.prototype.stat = async function ({ filename }) {
   if (!filename) throw new Error('filename is required');
 
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const { Bucket, Key } = getParts(filename);
   const command = new HeadObjectCommand({ Bucket, Key });
   const response = await s3Client.send(command);

package/index.js

@@ -364,6 +364,10 @@ function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
   // eslint-disable-next-line no-restricted-globals
   if (isNaN(ts)) throw new Error(`getTimelineEntryUUID got an invalid date:${o.ts || '<blank>'}`);
   const idString = `${ts.toISOString()}-${o.person_id}-${o.entry_type_id}-${o.source_code_id || 0}`;
+
+  if (!uuidIsValid(o.input_id)) {
+    throw new Error(`Invalid input_id:'${o.input_id}', type ${typeof o.input_id} -- should be a uuid`);
+  }
   // get a temp ID
   const uuid = uuidv5(idString, o.input_id);
   // Change out the ts to match the v7 sorting.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {