@engine9-io/input-tools 1.4.2 → 1.5.1

package/file/FileUtilities.js

@@ -15,6 +15,7 @@ const debug = require('debug')('FileWorker');
 const csv = require('csv');
 const JSON5 = require('json5');// Useful for parsing extended JSON
 const languageEncoding = require('detect-file-encoding-and-language');
+const R2Worker = require('./R2');
 const S3Worker = require('./S3');
 const ParquetWorker = require('./Parquet');
 const { streamPacket } = require('./tools');
@@ -401,7 +402,7 @@ Worker.prototype.objectStreamToFile = async function (options) {
 Worker.prototype.transform = async function (options) {
   const worker = this;
 
-  const filename = worker.getFilename(options);
+  const { filename } = options;
 
   debug(`Transforming ${filename}`);
 
@@ -498,10 +499,14 @@ Worker.prototype.stream = async function (
       const pq = new ParquetWorker(this);
       stream = (await pq.stream({ filename, columns, limit })).stream;
       encoding = 'object';
-    } else if (filename.indexOf('s3://') === 0) {
+    } else if (filename.startsWith('s3://')) {
       const s3Worker = new S3Worker(this);
       stream = (await s3Worker.stream({ filename, columns, limit })).stream;
       encoding = 'UTF-8';
+    } else if (filename.startsWith('r2://')) {
+      const r2Worker = new R2Worker(this);
+      stream = (await r2Worker.stream({ filename, columns, limit })).stream;
+      encoding = 'UTF-8';
     } else {
       // Check if the file exists, and fast fail if not
       // Otherwise the stream hangs out as a handle
@@ -541,13 +546,13 @@ Worker.prototype.sample.metadata = {
 
 Worker.prototype.write = async function (opts) {
   const { filename, content } = opts;
-  if (filename.indexOf('s3://') === 0) {
-    const s3Worker = new S3Worker(this);
+  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
     const parts = filename.split('/');
     const directory = parts.slice(0, -1).join('/');
     const file = parts.slice(-1)[0];
     // debug(JSON.stringify({ parts, directory, file }));
-    await s3Worker.write({
+    await worker.write({
       directory,
       file,
       content,
@@ -559,7 +564,7 @@ Worker.prototype.write = async function (opts) {
 };
 Worker.prototype.write.metadata = {
   options: {
-    filename: { description: 'Location to write content to, can be local or s3://' },
+    filename: { description: 'Location to write content to, can be local or s3:// or r2://' },
     content: {},
   },
 };
@@ -596,9 +601,9 @@ Worker.prototype.json.metadata = {
 
 Worker.prototype.list = async function ({ directory }) {
   if (!directory) throw new Error('directory is required');
-  if (directory.indexOf('s3://') === 0) {
-    const s3Worker = new S3Worker(this);
-    return s3Worker.list({ directory });
+  if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
+    const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
+    return worker.list({ directory });
   }
   const a = await fsp.readdir(directory, { withFileTypes: true });
   return a.map((f) => ({
@@ -614,9 +619,9 @@ Worker.prototype.list.metadata = {
 
 Worker.prototype.listAll = async function ({ directory }) {
   if (!directory) throw new Error('directory is required');
-  if (directory.indexOf('s3://') === 0) {
-    const s3Worker = new S3Worker(this);
-    return s3Worker.listAll({ directory });
+  if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
+    const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
+    return worker.listAll({ directory });
   }
   const a = await fsp.readdir(directory, { recursive: true });
 
@@ -630,9 +635,9 @@ Worker.prototype.listAll.metadata = {
 
 Worker.prototype.empty = async function ({ directory }) {
   if (!directory) throw new Error('directory is required');
-  if (directory.indexOf('s3://') === 0) {
+  if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
     // currently not emptying S3 this way -- dangerous
-    throw new Error('Cannot empty an s3:// directory');
+    throw new Error('Cannot empty an s3:// or r2:// directory');
   }
   const removed = [];
   // eslint-disable-next-line no-restricted-syntax
@@ -650,17 +655,22 @@ Worker.prototype.empty.metadata = {
 
 Worker.prototype.move = async function ({ filename, target }) {
   if (!target) throw new Error('target is required');
-  if (target.indexOf('s3://') === 0) {
-    const s3Worker = new S3Worker(this);
+  if (target.startsWith('s3://') || target.startsWith('r2://')) {
+    if ((target.startsWith('s3://') && filename.startsWith('r2://'))
+      || (target.startsWith('r2://') && filename.startsWith('s3://'))) {
+      throw new Error('Cowardly not copying between services');
+    }
 
-    if (filename.indexOf('s3://') === 0) {
+    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
+
+    if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
       // We need to copy and delete
-      const output = await s3Worker.copy({ filename, target });
-      await s3Worker.remove({ filename });
+      const output = await worker.copy({ filename, target });
+      await worker.remove({ filename });
       return output;
     }
     const parts = target.split('/');
-    return s3Worker.put({ filename, directory: parts.slice(0, -1).join('/'), file: parts.slice(-1)[0] });
+    return worker.put({ filename, directory: parts.slice(0, -1).join('/'), file: parts.slice(-1)[0] });
   }
   await fsp.mkdir(path.dirname(target), { recursive: true });
   await fsp.rename(filename, target);
@@ -675,9 +685,9 @@ Worker.prototype.move.metadata = {
 
 Worker.prototype.stat = async function ({ filename }) {
   if (!filename) throw new Error('filename is required');
-  if (filename.indexOf('s3://') === 0) {
-    const s3Worker = new S3Worker(this);
-    return s3Worker.stat({ filename });
+  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
+    return worker.stat({ filename });
   }
   const {
     ctime,
@@ -700,11 +710,25 @@ Worker.prototype.stat.metadata = {
   },
 };
 
+Worker.prototype.download = async function ({ filename }) {
+  if (!filename) throw new Error('filename is required');
+  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
+    return worker.download({ filename });
+  }
+  throw new Error('Cannot download a local file');
+};
+Worker.prototype.download.metadata = {
+  options: {
+    filename: {},
+  },
+};
+
 Worker.prototype.head = async function (options) {
-  const { stream } = await this.fileToObjectStream(options);
+  const limit = options.limit || 3;
+  const { stream } = await this.fileToObjectStream({ ...options, limit });
   const chunks = [];
 
-  const limit = options.limit || 3;
   let counter = 0;
   // eslint-disable-next-line no-restricted-syntax
   for await (const chunk of stream) {

package/file/Parquet.js

@@ -45,7 +45,7 @@ Worker.prototype.schema.metadata = {
 };
 
 function cleanColumnName(name) {
-  name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
+  return name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
 }
 
 Worker.prototype.stream = async function (options) {
@@ -60,11 +60,10 @@ Worker.prototype.stream = async function (options) {
     if (typeof options.columns === 'string') requestedColumns = options.columns.split(',').map((d) => d.trim());
     else requestedColumns = options.columns.map((d) => (d.name ? d.name.trim() : d.trim()));
     requestedColumns.forEach((c) => {
-      columns = columns.concat(
-        fieldList.filter((f) => (
-          f.name === c || cleanColumnName(f.name) === cleanColumnName(c)
-        )).map((f) => f.name),
-      );
+      const matchingCols = fieldList.filter((f) => (
+        f.name === c || cleanColumnName(f.name) === cleanColumnName(c)
+      )).map((f) => f.name);
+      columns = columns.concat(matchingCols);
     });
   }
   let limit = 0;

package/file/R2.js

@@ -0,0 +1,36 @@
+const util = require('node:util');
+const {
+  S3Client,
+} = require('@aws-sdk/client-s3');
+const S3 = require('./S3');
+
+function R2(worker) {
+  S3.call(this, worker);
+}
+util.inherits(R2, S3);
+
+R2.prototype.getClient = function () {
+  const missing = ['CLOUDFLARE_R2_ACCOUNT_ID', 'CLOUDFLARE_R2_ACCESS_KEY_ID', 'CLOUDFLARE_R2_SECRET_ACCESS_KEY']
+    .filter((r) => !process.env[r]);
+  if (missing.length > 0) throw new Error(`Missing environment variables for Cloudflare access:${missing.join(',')}`);
+  const ACCOUNT_ID = process.env.CLOUDFLARE_R2_ACCOUNT_ID;
+  const ACCESS_KEY_ID = process.env.CLOUDFLARE_R2_ACCESS_KEY_ID;
+  const SECRET_ACCESS_KEY = process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY;
+
+  if (!this.client) {
+    this.client = new S3Client({
+      // R2 does not strictly require a region, but the SDK expects one. 'auto' works fine.
+      region: 'auto',
+      endpoint: `https://${ACCOUNT_ID}.r2.cloudflarestorage.com`,
+      credentials: {
+        accessKeyId: ACCESS_KEY_ID,
+        secretAccessKey: SECRET_ACCESS_KEY,
+      },
+      forcePathStyle: true, // Important for R2 compatibility
+
+    });
+  }
+  return this.client;
+};
+
+module.exports = R2;

package/file/S3.js

@@ -16,7 +16,10 @@ const { getTempFilename } = require('./tools');
 function Worker() {}
 
 function getParts(filename) {
-  if (!filename || filename.indexOf('s3://') !== 0) throw new Error(`Invalid filename for s3:${filename}`);
+  if (!filename) throw new Error(`Invalid filename: ${filename}`);
+  if (!filename.startsWith('r2://') && !filename.startsWith('s3://')) {
+    throw new Error(`Invalid filename, must start with r2:// or s3://: ${filename}`);
+  }
   const parts = filename.split('/');
   const Bucket = parts[2];
   const Key = parts.slice(3).join('/');
@@ -102,7 +105,7 @@ Worker.prototype.remove.metadata = {
 Worker.prototype.download = async function ({ filename }) {
   const file = filename.split('/').pop();
   const localPath = await getTempFilename({ targetFilename: file });
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const { Bucket, Key } = getParts(filename);
   const command = new GetObjectCommand({ Bucket, Key });
   debug(`Downloading ${file} to ${localPath}`);
@@ -139,7 +142,7 @@ Worker.prototype.put = async function (options) {
   const ContentType = mime.lookup(file);
 
   debug(`Putting ${filename} to ${JSON.stringify({ Bucket, Key, ContentType })}}`);
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
 
   const command = new PutObjectCommand({
     Bucket, Key, Body, ContentType,
@@ -166,7 +169,7 @@ Worker.prototype.write = async function (options) {
   const Body = content;
 
   debug(`Writing content of length ${content.length} to ${JSON.stringify({ Bucket, Key })}}`);
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const ContentType = mime.lookup(file);
 
   const command = new PutObjectCommand({
@@ -188,7 +191,7 @@ Worker.prototype.list = async function ({ directory }) {
   let dir = directory;
   while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
   const { Bucket, Key: Prefix } = getParts(dir);
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const command = new ListObjectsV2Command({
     Bucket,
     Prefix: `${Prefix}/`,
@@ -248,7 +251,7 @@ Worker.prototype.listAll.metadata = {
 Worker.prototype.stat = async function ({ filename }) {
   if (!filename) throw new Error('filename is required');
 
-  const s3Client = new S3Client({});
+  const s3Client = this.getClient();
   const { Bucket, Key } = getParts(filename);
   const command = new HeadObjectCommand({ Bucket, Key });
   const response = await s3Client.send(command);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.4.2",
+  "version": "1.5.1",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {
@@ -17,7 +17,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.723.0",
-    "@dsnp/parquetjs": "^1.8.6",
+    "@dsnp/parquetjs": "^1.8.7",
     "archiver": "^7.0.1",
     "async-mutex": "^0.5.0",
     "csv": "^6.3.11",