@engine9-io/input-tools 1.7.6 → 1.7.8

package/.prettierrc

@@ -0,0 +1,7 @@
+{
+  "semi": true,
+  "singleQuote": true,
+  "tabWidth": 2,
+  "printWidth": 120,
+  "trailingComma": "none"
+}

package/file/FileUtilities.js

@@ -153,7 +153,7 @@ Worker.prototype.detectEncoding.metadata = {
 Internal method to transform a file into a stream of objects.
 */
 Worker.prototype.fileToObjectStream = async function (options) {
-  const { filename, columns, limit: limitOption } = options;
+  const { filename, columns, limit: limitOption,format:formatOverride } = options;
 
   // handle stream item
   if (options.stream) {
@@ -203,14 +203,15 @@ Worker.prototype.fileToObjectStream = async function (options) {
   } else {
     stream.setEncoding(encoding);
   }
+  let format=formatOverride || postfix;
 
-  if (postfix === 'csv') {
+  if (format === 'csv') {
     const csvTransforms = this.csvToObjectTransforms({ ...options });
     transforms = transforms.concat(csvTransforms.transforms);
-  } else if (postfix === 'txt') {
+  } else if (format === 'txt') {
     const csvTransforms = this.csvToObjectTransforms({ ...options, delimiter: '\t' });
     transforms = transforms.concat(csvTransforms.transforms);
-  } else if (postfix === 'jsonl') {
+  } else if (format === 'jsonl') {
     /* Type of JSON that has the names in an array in the first record,
     and the values in JSON arrays thereafter
     */

package/file/R2.js

@@ -6,6 +6,7 @@ const S3 = require('./S3');
 
 function R2(worker) {
   S3.call(this, worker);
+  this.prefix='r2';
 }
 util.inherits(R2, S3);
 

package/file/S3.js

@@ -12,7 +12,9 @@ const {
 } = require('@aws-sdk/client-s3');
 const { getTempFilename } = require('./tools');
 
-function Worker() {}
+function Worker() {
+  this.prefix='s3';
+}
 
 function getParts(filename) {
   if (!filename) throw new Error(`Invalid filename: ${filename}`);
@@ -247,7 +249,7 @@ Worker.prototype.listAll = async function ({ directory }) {
     debug(`Sending List command with prefix ${Prefix} with ContinuationToken ${ContinuationToken}`);
     // eslint-disable-next-line no-await-in-loop
     const result = await s3Client.send(command);
-    const newFiles = (result.Contents?.map((d) => `s3://${Bucket}/${d.Key}`) || []);
+    const newFiles = (result.Contents?.map((d) => `${this.prefix}://${Bucket}/${d.Key}`) || []);
     debug(`Retrieved ${newFiles.length} new files, total ${files.length},sample ${newFiles.slice(0, 3).join(',')}`);
     files.push(...newFiles);
     ContinuationToken = result.NextContinuationToken;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.7.6",
+  "version": "1.7.8",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {