@engine9-io/input-tools 1.9.11 → 2.0.0

package/file/FileUtilities.js

@@ -1,1196 +1,1124 @@
-const fs = require('node:fs');
-
+import fs from 'node:fs';
+import path from 'node:path';
+import zlib from 'node:zlib';
+import nodestream from 'node:stream';
+import promises from 'node:stream/promises';
+import csv$0 from 'csv';
+import debug$0 from 'debug';
+import xlstream from 'xlstream';
+import JSON5 from 'json5';
+import languageEncoding from 'detect-file-encoding-and-language';
+import R2Worker from './R2.js';
+import S3Worker from './S3.js';
+import ParquetWorker from './Parquet.js';
+import { bool, getTempFilename, getStringArray, getTempDir, makeStrings, streamPacket, relativeDate } from './tools.js';
 const fsp = fs.promises;
-const path = require('node:path');
-const zlib = require('node:zlib');
-const { Readable, Transform, PassThrough, Writable } = require('node:stream');
-const { pipeline } = require('node:stream/promises');
-const { stringify } = require('csv');
-
-const debug = require('debug')('@engine9-io/file');
-
-const { getXlsxStream } = require('xlstream');
-const csv = require('csv');
-const JSON5 = require('json5');
-
-const languageEncoding = require('detect-file-encoding-and-language');
-const R2Worker = require('./R2');
-const S3Worker = require('./S3');
-const ParquetWorker = require('./Parquet');
-
-const {
-  bool,
-  getTempFilename,
-  getStringArray,
-  getTempDir,
-  makeStrings,
-  streamPacket,
-  relativeDate
-} = require('./tools');
-
+const { Readable, Transform, PassThrough, Writable } = nodestream;
+const { pipeline } = promises;
+const { stringify } = csv$0;
+const debug = debug$0('@engine9-io/file');
+const { getXlsxStream } = xlstream;
+const csv = csv$0;
 function Worker({ accountId }) {
-  this.accountId = accountId;
+    this.accountId = accountId;
 }
-
 class LineReaderTransform extends Transform {
-  constructor(options = {}) {
-    super({ ...options, readableObjectMode: true });
-    this.buffer = '';
-  }
-
-  _transform(chunk, encoding, callback) {
-    this.buffer += chunk.toString();
-    const lines = this.buffer.split(/\r?\n/);
-    this.buffer = lines.pop();
-    lines.forEach((line) => this.push(line));
-    callback();
-  }
-
-  _flush(callback) {
-    if (this.buffer) {
-      this.push(this.buffer);
-    }
-    callback();
-  }
+    constructor(options = {}) {
+        super({ ...options, readableObjectMode: true });
+        this.buffer = '';
+    }
+    _transform(chunk, encoding, callback) {
+        this.buffer += chunk.toString();
+        const lines = this.buffer.split(/\r?\n/);
+        this.buffer = lines.pop();
+        lines.forEach((line) => this.push(line));
+        callback();
+    }
+    _flush(callback) {
+        if (this.buffer) {
+            this.push(this.buffer);
+        }
+        callback();
+    }
 }
-
 Worker.prototype.csvToObjectTransforms = function (options) {
-  const transforms = [];
-  const delimiter = options.delimiter || ',';
-
-  const headerMapping =
-    options.headerMapping ||
-    function (d) {
-      return d;
+    const transforms = [];
+    const delimiter = options.delimiter || ',';
+    const headerMapping = options.headerMapping ||
+        function (d) {
+            return d;
+        };
+    let lastLine = null;
+    let head = null;
+    const skipLinesWithError = bool(options.skip_lines_with_error, false);
+    const parserOptions = {
+        relax: true,
+        skip_empty_lines: true,
+        delimiter,
+        max_limit_on_data_read: 10000000,
+        skip_lines_with_error: skipLinesWithError
     };
-  let lastLine = null;
-  let head = null;
-
-  const skipLinesWithError = bool(options.skip_lines_with_error, false);
-  const parserOptions = {
-    relax: true,
-    skip_empty_lines: true,
-    delimiter,
-    max_limit_on_data_read: 10000000,
-    skip_lines_with_error: skipLinesWithError
-  };
-  if (options.skip) parserOptions.from_line = options.skip;
-  if (options.relax_column_count) parserOptions.relax_column_count = true;
-  if (options.quote_escape) {
-    parserOptions.escape = options.quote_escape;
-  }
-  if (options.limit) {
-    parserOptions.to = options.limit;
-  }
-
-  debug('Parser options=', parserOptions);
-  const parser = csv.parse(parserOptions);
-  parser.on('error', (error) => {
-    debug('fileToObjectStream: Error parsing csv file');
-    debug(lastLine);
-    throw new Error(error);
-  });
-
-  const blankAndHeaderCheck = new Transform({
-    objectMode: true,
-    transform(row, enc, cb) {
-      // Blank rows
-      if (row.length === 0) return cb();
-      if (row.length === 1 && !row[0]) return cb();
-
-      if (!head) {
-        head = row.map(headerMapping);
-        return cb();
-      }
-
-      const o = {};
-      head.forEach((_h, i) => {
-        const h = _h.trim();
-        if (h) {
-          o[h] = row[i];
+    if (options.skip)
+        parserOptions.from_line = options.skip;
+    if (options.relax_column_count)
+        parserOptions.relax_column_count = true;
+    if (options.quote_escape) {
+        parserOptions.escape = options.quote_escape;
+    }
+    if (options.limit) {
+        parserOptions.to = options.limit;
+    }
+    debug('Parser options=', parserOptions);
+    const parser = csv.parse(parserOptions);
+    parser.on('error', (error) => {
+        debug('fileToObjectStream: Error parsing csv file');
+        debug(lastLine);
+        throw new Error(error);
+    });
+    const blankAndHeaderCheck = new Transform({
+        objectMode: true,
+        transform(row, enc, cb) {
+            // Blank rows
+            if (row.length === 0)
+                return cb();
+            if (row.length === 1 && !row[0])
+                return cb();
+            if (!head) {
+                head = row.map(headerMapping);
+                return cb();
+            }
+            const o = {};
+            head.forEach((_h, i) => {
+                const h = _h.trim();
+                if (h) {
+                    o[h] = row[i];
+                }
+            });
+            lastLine = row.join(delimiter);
+            return cb(null, o);
         }
-      });
-
-      lastLine = row.join(delimiter);
-      return cb(null, o);
-    }
-  });
-
-  transforms.push(parser);
-  transforms.push(blankAndHeaderCheck);
-
-  return { transforms };
+    });
+    transforms.push(parser);
+    transforms.push(blankAndHeaderCheck);
+    return { transforms };
 };
-
 Worker.prototype.detectEncoding = async function (options) {
-  if (options.encoding_override) return { encoding: options.encoding_override };
-  // Limit to only the top N bytes -- for perfomance
-  // Be wary, though, as gzip files may require a certain minimum number of bytes to decompress
-  const bytes = 64 * 1024;
-  const buff = Buffer.alloc(bytes);
-  const fd = await fsp.open(options.filename);
-  await fd.read(buff, 0, bytes);
-  let finalBuff = buff;
-  if (options.filename.slice(-3) === '.gz') {
-    // This code deals with scenarios where the buffer coming in may not be exactly the gzip
-    // needed chunk size.
-    finalBuff = await new Promise((resolve, reject) => {
-      const bufferBuilder = [];
-      const decompressStream = zlib
-        .createGunzip()
-        .on('data', (chunk) => {
-          bufferBuilder.push(chunk);
-        })
-        .on('close', () => {
-          resolve(Buffer.concat(bufferBuilder));
-        })
-        .on('error', (err) => {
-          if (err.errno !== -5) {
-            // EOF: expected
-            reject(err);
-          }
+    if (options.encoding_override)
+        return { encoding: options.encoding_override };
+    // Limit to only the top N bytes -- for perfomance
+    // Be wary, though, as gzip files may require a certain minimum number of bytes to decompress
+    const bytes = 64 * 1024;
+    const buff = Buffer.alloc(bytes);
+    const fd = await fsp.open(options.filename);
+    await fd.read(buff, 0, bytes);
+    let finalBuff = buff;
+    if (options.filename.slice(-3) === '.gz') {
+        // This code deals with scenarios where the buffer coming in may not be exactly the gzip
+        // needed chunk size.
+        finalBuff = await new Promise((resolve, reject) => {
+            const bufferBuilder = [];
+            const decompressStream = zlib
+                .createGunzip()
+                .on('data', (chunk) => {
+                bufferBuilder.push(chunk);
+            })
+                .on('close', () => {
+                resolve(Buffer.concat(bufferBuilder));
+            })
+                .on('error', (err) => {
+                if (err.errno !== -5) {
+                    // EOF: expected
+                    reject(err);
+                }
+            });
+            decompressStream.write(buff);
+            decompressStream.end();
         });
-      decompressStream.write(buff);
-      decompressStream.end();
-    });
-  }
-
-  return languageEncoding(finalBuff);
+    }
+    return languageEncoding(finalBuff);
 };
-
 Worker.prototype.detectEncoding.metadata = {
-  options: {
-    filename: { required: true }
-  }
+    options: {
+        filename: { required: true }
+    }
 };
-
 Worker.prototype.xlsxToObjectStream = async function (options) {
-  let { filename } = options;
-
-  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
-    // We need to copy and delete
-    let worker = null;
-    if (filename.startsWith('r2://')) {
-      worker = new R2Worker(this);
-    } else {
-      worker = new S3Worker(this);
-    }
-    const target = getTempFilename({ targetFilename: filename.split('/').pop() });
-
-    await worker.copy({ filename, target });
-    filename = target;
-  }
-  let stream = await getXlsxStream({
-    filePath: filename,
-    sheet: 0
-  });
-  let keys = null;
-  stream = stream.pipe(
-    new Transform({
-      objectMode: true,
-      transform(d, enc, cb) {
-        if (!keys) {
-          keys = d?.raw.arr;
-          cb();
-        } else {
-          let o = {};
-          keys.forEach((k, i) => {
-            o[k] = d?.raw?.arr?.[i];
-          });
-          cb(null, o);
+    let { filename } = options;
+    if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+        // We need to copy and delete
+        let worker = null;
+        if (filename.startsWith('r2://')) {
+            worker = new R2Worker(this);
+        }
+        else {
+            worker = new S3Worker(this);
+        }
+        const target = getTempFilename({ targetFilename: filename.split('/').pop() });
+        await worker.copy({ filename, target });
+        filename = target;
+    }
+    let stream = await getXlsxStream({
+        filePath: filename,
+        sheet: 0
+    });
+    let keys = null;
+    stream = stream.pipe(new Transform({
+        objectMode: true,
+        transform(d, enc, cb) {
+            if (!keys) {
+                keys = d?.raw.arr;
+                cb();
+            }
+            else {
+                let o = {};
+                keys.forEach((k, i) => {
+                    o[k] = d?.raw?.arr?.[i];
+                });
+                cb(null, o);
+            }
         }
-      }
-    })
-  );
-
-  return { stream };
+    }));
+    return { stream };
 };
-
 Worker.prototype.getFormat = async function (options) {
-  const { sourcePostfix, filename, format: formatOverride } = options;
-  let postfix = sourcePostfix || filename.toLowerCase().split('.').pop();
-
-  if (postfix === 'gz') {
-    postfix = filename.toLowerCase().split('.');
-    postfix = postfix[postfix.length - 2];
-  }
-  return formatOverride || postfix;
+    const { sourcePostfix, filename, format: formatOverride } = options;
+    let postfix = sourcePostfix || filename.toLowerCase().split('.').pop();
+    if (postfix === 'gz') {
+        postfix = filename.toLowerCase().split('.');
+        postfix = postfix[postfix.length - 2];
+    }
+    return formatOverride || postfix;
 };
-
 /*
 Commonly used method to transform a file into a stream of objects.
 */
 Worker.prototype.fileToObjectStream = async function (options) {
-  const { filename, columns, limit: limitOption, format: formatOverride } = options;
-
-  // handle stream item
-  if (options.stream) {
-    if (Array.isArray(options.stream)) {
-      return { stream: Readable.from(options.stream) };
-    }
-    // probably already a stream
-    if (typeof options.stream === 'object') return { stream: options.stream };
-    throw new Error(`Invalid stream type:${typeof options.stream}`);
-  }
-  let limit;
-  if (limitOption) limit = parseInt(limitOption, 10);
-  if (!filename) throw new Error('fileToObjectStream: filename is required');
-  if (filename.split('.').pop().toLowerCase() === 'xlsx') {
-    return this.xlsxToObjectStream(options);
-  }
-  let postfix = options.sourcePostfix || filename.toLowerCase().split('.').pop();
-  if (postfix === 'zip') {
-    debug('Invalid filename:', { filename });
-    throw new Error('Cowardly refusing to turn a .zip file into an object stream, turn into a csv first');
-  }
-
-  const streamInfo = await this.stream({
-    filename,
-    columns,
-    limit
-  });
-  const { encoding } = streamInfo;
-  let { stream } = streamInfo;
-  if (!stream) throw new Error(`No stream found in fileToObjectStream from filename ${filename}`);
-  if (encoding === 'object') {
-    // already an object
-    return { stream };
-  }
-
-  let count = 0;
-
-  let transforms = [];
-
-  if (postfix === 'gz') {
-    const gunzip = zlib.createGunzip();
-    transforms.push(gunzip);
-    gunzip.setEncoding(encoding);
-    // encoding = null;// Default encoding
-    postfix = filename.toLowerCase().split('.');
-    postfix = postfix[postfix.length - 2];
-    debug(`Using gunzip parser because postfix is .gz, encoding=${encoding}`);
-  } else {
-    stream.setEncoding(encoding);
-  }
-  let format = formatOverride || postfix;
-
-  debug(`Reading file ${filename} with encoding: ${encoding} and format ${format}`);
-
-  if (format === 'csv') {
-    const csvTransforms = this.csvToObjectTransforms({ ...options });
-    transforms = transforms.concat(csvTransforms.transforms);
-  } else if (format === 'txt') {
-    const csvTransforms = this.csvToObjectTransforms({ ...options, delimiter: '\t' });
-    transforms = transforms.concat(csvTransforms.transforms);
-  } else if (format === 'jsonl') {
-    /* Type of JSON that has the names in an array in the first record,
-    and the values in JSON arrays thereafter
-    */
-    let headers = null;
-
-    const lineReader = new LineReaderTransform();
-
-    const jsonlTransform = new Transform({
-      objectMode: true,
-      transform(d, enc, cb) {
-        if (!d) return cb();
-        let obj;
-        try {
-          obj = JSON5.parse(d);
-        } catch (e) {
-          debug('Invalid line:');
-          debug(d);
-          throw e;
-        }
-        /* JSONL could potentially start with an array of names,
-        in which case we need to map the subsequent values
-      */
-        if (headers === null) {
-          if (Array.isArray(obj)) {
-            headers = obj;
-            return cb();
-          }
-          headers = false;
+    const { filename, columns, limit: limitOption, format: formatOverride } = options;
+    // handle stream item
+    if (options.stream) {
+        if (Array.isArray(options.stream)) {
+            return { stream: Readable.from(options.stream) };
         }
-        if (headers) {
-          const mapped = {};
-          headers.forEach((name, i) => {
-            mapped[name] = obj[i];
-          });
-          this.push(mapped);
-        } else {
-          this.push(obj);
+        // probably already a stream
+        if (typeof options.stream === 'object')
+            return { stream: options.stream };
+        throw new Error(`Invalid stream type:${typeof options.stream}`);
+    }
+    let limit;
+    if (limitOption)
+        limit = parseInt(limitOption, 10);
+    if (!filename)
+        throw new Error('fileToObjectStream: filename is required');
+    if (filename.split('.').pop().toLowerCase() === 'xlsx') {
+        return this.xlsxToObjectStream(options);
+    }
+    let postfix = options.sourcePostfix || filename.toLowerCase().split('.').pop();
+    if (postfix === 'zip') {
+        debug('Invalid filename:', { filename });
+        throw new Error('Cowardly refusing to turn a .zip file into an object stream, turn into a csv first');
+    }
+    const streamInfo = await this.stream({
+        filename,
+        columns,
+        limit
+    });
+    const { encoding } = streamInfo;
+    let { stream } = streamInfo;
+    if (!stream)
+        throw new Error(`No stream found in fileToObjectStream from filename ${filename}`);
+    if (encoding === 'object') {
+        // already an object
+        return { stream };
+    }
+    let count = 0;
+    let transforms = [];
+    if (postfix === 'gz') {
+        const gunzip = zlib.createGunzip();
+        transforms.push(gunzip);
+        gunzip.setEncoding(encoding);
+        // encoding = null;// Default encoding
+        postfix = filename.toLowerCase().split('.');
+        postfix = postfix[postfix.length - 2];
+        debug(`Using gunzip parser because postfix is .gz, encoding=${encoding}`);
+    }
+    else {
+        stream.setEncoding(encoding);
+    }
+    let format = formatOverride || postfix;
+    debug(`Reading file ${filename} with encoding: ${encoding} and format ${format}`);
+    if (format === 'csv') {
+        const csvTransforms = this.csvToObjectTransforms({ ...options });
+        transforms = transforms.concat(csvTransforms.transforms);
+    }
+    else if (format === 'txt') {
+        const csvTransforms = this.csvToObjectTransforms({ ...options, delimiter: '\t' });
+        transforms = transforms.concat(csvTransforms.transforms);
+    }
+    else if (format === 'jsonl') {
+        /* Type of JSON that has the names in an array in the first record,
+            and the values in JSON arrays thereafter
+            */
+        let headers = null;
+        const lineReader = new LineReaderTransform();
+        const jsonlTransform = new Transform({
+            objectMode: true,
+            transform(d, enc, cb) {
+                if (!d)
+                    return cb();
+                let obj;
+                try {
+                    obj = JSON5.parse(d);
+                }
+                catch (e) {
+                    debug('Invalid line:');
+                    debug(d);
+                    throw e;
+                }
+                /* JSONL could potentially start with an array of names,
+                        in which case we need to map the subsequent values
+                      */
+                if (headers === null) {
+                    if (Array.isArray(obj)) {
+                        headers = obj;
+                        return cb();
+                    }
+                    headers = false;
+                }
+                if (headers) {
+                    const mapped = {};
+                    headers.forEach((name, i) => {
+                        mapped[name] = obj[i];
+                    });
+                    this.push(mapped);
+                }
+                else {
+                    this.push(obj);
+                }
+                return cb();
+            }
+        });
+        transforms.push(lineReader);
+        transforms.push(jsonlTransform);
+    }
+    else {
+        throw new Error(`Unsupported file type: ${postfix}`);
+    }
+    const countAndDebug = new Transform({
+        objectMode: true,
+        transform(d, enc, cb) {
+            if (count === 0) {
+                debug('Sample object from file:', d);
+            }
+            count += 1;
+            if ((count < 5000 && count % 1000 === 0) || count % 50000 === 0) {
+                debug(`fileToObjectStream transformed ${count} lines`);
+            }
+            this.push(d);
+            cb();
+        },
+        flush(cb) {
+            // If there's no records at all, push a dummy record, and specify 0 records
+            // Don't push dummy records anymore -- legacy cruft
+            debug(`Completed reading file, records=${count}`);
+            /* if (count === 0) {
+                    const o = { _is_placeholder: true };
+            
+                    if (head) head.forEach((c) => { o[c] = null; });
+                    this.push(o);
+                  } */
+            cb();
         }
-        return cb();
-      }
     });
-
-    transforms.push(lineReader);
-    transforms.push(jsonlTransform);
-  } else {
-    throw new Error(`Unsupported file type: ${postfix}`);
-  }
-  const countAndDebug = new Transform({
-    objectMode: true,
-    transform(d, enc, cb) {
-      if (count === 0) {
-        debug('Sample object from file:', d);
-      }
-      count += 1;
-      if ((count < 5000 && count % 1000 === 0) || count % 50000 === 0) {
-        debug(`fileToObjectStream transformed ${count} lines`);
-      }
-      this.push(d);
-      cb();
-    },
-    flush(cb) {
-      // If there's no records at all, push a dummy record, and specify 0 records
-      // Don't push dummy records anymore -- legacy cruft
-      debug(`Completed reading file, records=${count}`);
-      /* if (count === 0) {
-        const o = { _is_placeholder: true };
-
-        if (head) head.forEach((c) => { o[c] = null; });
-        this.push(o);
-      } */
-      cb();
-    }
-  });
-
-  transforms.push(countAndDebug);
-  transforms.forEach((t) => {
-    stream = stream.pipe(t);
-  });
-
-  return { stream };
+    transforms.push(countAndDebug);
+    transforms.forEach((t) => {
+        stream = stream.pipe(t);
+    });
+    return { stream };
 };
 Worker.prototype.getFileWriterStream = async function (options = {}) {
-  const accountId = options.accountId || this.accountId;
-  if (!accountId) throw new Error('getFileWriterStream has no accountId');
-  const targetFormat = options.targetFormat || 'csv';
-  const tempDir = await getTempDir({ accountId });
-  let { fileExtendedType } = options;
-  if (fileExtendedType) fileExtendedType += '.';
-  else fileExtendedType = '';
-  // So, this could change, but it's easier to read
-  // dates in a filename than UUIDs, so this is
-  // a unique-ish filename generator
-  const uniqueNumberedDate = `${new Date().toISOString().replace(/[^0-9]*/g, '')}.${Math.floor(Math.random() * 1000)}`;
-  let filename = `${tempDir}${path.sep}${uniqueNumberedDate}.${fileExtendedType}${targetFormat}`;
-  if (bool(options.gzip, false)) filename += '.gz';
-  const stream = fs.createWriteStream(filename);
-  debug('FileWriterStream writing to file ', filename);
-
-  return { filename, stream };
+    const accountId = options.accountId || this.accountId;
+    if (!accountId)
+        throw new Error('getFileWriterStream has no accountId');
+    const targetFormat = options.targetFormat || 'csv';
+    const tempDir = await getTempDir({ accountId });
+    let { fileExtendedType } = options;
+    if (fileExtendedType)
+        fileExtendedType += '.';
+    else
+        fileExtendedType = '';
+    // So, this could change, but it's easier to read
+    // dates in a filename than UUIDs, so this is
+    // a unique-ish filename generator
+    const uniqueNumberedDate = `${new Date().toISOString().replace(/[^0-9]*/g, '')}.${Math.floor(Math.random() * 1000)}`;
+    let filename = `${tempDir}${path.sep}${uniqueNumberedDate}.${fileExtendedType}${targetFormat}`;
+    if (bool(options.gzip, false))
+        filename += '.gz';
+    const stream = fs.createWriteStream(filename);
+    debug('FileWriterStream writing to file ', filename);
+    return { filename, stream };
 };
-
 Worker.prototype.getOutputStreams = async function (options) {
-  const { filename, stream: fileWriterStream } = await this.getFileWriterStream(options);
-
-  let { transform } = options;
-  if (typeof options.transform === 'function') {
-    if (options.transform.length === 3) {
-      transform = new Transform({
-        objectMode: true,
-        async transform(item, encoding, cb) {
-          options.transform(item, encoding, cb);
+    const { filename, stream: fileWriterStream } = await this.getFileWriterStream(options);
+    let { transform } = options;
+    if (typeof options.transform === 'function') {
+        if (options.transform.length === 3) {
+            transform = new Transform({
+                objectMode: true,
+                async transform(item, encoding, cb) {
+                    options.transform(item, encoding, cb);
+                }
+            });
         }
-      });
-    } else {
-      transform = new Transform({
-        objectMode: true,
-        async transform(item, encoding, cb) {
-          cb(null, options.transform(item));
+        else {
+            transform = new Transform({
+                objectMode: true,
+                async transform(item, encoding, cb) {
+                    cb(null, options.transform(item));
+                }
+            });
         }
-      });
-    }
-  } else if (options.transform) {
-    transform = options.transform;
-  }
-  const { flatten } = options;
-  let flattenTransform = null;
-
-  if (bool(flatten, false)) {
-    flattenTransform = new Transform({
-      objectMode: true,
-      async transform(item, enc, cb) {
-        // first item establishes the keys to use
-        let o = {};
-        Object.keys(item).forEach((k) => {
-          let v = item[k];
-          if (!o[k]) {
-            if (typeof v === 'object') {
-              while (Array.isArray(v)) [v] = v; // get first array item
-              o = { ...o, ...v };
-            } else {
-              o[k] = v;
+    }
+    else if (options.transform) {
+        transform = options.transform;
+    }
+    const { flatten } = options;
+    let flattenTransform = null;
+    if (bool(flatten, false)) {
+        flattenTransform = new Transform({
+            objectMode: true,
+            async transform(item, enc, cb) {
+                // first item establishes the keys to use
+                let o = {};
+                Object.keys(item).forEach((k) => {
+                    let v = item[k];
+                    if (!o[k]) {
+                        if (typeof v === 'object') {
+                            while (Array.isArray(v))
+                                [v] = v; // get first array item
+                            o = { ...o, ...v };
+                        }
+                        else {
+                            o[k] = v;
+                        }
+                    }
+                });
+                cb(null, o);
             }
-          }
         });
-        cb(null, o);
-      }
-    });
-  }
-
-  const stats = {
-    records: 0
-  };
-  let stringifier;
-  if (options.targetFormat === 'jsonl') {
-    stringifier = new Transform({
-      objectMode: true,
-      transform(d, encoding, cb) {
-        cb(false, `${JSON.stringify(d)}\n`);
-      }
-    });
-  } else {
-    stringifier = stringify({ header: true });
-  }
-  let gzip = new PassThrough();
-  if (options.gzip) {
-    gzip = zlib.createGzip();
-  }
-  const streams = [
-    transform,
-    flattenTransform,
-    new Transform({
-      objectMode: true,
-      transform(d, enc, cb) {
-        stats.records += 1;
-        cb(null, d);
-      }
-    }),
-    stringifier,
-    gzip,
-    fileWriterStream
-  ].filter(Boolean);
-  return { filename, streams, stats };
+    }
+    const stats = {
+        records: 0
+    };
+    let stringifier;
+    if (options.targetFormat === 'jsonl') {
+        stringifier = new Transform({
+            objectMode: true,
+            transform(d, encoding, cb) {
+                cb(false, `${JSON.stringify(d)}\n`);
+            }
+        });
+    }
+    else {
+        stringifier = stringify({ header: true });
+    }
+    let gzip = new PassThrough();
+    if (options.gzip) {
+        gzip = zlib.createGzip();
+    }
+    const streams = [
+        transform,
+        flattenTransform,
+        new Transform({
+            objectMode: true,
+            transform(d, enc, cb) {
+                stats.records += 1;
+                cb(null, d);
+            }
+        }),
+        stringifier,
+        gzip,
+        fileWriterStream
+    ].filter(Boolean);
+    return { filename, streams, stats };
 };
 Worker.prototype.objectStreamToFile = async function (options) {
-  const { filename, streams, stats } = await this.getOutputStreams(options);
-  const { stream: inStream } = options;
-  streams.unshift(inStream);
-  await pipeline(streams);
-  return { filename, records: stats.records };
+    const { filename, streams, stats } = await this.getOutputStreams(options);
+    const { stream: inStream } = options;
+    streams.unshift(inStream);
+    await pipeline(streams);
+    return { filename, records: stats.records };
 };
-
 Worker.prototype.transform = async function (options) {
-  const worker = this;
-
-  const { filename } = options;
-
-  debug(`Transforming ${filename}`);
-
-  options.filename = filename;
-  let { stream } = await worker.fileToObjectStream(options);
-  if (typeof stream.pipe !== 'function') {
-    debug(stream);
-    throw new Error('No pipe in stream');
-  }
-
-  let t = options.transform;
-
-  // No longer need this
-  delete options.transform;
-  if (!t) {
-    t = function (d, enc, cb) {
-      d.is_test_transform = true;
-      cb(null, d);
-    };
-  }
-
-  if (!Array.isArray(t)) t = [t];
-  Object.keys(t).forEach((key) => {
-    let f = t[key];
-    if (typeof f === 'function') {
-      f = new Transform({
-        objectMode: true,
-        transform: f
-      });
-    }
-
-    stream = stream.pipe(f);
-  });
-
-  const { targetFormat } = options;
-
-  if (
-    !targetFormat &&
-    (filename.toLowerCase().slice(-4) === '.csv' || filename.toLowerCase().slice(-7) === '.csv.gz')
-  ) {
-    options.targetFormat = 'csv';
-  }
-
-  return worker.objectStreamToFile({ ...options, stream });
+    const worker = this;
+    const { filename } = options;
+    debug(`Transforming ${filename}`);
+    options.filename = filename;
+    let { stream } = await worker.fileToObjectStream(options);
+    if (typeof stream.pipe !== 'function') {
+        debug(stream);
+        throw new Error('No pipe in stream');
+    }
+    let t = options.transform;
+    // No longer need this
+    delete options.transform;
+    if (!t) {
+        t = function (d, enc, cb) {
+            d.is_test_transform = true;
+            cb(null, d);
+        };
+    }
+    if (!Array.isArray(t))
+        t = [t];
+    Object.keys(t).forEach((key) => {
+        let f = t[key];
+        if (typeof f === 'function') {
+            f = new Transform({
+                objectMode: true,
+                transform: f
+            });
+        }
+        stream = stream.pipe(f);
+    });
+    const { targetFormat } = options;
+    if (!targetFormat &&
+        (filename.toLowerCase().slice(-4) === '.csv' || filename.toLowerCase().slice(-7) === '.csv.gz')) {
+        options.targetFormat = 'csv';
+    }
+    return worker.objectStreamToFile({ ...options, stream });
 };
-
 Worker.prototype.transform.metadata = {
-  options: {
-    sourcePostfix: { description: "Override the source postfix, if for example it's a csv" },
-    encoding: { description: 'Manual override of source file encoding' },
-    names: { description: 'Target field names (e.g. my_new_field,x,y,z)' },
-    values: {
-      description:
-        "Comma delimited source field name, or Handlebars [[ ]] merge fields (e.g. 'my_field,x,y,z', '[[field1]]-[[field2]]', etc)"
-    },
-    targetFilename: { description: 'Custom name of the output file (default auto-generated)' },
-    targetFormat: { description: 'Output format -- csv supported, or none for txt (default)' },
-    targetRowDelimiter: { description: 'Row delimiter (default \n)' },
-    targetFieldDelimiter: { description: 'Field delimiter (default \t or ,)' }
-  }
+    options: {
+        sourcePostfix: { description: "Override the source postfix, if for example it's a csv" },
+        encoding: { description: 'Manual override of source file encoding' },
+        names: { description: 'Target field names (e.g. my_new_field,x,y,z)' },
+        values: {
+            description: "Comma delimited source field name, or Handlebars [[ ]] merge fields (e.g. 'my_field,x,y,z', '[[field1]]-[[field2]]', etc)"
+        },
+        targetFilename: { description: 'Custom name of the output file (default auto-generated)' },
+        targetFormat: { description: 'Output format -- csv supported, or none for txt (default)' },
+        targetRowDelimiter: { description: 'Row delimiter (default \n)' },
+        targetFieldDelimiter: { description: 'Field delimiter (default \t or ,)' }
+    }
 };
 Worker.prototype.testTransform = async function (options) {
-  return this.transform({
-    ...options,
-    transform(d, enc, cb) {
-      d.transform_time = new Date();
-      cb(null, d);
-    }
-  });
+    return this.transform({
+        ...options,
+        transform(d, enc, cb) {
+            d.transform_time = new Date();
+            cb(null, d);
+        }
+    });
 };
 Worker.prototype.testTransform.metadata = {
-  options: {
-    filename: true
-  }
+    options: {
+        filename: true
+    }
 };
-
 /* Get a stream from an actual stream, or an array, or a file */
 Worker.prototype.stream = async function (options) {
-  const { stream: inputStream, packet, type, columns, limit, filename: filenameOpt } = options;
-  let filename = filenameOpt;
-
-  if (inputStream) {
-    if (Array.isArray(inputStream)) {
-      return { stream: Readable.from(inputStream) };
-    }
-    // probably already a stream
-    if (typeof inputStream === 'object') return { stream: inputStream, encoding: 'object' };
-    throw new Error(`Invalid stream type:${typeof inputStream}`);
-  } else if (filename) {
-    if (filename.startsWith('engine9-accounts/')) {
-      filename = `${process.env.ENGINE9_ACCOUNT_DIR}/${filename.slice('engine9-accounts/'.length)}`;
-      // debug(`Prepending file with ${process.env.ENGINE9_ACCOUNT_DIR}, filename=${filename}`);
-    } else {
-      // debug(`Not prepending filename:${filename}`);
-    }
-    let encoding;
-    let stream;
-    if (filename.slice(-8) === '.parquet') {
-      const pq = new ParquetWorker(this);
-      stream = (await pq.stream({ filename, columns, limit })).stream;
-      encoding = 'object';
-    } else if (filename.startsWith('s3://')) {
-      const s3Worker = new S3Worker(this);
-      stream = (await s3Worker.stream({ filename, columns, limit })).stream;
-      encoding = 'UTF-8';
-    } else if (filename.startsWith('r2://')) {
-      const r2Worker = new R2Worker(this);
-      stream = (await r2Worker.stream({ filename, columns, limit })).stream;
-      encoding = 'UTF-8';
-    } else {
-      // Check if the file exists, and fast fail if not
-      // Otherwise the stream hangs out as a handle
-      try {
-        await fsp.stat(filename);
-      } catch (e) {
-        debug(`Error reading file ${filename}, current directory: ${process.cwd()},__dirname:${__dirname}`);
-        throw e;
-      }
-      stream = fs.createReadStream(filename);
-      encoding = (await this.detectEncoding({ filename })).encoding;
-    }
-    return { stream, encoding };
-  } else if (packet) {
-    let { stream: packetStream } = await streamPacket({ packet, type, limit });
-    const { transforms } = this.csvToObjectTransforms({});
-    transforms.forEach((t) => {
-      packetStream = packetStream.pipe(t);
-    });
-    return { stream: packetStream };
-  } else {
-    throw new Error('stream must be passed a stream, filename, or packet');
-  }
+    const { stream: inputStream, packet, type, columns, limit, filename: filenameOpt } = options;
+    let filename = filenameOpt;
+    if (inputStream) {
+        if (Array.isArray(inputStream)) {
+            return { stream: Readable.from(inputStream) };
+        }
+        // probably already a stream
+        if (typeof inputStream === 'object')
+            return { stream: inputStream, encoding: 'object' };
+        throw new Error(`Invalid stream type:${typeof inputStream}`);
+    }
+    else if (filename) {
+        if (filename.startsWith('engine9-accounts/')) {
+            filename = `${process.env.ENGINE9_ACCOUNT_DIR}/${filename.slice('engine9-accounts/'.length)}`;
+            // debug(`Prepending file with ${process.env.ENGINE9_ACCOUNT_DIR}, filename=${filename}`);
+        }
+        else {
+            // debug(`Not prepending filename:${filename}`);
+        }
+        let encoding;
+        let stream;
+        if (filename.slice(-8) === '.parquet') {
+            const pq = new ParquetWorker(this);
+            stream = (await pq.stream({ filename, columns, limit })).stream;
+            encoding = 'object';
+        }
+        else if (filename.startsWith('s3://')) {
+            const s3Worker = new S3Worker(this);
+            stream = (await s3Worker.stream({ filename, columns, limit })).stream;
+            encoding = 'UTF-8';
+        }
+        else if (filename.startsWith('r2://')) {
+            const r2Worker = new R2Worker(this);
+            stream = (await r2Worker.stream({ filename, columns, limit })).stream;
+            encoding = 'UTF-8';
+        }
+        else {
+            // Check if the file exists, and fast fail if not
+            // Otherwise the stream hangs out as a handle
+            try {
+                await fsp.stat(filename);
+            }
+            catch (e) {
+                debug(`Error reading file ${filename}, current directory: ${process.cwd()},__dirname:${__dirname}`);
+                throw e;
+            }
+            stream = fs.createReadStream(filename);
+            encoding = (await this.detectEncoding({ filename })).encoding;
+        }
+        return { stream, encoding };
+    }
+    else if (packet) {
+        let { stream: packetStream } = await streamPacket({ packet, type, limit });
+        const { transforms } = this.csvToObjectTransforms({});
+        transforms.forEach((t) => {
+            packetStream = packetStream.pipe(t);
+        });
+        return { stream: packetStream };
+    }
+    else {
+        throw new Error('stream must be passed a stream, filename, or packet');
+    }
 };
-
 Worker.prototype.sample = async function (opts) {
-  opts.limit = opts.limit || 10;
-  const { stream } = await this.fileToObjectStream(opts);
-  return stream.toArray();
+    opts.limit = opts.limit || 10;
+    const { stream } = await this.fileToObjectStream(opts);
+    return stream.toArray();
 };
 Worker.prototype.sample.metadata = {
-  options: {
-    filename: {}
-  }
+    options: {
+        filename: {}
+    }
 };
 Worker.prototype.toArray = async function (opts) {
-  const { stream } = await this.fileToObjectStream(opts);
-  return stream.toArray();
+    const { stream } = await this.fileToObjectStream(opts);
+    return stream.toArray();
 };
 Worker.prototype.toArray.metadata = {
-  options: {
-    filename: {}
-  }
+    options: {
+        filename: {}
+    }
 };
-
 Worker.prototype.write = async function (opts) {
-  const { filename, content } = opts;
-  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
-    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    const parts = filename.split('/');
-    const directory = parts.slice(0, -1).join('/');
-    const file = parts.slice(-1)[0];
-    // debug(JSON.stringify({ parts, directory, file }));
-    await worker.write({
-      directory,
-      file,
-      content
-    });
-  } else {
-    const directory = path.dirname(filename);
-    await fsp.mkdir(directory, { recursive: true });
-    await fsp.writeFile(filename, content);
-  }
-  return { success: true, filename };
+    const { filename, content } = opts;
+    if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+        const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
+        const parts = filename.split('/');
+        const directory = parts.slice(0, -1).join('/');
+        const file = parts.slice(-1)[0];
+        // debug(JSON.stringify({ parts, directory, file }));
+        await worker.write({
+            directory,
+            file,
+            content
+        });
+    }
+    else {
+        const directory = path.dirname(filename);
+        await fsp.mkdir(directory, { recursive: true });
+        await fsp.writeFile(filename, content);
+    }
+    return { success: true, filename };
 };
 Worker.prototype.write.metadata = {
-  options: {
-    filename: { description: 'Location to write content to, can be local or s3:// or r2://' },
-    content: {}
-  }
+    options: {
+        filename: { description: 'Location to write content to, can be local or s3:// or r2://' },
+        content: {}
+    }
 };
-
 async function streamToString(stream) {
-  // lets have a ReadableStream as a stream variable
-  const chunks = [];
-
-  for await (const chunk of stream) {
-    chunks.push(Buffer.from(chunk));
-  }
-
-  return Buffer.concat(chunks).toString('utf-8');
+    // lets have a ReadableStream as a stream variable
+    const chunks = [];
+    for await (const chunk of stream) {
+        chunks.push(Buffer.from(chunk));
+    }
+    return Buffer.concat(chunks).toString('utf-8');
 }
 /*
 Retrieves and parsed
 */
 Worker.prototype.json = async function (opts) {
-  const { stream } = await this.stream(opts);
-  const str = await streamToString(stream);
-  try {
-    return JSON5.parse(str);
-  } catch (e) {
-    debug(e);
-    throw new Error(`Unparseable JSON received: ${opts.filename || '(native stream)'}`);
-  }
+    const { stream } = await this.stream(opts);
+    const str = await streamToString(stream);
+    try {
+        return JSON5.parse(str);
+    }
+    catch (e) {
+        debug(e);
+        throw new Error(`Unparseable JSON received: ${opts.filename || '(native stream)'}`);
+    }
 };
 Worker.prototype.json.metadata = {
-  options: {
-    filename: { description: 'Get a javascript object from a file' }
-  }
+    options: {
+        filename: { description: 'Get a javascript object from a file' }
+    }
 };
-
 Worker.prototype.list = async function ({ directory, start: s, end: e }) {
-  if (!directory) throw new Error('directory is required');
-  let start = null;
-  let end = null;
-  if (s) start = relativeDate(s);
-  if (e) end = relativeDate(e);
-
-  if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
-    const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    return worker.list({ directory, start, end });
-  }
-  const a = await fsp.readdir(directory, { withFileTypes: true });
-
-  const withModified = [];
-  for (const file of a) {
-    const fullPath = path.join(directory, file.name);
-    const stats = await fsp.stat(fullPath);
-    if (start && stats.mtime < start.getTime()) {
-      //do not include
-    } else if (end && stats.mtime > end.getTime()) {
-      //do nothing
-    } else {
-      withModified.push({
-        name: file.name,
-        type: file.isDirectory() ? 'directory' : 'file',
-        modifiedAt: new Date(stats.mtime).toISOString()
-      });
-    }
-  }
-
-  return withModified;
+    if (!directory)
+        throw new Error('directory is required');
+    let start = null;
+    let end = null;
+    if (s)
+        start = relativeDate(s);
+    if (e)
+        end = relativeDate(e);
+    if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
+        const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
+        return worker.list({ directory, start, end });
+    }
+    const a = await fsp.readdir(directory, { withFileTypes: true });
+    const withModified = [];
+    for (const file of a) {
+        const fullPath = path.join(directory, file.name);
+        const stats = await fsp.stat(fullPath);
+        if (start && stats.mtime < start.getTime()) {
+            //do not include
+        }
+        else if (end && stats.mtime > end.getTime()) {
+            //do nothing
+        }
+        else {
+            withModified.push({
+                name: file.name,
+                type: file.isDirectory() ? 'directory' : 'file',
+                modifiedAt: new Date(stats.mtime).toISOString()
+            });
+        }
+    }
+    return withModified;
 };
 Worker.prototype.list.metadata = {
-  options: {
-    directory: { required: true }
-  }
+    options: {
+        directory: { required: true }
+    }
 };
-
 Worker.prototype.listAll = async function ({ directory, start: s, end: e }) {
-  if (!directory) throw new Error('directory is required');
-  let start = null;
-  let end = null;
-  if (s) start = relativeDate(s).getTime();
-  if (e) end = relativeDate(e).getTime();
-  if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
-    const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    return worker.listAll({ directory, start, end });
-  }
-  const a = await fsp.readdir(directory, { recursive: true });
-
-  let files = a.map((f) => `${directory}/${f}`);
-  if (!start && !end) {
-    return files;
-  }
-  const pLimit = await import('p-limit');
-
-  const limitedMethod = pLimit.default(10);
-  const filesWithinLimit = [];
-
-  await Promise.all(
-    files.map((filename) =>
-      limitedMethod(async () => {
+    if (!directory)
+        throw new Error('directory is required');
+    let start = null;
+    let end = null;
+    if (s)
+        start = relativeDate(s).getTime();
+    if (e)
+        end = relativeDate(e).getTime();
+    if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
+        const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
+        return worker.listAll({ directory, start, end });
+    }
+    const a = await fsp.readdir(directory, { recursive: true });
+    let files = a.map((f) => `${directory}/${f}`);
+    if (!start && !end) {
+        return files;
+    }
+    const pLimit = await import('p-limit');
+    const limitedMethod = pLimit.default(10);
+    const filesWithinLimit = [];
+    await Promise.all(files.map((filename) => limitedMethod(async () => {
         const stats = await fsp.stat(filename);
         if (start && stats.mtime < start) {
-          //do not include
-        } else if (end && stats.mtime > end) {
-          //do nothing
-        } else {
-          filesWithinLimit.push({
-            name: filename,
-            type: stats.isDirectory() ? 'directory' : 'file',
-            modifiedAt: new Date(stats.mtime).toISOString()
-          });
+            //do not include
+        }
+        else if (end && stats.mtime > end) {
+            //do nothing
         }
-      })
-    )
-  );
-  return filesWithinLimit;
+        else {
+            filesWithinLimit.push({
+                name: filename,
+                type: stats.isDirectory() ? 'directory' : 'file',
+                modifiedAt: new Date(stats.mtime).toISOString()
+            });
+        }
+    })));
+    return filesWithinLimit;
 };
 Worker.prototype.listAll.metadata = {
-  options: {
-    directory: { required: true },
-    start: {},
-    end: {}
-  }
+    options: {
+        directory: { required: true },
+        start: {},
+        end: {}
+    }
 };
-
 Worker.prototype.moveAll = async function (options) {
-  const { directory, targetDirectory } = options;
-  if (!directory) throw new Error('directory is required');
-  if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
-    const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    return worker.moveAll(options);
-  }
-  const a = await this.listAll(options);
-
-  let configs = a.map((f) => {
-    let filename = typeof f === 'string' ? f : f.filename;
-    return {
-      filename,
-      target: filename.replace(directory, targetDirectory)
-    };
-  });
-  const pLimit = await import('p-limit');
-
-  const limitedMethod = pLimit.default(10);
-
-  return Promise.all(configs.map(({ filename, target }) => limitedMethod(async () => this.move({ filename, target }))));
+    const { directory, targetDirectory } = options;
+    if (!directory)
+        throw new Error('directory is required');
+    if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
+        const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
+        return worker.moveAll(options);
+    }
+    const a = await this.listAll(options);
+    let configs = a.map((f) => {
+        let filename = typeof f === 'string' ? f : f.filename;
+        return {
+            filename,
+            target: filename.replace(directory, targetDirectory)
+        };
+    });
+    const pLimit = await import('p-limit');
+    const limitedMethod = pLimit.default(10);
+    return Promise.all(configs.map(({ filename, target }) => limitedMethod(async () => this.move({ filename, target }))));
 };
 Worker.prototype.moveAll.metadata = {
-  options: {
-    directory: { required: true },
-    targetDirectory: { required: true }
-  }
+    options: {
+        directory: { required: true },
+        targetDirectory: { required: true }
+    }
 };
-
 Worker.prototype.empty = async function ({ directory }) {
-  if (!directory) throw new Error('directory is required');
-  if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
-    // currently not emptying S3 this way -- dangerous
-    throw new Error('Cannot empty an s3:// or r2:// directory');
-  }
-  const removed = [];
-
-  for (const file of await fsp.readdir(directory)) {
-    removed.push(file);
-    await fsp.unlink(path.join(directory, file));
-  }
-  return { directory, removed };
+    if (!directory)
+        throw new Error('directory is required');
+    if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
+        // currently not emptying S3 this way -- dangerous
+        throw new Error('Cannot empty an s3:// or r2:// directory');
+    }
+    const removed = [];
+    for (const file of await fsp.readdir(directory)) {
+        removed.push(file);
+        await fsp.unlink(path.join(directory, file));
+    }
+    return { directory, removed };
 };
 Worker.prototype.empty.metadata = {
-  options: {
-    directory: { required: true }
-  }
+    options: {
+        directory: { required: true }
+    }
 };
-
 Worker.prototype.removeAll = async function (options) {
-  const filenames = await this.listAll(options);
-
-  const pLimit = await import('p-limit');
-
-  const limitedMethod = pLimit.default(10);
-
-  return Promise.all(filenames.map((filename) => limitedMethod(async () => this.remove({ filename }))));
+    const filenames = await this.listAll(options);
+    const pLimit = await import('p-limit');
+    const limitedMethod = pLimit.default(10);
+    return Promise.all(filenames.map((filename) => limitedMethod(async () => this.remove({ filename }))));
 };
 Worker.prototype.removeAll.metadata = {
-  options: {
-    directory: { required: true },
-    start: {},
-    end: {}
-  }
+    options: {
+        directory: { required: true },
+        start: {},
+        end: {}
+    }
 };
-
 Worker.prototype.remove = async function ({ filename }) {
-  if (!filename) throw new Error('filename is required');
-  if (typeof filename !== 'string') throw new Error(`filename isn't a string:${JSON.stringify(filename)}`);
-  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
-    let worker = null;
-    if (filename.startsWith('r2://')) {
-      worker = new R2Worker(this);
-    } else {
-      worker = new S3Worker(this);
-    }
-
-    await worker.remove({ filename });
-  } else {
-    fsp.unlink(filename);
-  }
-
-  return { removed: filename };
+    if (!filename)
+        throw new Error('filename is required');
+    if (typeof filename !== 'string')
+        throw new Error(`filename isn't a string:${JSON.stringify(filename)}`);
+    if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+        let worker = null;
+        if (filename.startsWith('r2://')) {
+            worker = new R2Worker(this);
+        }
+        else {
+            worker = new S3Worker(this);
+        }
+        await worker.remove({ filename });
+    }
+    else {
+        fsp.unlink(filename);
+    }
+    return { removed: filename };
 };
 Worker.prototype.remove.metadata = {
-  options: {
-    filename: {}
-  }
+    options: {
+        filename: {}
+    }
 };
-
 Worker.prototype.move = async function ({ filename, target, remove = true }) {
-  if (!target) throw new Error('target is required');
-  if (typeof target !== 'string') throw new Error(`target isn't a string:${JSON.stringify(target)}`);
-  if (target.startsWith('s3://') || target.startsWith('r2://')) {
-    if (
-      (target.startsWith('s3://') && filename.startsWith('r2://')) ||
-      (target.startsWith('r2://') && filename.startsWith('s3://'))
-    ) {
-      throw new Error('Cowardly not copying between services');
-    }
-
-    let worker = null;
-    if (target.startsWith('r2://')) {
-      worker = new R2Worker(this);
-    } else {
-      worker = new S3Worker(this);
-    }
-
-    if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
-      // We need to copy and delete
-      const output = await worker.copy({ filename, target });
-      if (remove) await worker.remove({ filename });
-      return output;
-    }
-    const parts = target.split('/');
-    return worker.put({ filename, directory: parts.slice(0, -1).join('/'), file: parts.slice(-1)[0] });
-  }
-  await fsp.mkdir(path.dirname(target), { recursive: true });
-  if (remove) {
-    try {
-      await fsp.rename(filename, target);
-    } catch (e) {
-      //it may be a filesystem issue moving between items
-      debug('Assuming this is a filesystem crosslink error, ignoring ', e.getMessage());
-      await fsp.copyFile(filename, target);
-      await fsp.unlink(filename);
-    }
-  } else {
-    await fsp.copyFile(filename, target);
-  }
-  return { filename: target };
+    if (!target)
+        throw new Error('target is required');
+    if (typeof target !== 'string')
+        throw new Error(`target isn't a string:${JSON.stringify(target)}`);
+    if (target.startsWith('s3://') || target.startsWith('r2://')) {
+        if ((target.startsWith('s3://') && filename.startsWith('r2://')) ||
+            (target.startsWith('r2://') && filename.startsWith('s3://'))) {
+            throw new Error('Cowardly not copying between services');
+        }
+        let worker = null;
+        if (target.startsWith('r2://')) {
+            worker = new R2Worker(this);
+        }
+        else {
+            worker = new S3Worker(this);
+        }
+        if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+            // We need to copy and delete
+            const output = await worker.copy({ filename, target });
+            if (remove)
+                await worker.remove({ filename });
+            return output;
+        }
+        const parts = target.split('/');
+        return worker.put({ filename, directory: parts.slice(0, -1).join('/'), file: parts.slice(-1)[0] });
+    }
+    await fsp.mkdir(path.dirname(target), { recursive: true });
+    if (remove) {
+        try {
+            await fsp.rename(filename, target);
+        }
+        catch (e) {
+            //it may be a filesystem issue moving between items
+            debug('Assuming this is a filesystem crosslink error, ignoring ', e.getMessage());
+            await fsp.copyFile(filename, target);
+            await fsp.unlink(filename);
+        }
+    }
+    else {
+        await fsp.copyFile(filename, target);
+    }
+    return { filename: target };
 };
 Worker.prototype.move.metadata = {
-  options: {
-    filename: {},
-    target: {}
-  }
+    options: {
+        filename: {},
+        target: {}
+    }
 };
-
 Worker.prototype.copy = async function (opts) {
-  return this.move({ ...opts, remove: false });
+    return this.move({ ...opts, remove: false });
 };
 Worker.prototype.copy.metadata = {
-  options: {
-    filename: {},
-    target: {}
-  }
+    options: {
+        filename: {},
+        target: {}
+    }
 };
-
 Worker.prototype.stat = async function ({ filename }) {
-  if (!filename) throw new Error('filename is required');
-  const output = {};
-
-  if (filename.slice(-8) === '.parquet') {
-    const pq = new ParquetWorker(this);
-    output.schema = (await pq.schema({ filename }))?.schema;
-    output.records = (await pq.meta({ filename }))?.records;
-  }
-
-  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
-    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    Object.assign(output, await worker.stat({ filename }));
-  } else {
-    const { ctime, birthtime, size } = await fsp.stat(filename);
-    const modifiedAt = new Date(ctime);
-    let createdAt = birthtime;
-    if (createdAt === 0 || !createdAt) createdAt = ctime;
-    createdAt = new Date(createdAt);
-    Object.assign(output, {
-      createdAt,
-      modifiedAt,
-      size
-    });
-  }
-  return output;
+    if (!filename)
+        throw new Error('filename is required');
+    const output = {};
+    if (filename.slice(-8) === '.parquet') {
+        const pq = new ParquetWorker(this);
+        output.schema = (await pq.schema({ filename }))?.schema;
+        output.records = (await pq.meta({ filename }))?.records;
+    }
+    if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+        const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
+        Object.assign(output, await worker.stat({ filename }));
+    }
+    else {
+        const { ctime, birthtime, size } = await fsp.stat(filename);
+        const modifiedAt = new Date(ctime);
+        let createdAt = birthtime;
+        if (createdAt === 0 || !createdAt)
+            createdAt = ctime;
+        createdAt = new Date(createdAt);
+        Object.assign(output, {
+            createdAt,
+            modifiedAt,
+            size
+        });
+    }
+    return output;
 };
 Worker.prototype.stat.metadata = {
-  options: {
-    filename: {}
-  }
+    options: {
+        filename: {}
+    }
 };
-
 Worker.prototype.download = async function ({ filename }) {
-  if (!filename) throw new Error('filename is required');
-  if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
-    const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    return worker.download({ filename });
-  }
-  throw new Error('Cannot download a local file');
+    if (!filename)
+        throw new Error('filename is required');
+    if (filename.startsWith('s3://') || filename.startsWith('r2://')) {
+        const worker = new (filename.startsWith('r2://') ? R2Worker : S3Worker)(this);
+        return worker.download({ filename });
+    }
+    throw new Error('Cannot download a local file');
 };
 Worker.prototype.download.metadata = {
-  options: {
-    filename: {}
-  }
+    options: {
+        filename: {}
+    }
 };
-
 Worker.prototype.head = async function (options) {
-  const limit = options.limit || 3;
-  const { stream } = await this.fileToObjectStream({ ...options, limit });
-  const chunks = [];
-
-  let counter = 0;
-
-  for await (const chunk of stream) {
-    chunks.push(chunk);
-    counter += 1;
-    if (counter >= limit) break;
-  }
-
-  return chunks;
+    const limit = options.limit || 3;
+    const { stream } = await this.fileToObjectStream({ ...options, limit });
+    const chunks = [];
+    let counter = 0;
+    for await (const chunk of stream) {
+        chunks.push(chunk);
+        counter += 1;
+        if (counter >= limit)
+            break;
+    }
+    return chunks;
 };
-
 Worker.prototype.head.metadata = {
-  options: {
-    filename: { required: true }
-  }
+    options: {
+        filename: { required: true }
+    }
 };
-
 Worker.prototype.columns = async function (options) {
-  const head = await this.head(options);
-  if (head.length == 0) {
+    const head = await this.head(options);
+    if (head.length == 0) {
+        return {
+            records: 0,
+            likelyHeaderLines: 0,
+            columns: []
+        };
+    }
+    let likelyHeaderLines = 1;
+    const columns = Object.keys(head[0]);
+    let s = columns.join(',');
+    if (s.match(/[()@#%!]/)) {
+        likelyHeaderLines = 0;
+    }
     return {
-      records: 0,
-      likelyHeaderLines: 0,
-      columns: []
+        likelyHeaderLines,
+        columns
     };
-  }
-
-  let likelyHeaderLines = 1;
-  const columns = Object.keys(head[0]);
-  let s = columns.join(',');
-  if (s.match(/[()@#%!]/)) {
-    likelyHeaderLines = 0;
-  }
-  return {
-    likelyHeaderLines,
-    columns
-  };
 };
-
 Worker.prototype.columns.metadata = {
-  options: {
-    filename: { required: true }
-  }
+    options: {
+        filename: { required: true }
+    }
 };
-
 Worker.prototype.count = async function (options) {
-  const { stream } = await this.fileToObjectStream(options);
-  const sample = [];
-
-  const limit = options.limit || 5;
-  let records = 0;
-
-  for await (const chunk of stream) {
-    records += 1;
-    if (records < limit) {
-      sample.push(chunk);
-    }
-  }
-
-  return { sample, records };
+    const { stream } = await this.fileToObjectStream(options);
+    const sample = [];
+    const limit = options.limit || 5;
+    let records = 0;
+    for await (const chunk of stream) {
+        records += 1;
+        if (records < limit) {
+            sample.push(chunk);
+        }
+    }
+    return { sample, records };
 };
-
 Worker.prototype.count.metadata = {
-  options: {
-    filename: { required: true }
-  }
+    options: {
+        filename: { required: true }
+    }
 };
-
 // Get a set of unique entries from a uniqueFunction
 // This could be large
 Worker.prototype.getUniqueSet = async function (options) {
-  const existingFiles = getStringArray(options.filenames);
-  const sample = {};
-
-  let { uniqueFunction } = options;
-  if (!uniqueFunction) {
-    uniqueFunction = (o) => JSON.stringify(o);
-  }
-  const uniqueSet = new Set();
-
-  for (const filename of existingFiles) {
-    const { stream: existsStream } = await this.fileToObjectStream({ filename });
-    await pipeline(
-      existsStream,
-      new Transform({
-        objectMode: true,
-        transform(d, enc, cb) {
-          const v = uniqueFunction(makeStrings(d)) || '';
-          if (uniqueSet.size < 3) {
-            sample[v] = d;
-          }
-          uniqueSet.add(v);
-          cb(null, d);
-        }
-      }),
-      new Writable({
-        objectMode: true,
-        write(d, enc, cb) {
-          cb();
-        }
-      })
-    );
-    debug(`Finished loading ${filename}`);
-  }
-  return { uniqueFunction, uniqueSet, sample };
+    const existingFiles = getStringArray(options.filenames);
+    const sample = {};
+    let { uniqueFunction } = options;
+    if (!uniqueFunction) {
+        uniqueFunction = (o) => JSON.stringify(o);
+    }
+    const uniqueSet = new Set();
+    for (const filename of existingFiles) {
+        const { stream: existsStream } = await this.fileToObjectStream({ filename });
+        await pipeline(existsStream, new Transform({
+            objectMode: true,
+            transform(d, enc, cb) {
+                const v = uniqueFunction(makeStrings(d)) || '';
+                if (uniqueSet.size < 3) {
+                    sample[v] = d;
+                }
+                uniqueSet.add(v);
+                cb(null, d);
+            }
+        }), new Writable({
+            objectMode: true,
+            write(d, enc, cb) {
+                cb();
+            }
+        }));
+        debug(`Finished loading ${filename}`);
+    }
+    return { uniqueFunction, uniqueSet, sample };
 };
-
 Worker.prototype.getUniqueStream = async function (options) {
-  const includeDuplicateSourceRecords = bool(options.includeDuplicateSourceRecords, false);
-
-  const { uniqueSet, uniqueFunction, sample } = await this.getUniqueSet({
-    filenames: options.existingFiles,
-    uniqueFunction: options.uniqueFunction
-  });
-
-  const { stream: inStream } = await this.fileToObjectStream(options);
-  const uniqueStream = inStream.pipe(
-    new Transform({
-      objectMode: true,
-      transform(d, enc, cb) {
-        const v = uniqueFunction(makeStrings(d)) || '';
-
-        if (!v) {
-          // falsey unique function includes
-          // by default
-          cb(null, d);
-        } else if (uniqueSet.has(v)) {
-          // do nothing
-          cb();
-        } else {
-          if (!includeDuplicateSourceRecords) {
-            // add it to the set for the next time
-            uniqueSet.add(v);
-          }
-          cb(null, d);
+    const includeDuplicateSourceRecords = bool(options.includeDuplicateSourceRecords, false);
+    const { uniqueSet, uniqueFunction, sample } = await this.getUniqueSet({
+        filenames: options.existingFiles,
+        uniqueFunction: options.uniqueFunction
+    });
+    const { stream: inStream } = await this.fileToObjectStream(options);
+    const uniqueStream = inStream.pipe(new Transform({
+        objectMode: true,
+        transform(d, enc, cb) {
+            const v = uniqueFunction(makeStrings(d)) || '';
+            if (!v) {
+                // falsey unique function includes
+                // by default
+                cb(null, d);
+            }
+            else if (uniqueSet.has(v)) {
+                // do nothing
+                cb();
+            }
+            else {
+                if (!includeDuplicateSourceRecords) {
+                    // add it to the set for the next time
+                    uniqueSet.add(v);
+                }
+                cb(null, d);
+            }
         }
-      }
-    })
-  );
-  return { stream: uniqueStream, sample };
+    }));
+    return { stream: uniqueStream, sample };
 };
-
 Worker.prototype.getUniqueStream.metadata = {
-  options: {
-    existingFiles: {},
-    uniqueFunction: {},
-    filename: { description: 'Specify a source filename or a stream' },
-    stream: { description: 'Specify a source filename or a stream' },
-    includeDuplicateSourceRecords: {
-      description: 'Sometimes you want the output to include source dupes, sometimes not, default false'
-    }
-  }
+    options: {
+        existingFiles: {},
+        uniqueFunction: {},
+        filename: { description: 'Specify a source filename or a stream' },
+        stream: { description: 'Specify a source filename or a stream' },
+        includeDuplicateSourceRecords: {
+            description: 'Sometimes you want the output to include source dupes, sometimes not, default false'
+        }
+    }
 };
 Worker.prototype.getUniqueFile = async function (options) {
-  const { stream, sample } = await this.getUniqueStream(options);
-  const { filename, records } = await this.objectStreamToFile({ stream });
-  return { filename, records, sample };
+    const { stream, sample } = await this.getUniqueStream(options);
+    const { filename, records } = await this.objectStreamToFile({ stream });
+    return { filename, records, sample };
 };
-
 Worker.prototype.getUniqueFile.metadata = {
-  options: {
-    existingFiles: {},
-    uniqueFunction: {},
-    filename: { description: 'Specify a source filename or a stream' },
-    stream: { description: 'Specify a source filename or a stream' },
-    includeDuplicateSourceRecords: {
-      description: 'Sometimes you want the output to include source dupes, sometimes not, default false'
-    }
-  }
+    options: {
+        existingFiles: {},
+        uniqueFunction: {},
+        filename: { description: 'Specify a source filename or a stream' },
+        stream: { description: 'Specify a source filename or a stream' },
+        includeDuplicateSourceRecords: {
+            description: 'Sometimes you want the output to include source dupes, sometimes not, default false'
+        }
+    }
 };
-
 /*
 diff that allows for unordered files, and doesn't store full objects in memory.
 Requires 2 passes of the files,
 but that's a better tradeoff than trying to store huge files in memory
 */
 Worker.prototype.diff = async function (options) {
-  const { fileA, fileB, uniqueFunction: ufOpt, columns, includeDuplicateSourceRecords } = options;
-  if (options.fields) throw new Error('fields is deprecated, use columns');
-
-  if (ufOpt && columns) throw new Error('fields and uniqueFunction cannot both be specified');
-  let uniqueFunction = ufOpt;
-  if (!uniqueFunction && columns) {
-    const farr = getStringArray(columns);
-    uniqueFunction = (o) => farr.map((f) => o[f] || '').join('.');
-  }
-
-  const left = await this.getUniqueFile({
-    existingFiles: [fileB],
-    filename: fileA,
-    uniqueFunction,
-    includeDuplicateSourceRecords
-  });
-  const right = await this.getUniqueFile({
-    existingFiles: [fileA],
-    filename: fileB,
-    uniqueFunction,
-    includeDuplicateSourceRecords
-  });
-
-  return {
-    left,
-    right
-  };
+    const { fileA, fileB, uniqueFunction: ufOpt, columns, includeDuplicateSourceRecords } = options;
+    if (options.fields)
+        throw new Error('fields is deprecated, use columns');
+    if (ufOpt && columns)
+        throw new Error('fields and uniqueFunction cannot both be specified');
+    let uniqueFunction = ufOpt;
+    if (!uniqueFunction && columns) {
+        const farr = getStringArray(columns);
+        uniqueFunction = (o) => farr.map((f) => o[f] || '').join('.');
+    }
+    const left = await this.getUniqueFile({
+        existingFiles: [fileB],
+        filename: fileA,
+        uniqueFunction,
+        includeDuplicateSourceRecords
+    });
+    const right = await this.getUniqueFile({
+        existingFiles: [fileA],
+        filename: fileB,
+        uniqueFunction,
+        includeDuplicateSourceRecords
+    });
+    return {
+        left,
+        right
+    };
 };
 Worker.prototype.diff.metadata = {
-  options: {
-    fileA: {},
-    fileB: {},
-    columns: { description: 'Columns to use for uniqueness -- aka primary key.  Defaults to JSON of line' },
-    uniqueFunction: {},
-    includeDuplicateSourceRecords: {
-      description: 'Sometimes you want the output to include source dupes, sometimes not, default false'
-    }
-  }
+    options: {
+        fileA: {},
+        fileB: {},
+        columns: { description: 'Columns to use for uniqueness -- aka primary key.  Defaults to JSON of line' },
+        uniqueFunction: {},
+        includeDuplicateSourceRecords: {
+            description: 'Sometimes you want the output to include source dupes, sometimes not, default false'
+        }
+    }
 };
-
-module.exports = Worker;
+export default Worker;