@engine9-io/input-tools 1.3.6

package/file/GoogleDrive.js

@@ -0,0 +1,45 @@
+const { google } = require('googleapis');
+const fs = require('node:fs');
+
+const fsp = fs.promises;
+
+function Worker() {}
+
+Worker.prototype.setAuth = async function () {
+  const keyFile = process.env.GOOGLE_APPLICATION_CREDENTIALS;
+  const settings = JSON.parse(await fsp.readFile(keyFile));
+  if (!settings.subject_to_impersonate) throw new Error(`You should include subject_to_impersonate in file ${keyFile}`);
+
+  const auth = new google.auth.GoogleAuth({
+    clientOptions: {
+      subject: settings.subject_to_impersonate,
+    },
+    keyFile,
+    scopes: ['https://www.googleapis.com/auth/drive'],
+  });
+  google.options({
+    auth,
+  });
+};
+
+Worker.prototype.list = async function ({ path }) {
+  await this.setAuth();
+  const drive = google.drive({ version: 'v3' });
+  const folderId = path;
+  const q = `'${folderId}' in parents and trashed=false`;
+  const raw = await drive.files.list({
+    pageSize: 150,
+    q,
+    supportsAllDrives: true, // include share drives as well
+    includeItemsFromAllDrives: true,
+  });
+
+  return raw.data?.files;
+};
+Worker.prototype.list.metadata = {
+  options: {
+    path: {},
+  },
+};
+
+module.exports = Worker;

package/file/Parquet.js

@@ -0,0 +1,138 @@
+const parquet = require('@dsnp/parquetjs');
+
+const { Readable } = require('node:stream');
+const debug = require('debug')('ParquetWorker');
+const { S3Client } = require('@aws-sdk/client-s3');
+const FileWorker = require('./FileUtilities');
+
+function Worker() {}
+
+async function getReader(options) {
+  const { filename } = options;
+  if (!filename) throw new Error('filename is required');
+  if (filename.indexOf('s3://') === 0) {
+    const client = new S3Client({});
+    const parts = filename.split('/');
+
+    return parquet.ParquetReader.openS3(client, {
+      Bucket: parts[2],
+      Key: parts.slice(3).join('/'),
+    });
+  }
+  return parquet.ParquetReader.openFile(filename);
+}
+
+Worker.prototype.meta = async function (options) {
+  const reader = await getReader(options);
+  return {
+    records: String(reader.metadata?.num_rows),
+  };
+  // getMetadata();
+};
+Worker.prototype.meta.metadata = {
+  options: {
+    path: {},
+  },
+};
+Worker.prototype.schema = async function (options) {
+  const reader = await getReader(options);
+  return reader.getSchema();
+};
+Worker.prototype.schema.metadata = {
+  options: {
+    path: {},
+  },
+};
+
+function cleanColumnName(name) {
+  name.toLowerCase().replace(/[^a-z0-9_]/g, '_');
+}
+
+Worker.prototype.stream = async function (options) {
+  const stream = new Readable({ objectMode: true });
+
+  const reader = await getReader(options);
+  let columns;
+  if (options.columns) {
+    const { fieldList } = await this.schema(options);
+    columns = [];
+    let requestedColumns = options.columns;
+    if (typeof options.columns === 'string') requestedColumns = options.columns.split(',').map((d) => d.trim());
+    else requestedColumns = options.columns.map((d) => (d.name ? d.name.trim() : d.trim()));
+    requestedColumns.forEach((c) => {
+      columns = columns.concat(
+        fieldList.filter((f) => (
+          f.name === c || cleanColumnName(f.name) === cleanColumnName(c)
+        )).map((f) => f.name),
+      );
+    });
+  }
+  let limit = 0;
+  if (parseInt(options.limit, 10) === options.limit) limit = parseInt(options.limit, 10);
+  // create a new cursor
+  debug(`Reading parquet file ${options.filename} with columns ${columns?.join(',')} and limit ${limit}`);
+  const cursor = reader.getCursor(columns);
+
+  // read all records from the file and print them
+  let record = null;
+  let counter = 0;
+
+  const start = new Date().getTime();
+  do {
+    // eslint-disable-next-line no-await-in-loop
+    record = await cursor.next();
+    counter += 1;
+    if (limit && counter > limit) {
+      debug(`Reached limit of ${limit}, stopping`);
+      break;
+    }
+    if (counter % 5000 === 0) {
+      const end = new Date().getTime();
+      debug(`Read ${counter} ${(counter * 1000) / (end - start)}/sec `);
+    }
+    stream.push(record);
+  } while (record);
+  stream.push(null);
+  await reader.close();
+
+  return { stream };
+};
+
+Worker.prototype.stream.metadata = {
+  options: {
+    path: {},
+  },
+};
+
+Worker.prototype.toFile = async function (options) {
+  const { stream } = await this.stream(options);
+  const fworker = new FileWorker(this);
+  return fworker.objectStreamToFile({ ...options, stream });
+};
+Worker.prototype.toFile.metadata = {
+  options: {
+    path: {},
+  },
+};
+
+Worker.prototype.stats = async function (options) {
+  const reader = await getReader(options);
+  const schema = reader.getSchema();
+  const fileMetadata = reader.getFileMetaData();
+  const rowGroups = reader.getRowGroups();
+
+  // const reader = await parquet.ParquetReader.openS3(client, getParams(options));
+  // return reader.getSchema();
+  return {
+    schema,
+    fileMetadata,
+    rowGroups,
+  };
+};
+Worker.prototype.stats.metadata = {
+  options: {
+    path: {},
+  },
+};
+
+module.exports = Worker;

package/file/S3.js

@@ -0,0 +1,246 @@
+const debug = require('debug')('S3Worker');
+const fs = require('node:fs');
+// eslint-disable-next-line import/no-unresolved
+const { mimeType: mime } = require('mime-type/with-db');
+const {
+  S3Client,
+  GetObjectCommand,
+  HeadObjectCommand,
+  GetObjectAttributesCommand, PutObjectCommand,
+  ListObjectsV2Command,
+} = require('@aws-sdk/client-s3');
+const { getTempFilename } = require('./tools');
+
+function Worker() {}
+
+function getParts(filename) {
+  if (!filename || filename.indexOf('s3://') !== 0) throw new Error(`Invalid filename for s3:${filename}`);
+  const parts = filename.split('/');
+  const Bucket = parts[2];
+  const Key = parts.slice(3).join('/');
+  return { Bucket, Key };
+}
+Worker.prototype.getClient = function () {
+  if (!this.client) this.client = new S3Client({});
+  return this.client;
+};
+
+Worker.prototype.getMetadata = async function ({ filename }) {
+  const s3Client = this.getClient();
+  const { Bucket, Key } = getParts(filename);
+
+  const resp = await s3Client.send(new GetObjectAttributesCommand({
+    Bucket,
+    Key,
+    ObjectAttributes: ['ETag', 'Checksum', 'ObjectParts', 'StorageClass', 'ObjectSize'],
+  }));
+
+  return resp;
+};
+Worker.prototype.getMetadata.metadata = {
+  options: {
+    filename: {},
+  },
+};
+
+Worker.prototype.stream = async function ({ filename }) {
+  const s3Client = new S3Client({});
+  const { Bucket, Key } = getParts(filename);
+  const command = new GetObjectCommand({ Bucket, Key });
+  try {
+    debug(`Streaming file ${Key}`);
+    const response = await s3Client.send(command);
+    return { stream: response.Body };
+  } catch (e) {
+    debug(`Could not stream filename:${filename}`);
+    throw e;
+  }
+};
+Worker.prototype.stream.metadata = {
+  options: {
+    filename: {},
+  },
+};
+
+Worker.prototype.download = async function ({ filename }) {
+  const file = filename.split('/').pop();
+  const localPath = await getTempFilename({ targetFilename: file });
+  const s3Client = new S3Client({});
+  const { Bucket, Key } = getParts(filename);
+  const command = new GetObjectCommand({ Bucket, Key });
+  debug(`Downloading ${file} to ${localPath}`);
+  const response = await s3Client.send(command);
+  const fileStream = fs.createWriteStream(localPath);
+
+  response.Body.pipe(fileStream);
+
+  return new Promise((resolve, reject) => {
+    fileStream.on('finish', async () => {
+      const { size } = await fs.promises.stat(localPath);
+      resolve({ size, filename: localPath });
+    });
+    fileStream.on('error', reject);
+  });
+};
+Worker.prototype.download.metadata = {
+  options: {
+    filename: {},
+  },
+};
+
+Worker.prototype.put = async function (options) {
+  const { filename, directory } = options;
+  if (!filename) throw new Error('Local filename required');
+  if (directory?.indexOf('s3://') !== 0) throw new Error(`directory path must start with s3://, is ${directory}`);
+
+  const file = options.file || filename.split('/').pop();
+  const parts = directory.split('/');
+  const Bucket = parts[2];
+  const Key = parts.slice(3).filter(Boolean).concat(file).join('/');
+  const Body = fs.createReadStream(filename);
+
+  const ContentType = mime.lookup(file);
+
+  debug(`Putting ${filename} to ${JSON.stringify({ Bucket, Key, ContentType })}}`);
+  const s3Client = new S3Client({});
+
+  const command = new PutObjectCommand({
+    Bucket, Key, Body, ContentType,
+  });
+
+  return s3Client.send(command);
+};
+Worker.prototype.put.metadata = {
+  options: {
+    filename: {},
+    directory: { description: 'Directory to put file, e.g. s3://foo-bar/dir/xyz' },
+    file: { description: 'Name of file, defaults to the filename' },
+  },
+};
+
+Worker.prototype.write = async function (options) {
+  const { directory, file, content } = options;
+
+  if (!directory?.indexOf('s3://') === 0) throw new Error('directory must start with s3://');
+  const parts = directory.split('/');
+
+  const Bucket = parts[2];
+  const Key = parts.slice(3).filter(Boolean).concat(file).join('/');
+  const Body = content;
+
+  debug(`Writing content of length ${content.length} to ${JSON.stringify({ Bucket, Key })}}`);
+  const s3Client = new S3Client({});
+  const ContentType = mime.lookup(file);
+
+  const command = new PutObjectCommand({
+    Bucket, Key, Body, ContentType,
+  });
+
+  return s3Client.send(command);
+};
+Worker.prototype.write.metadata = {
+  options: {
+    directory: { description: 'Directory to put file, e.g. s3://foo-bar/dir/xyz' },
+    file: { description: 'Name of file, defaults to the filename' },
+    content: { description: 'Contents of file' },
+  },
+};
+
+Worker.prototype.list = async function ({ directory }) {
+  if (!directory) throw new Error('directory is required');
+  let dir = directory;
+  while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
+  const { Bucket, Key: Prefix } = getParts(dir);
+  const s3Client = new S3Client({});
+  const command = new ListObjectsV2Command({
+    Bucket,
+    Prefix: `${Prefix}/`,
+    Delimiter: '/',
+  });
+
+  const { Contents: files, CommonPrefixes } = await s3Client.send(command);
+  // debug('Prefixes:', { CommonPrefixes });
+  const output = [].concat((CommonPrefixes || []).map((f) => ({
+    name: f.Prefix.slice(Prefix.length + 1, -1),
+    type: 'directory',
+  })))
+    .concat((files || []).map(({ Key }) => ({
+      name: Key.slice(Prefix.length + 1),
+      type: 'file',
+    })));
+
+  return output;
+};
+Worker.prototype.list.metadata = {
+  options: {
+    directory: { required: true },
+  },
+};
+/* List everything with the prefix */
+Worker.prototype.listAll = async function ({ directory }) {
+  if (!directory) throw new Error('directory is required');
+  let dir = directory;
+  while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
+  const { Bucket, Key: Prefix } = getParts(dir);
+  const s3Client = new S3Client({});
+  const files = [];
+  let ContinuationToken = null;
+  do {
+    const command = new ListObjectsV2Command({
+      Bucket,
+      Prefix: `${Prefix}/`,
+      ContinuationToken,
+      // Delimiter: '/',
+    });
+    debug(`Sending command with ContinuationToken ${ContinuationToken}`);
+    // eslint-disable-next-line no-await-in-loop
+    const result = await s3Client.send(command);
+    const newFiles = (result.Contents?.map((d) => `s3://${Bucket}/${d.Key}`) || []);
+    debug(`Retrieved ${newFiles.length} new files, total ${files.length},sample ${newFiles.slice(0, 3).join(',')}`);
+    files.push(...newFiles);
+    ContinuationToken = result.NextContinuationToken;
+  } while (ContinuationToken);
+  return files;
+};
+Worker.prototype.listAll.metadata = {
+  options: {
+    directory: { required: true },
+  },
+};
+
+Worker.prototype.stat = async function ({ filename }) {
+  if (!filename) throw new Error('filename is required');
+
+  const s3Client = new S3Client({});
+  const { Bucket, Key } = getParts(filename);
+  const command = new HeadObjectCommand({ Bucket, Key });
+  const response = await s3Client.send(command);
+
+  const {
+    // "AcceptRanges": "bytes",
+    ContentLength, // : "3191",
+    ContentType, // : "image/jpeg",
+    // ETag": "\"6805f2cfc46c0f04559748bb039d69ae\"",
+    LastModified, // : "2016-12-15T01:19:41.000Z",
+    // Metadata": {},
+    // VersionId": "null"
+
+  } = response;
+  const modifiedAt = new Date(LastModified);
+  const createdAt = modifiedAt;// Same for S3
+  const size = parseInt(ContentLength, 10);
+
+  return {
+    createdAt,
+    modifiedAt,
+    contentType: ContentType,
+    size,
+  };
+};
+Worker.prototype.stat.metadata = {
+  options: {
+    filename: {},
+  },
+};
+
+module.exports = Worker;

package/file/tools.js

@@ -0,0 +1,237 @@
+const fs = require('node:fs');
+
+const fsp = fs.promises;
+const path = require('node:path');
+const debug = require('debug')('@engine9/input-tools');
+const os = require('node:os');
+const { mkdirp } = require('mkdirp');
+const { Transform } = require('node:stream');
+
+const JSON5 = require('json5');
+const { PassThrough } = require('node:stream');
+const progress = require('debug')('info:@engine9/input-tools');
+const unzipper = require('unzipper');
+
+const {
+  v7: uuidv7,
+} = require('uuid');
+
+async function getTempDir({ accountId = 'engine9' }) {
+  const dir = [os.tmpdir(), accountId, new Date().toISOString().substring(0, 10)].join(path.sep);
+  try {
+    await mkdirp(dir);
+  } catch (err) {
+    if (err.code !== 'EEXIST') throw err;
+  }
+  return dir;
+}
+
+/*
+ Get a new, timestamp based filename, creating any necessary directories
+ options:
+ prefix/postfix of file
+ source:source file, used to generate friendly name
+*/
+async function getTempFilename(options) {
+  let dir = await getTempDir(options);
+
+  const target = options.targetFilename;
+  if (target) {
+    if (target.indexOf('/') === 0 || target.indexOf('\\') === 0) {
+      // assume a full directory path has been specified
+      return target;
+    }
+
+    // make a distinct directory, so we don't overwrite the file
+    dir = `${dir}/${new Date().toISOString().slice(0, -6).replace(/[^0-9]/g, '_')}`;
+
+    const newDir = await mkdirp(dir);
+
+    return `${newDir}/${target}`;
+  }
+  let { prefix } = options;
+  let { postfix } = options;
+  const { targetFormat } = options;
+  if (!postfix && targetFormat === 'csv') postfix = '.csv';
+  if (options.source) {
+    postfix = `_${options.source.split('/').pop()}`;
+    postfix = postfix.replace(/['"\\]/g, '').replace(/[^a-zA-Z0-9_.-]/g, '_');
+  }
+
+  if (prefix) prefix += '_';
+
+  const p = `${dir}/${prefix || ''}${uuidv7()}${postfix || '.txt'}`;
+  return p;
+}
+
+const {
+  S3Client,
+  HeadObjectCommand,
+  GetObjectCommand,
+} = require('@aws-sdk/client-s3');
+
+async function getPacketFiles({ packet }) {
+  if (packet.indexOf('s3://') === 0) {
+    const parts = packet.split('/');
+    const Bucket = parts[2];
+    const Key = parts.slice(3).join('/');
+    const s3Client = new S3Client({});
+
+    debug('Getting ', { Bucket, Key });
+
+    //    const directory = await unzipper.Open.s3(s3Client, { Bucket, Key });
+    let size = null;
+    const directory = await unzipper.Open.custom({
+      async size() {
+        const info = await s3Client.send(
+          new HeadObjectCommand({
+            Bucket,
+            Key,
+          }),
+        );
+        size = info.ContentLength;
+        progress(`Retrieving file of size ${size / (1024 * 1024)} MB`);
+        return info.ContentLength;
+      },
+
+      stream(offset, length) {
+        const ptStream = new PassThrough();
+        s3Client.send(
+          new GetObjectCommand({
+            Bucket,
+            Key,
+            Range: `bytes=${offset}-${length ?? ''}`,
+          }),
+        )
+          .then((response) => {
+            response.Body.pipe(ptStream);
+          })
+          .catch((error) => {
+            ptStream.emit('error', error);
+          });
+
+        return ptStream;
+      },
+    });
+
+    return directory;
+  }
+  const directory = await unzipper.Open.file(packet);
+  return directory;
+}
+
+async function getManifest({ packet }) {
+  if (!packet) throw new Error('no packet option specififed');
+  const { files } = await getPacketFiles({ packet });
+  const file = files.find((d) => d.path === 'manifest.json');
+  const content = await file.buffer();
+  const manifest = JSON.parse(content.toString());
+  return manifest;
+}
+
+function getBatchTransform({ batchSize = 100 }) {
+  return {
+    transform: new Transform({
+      objectMode: true,
+      transform(chunk, encoding, cb) {
+        this.buffer = (this.buffer || []).concat(chunk);
+        if (this.buffer.length >= batchSize) {
+          this.push(this.buffer);
+          this.buffer = [];
+        }
+        cb();
+      },
+      flush(cb) {
+        if (this.buffer?.length > 0) this.push(this.buffer);
+        cb();
+      },
+    }),
+  };
+}
+function getDebatchTransform() {
+  return {
+    transform: new Transform({
+      objectMode: true,
+      transform(chunk, encoding, cb) {
+        chunk.forEach((c) => this.push(c));
+        cb();
+      },
+    }),
+  };
+}
+
+async function getFile({ filename, packet, type }) {
+  if (!packet && !filename) throw new Error('no packet option specififed');
+  let content = null;
+  let filePath = null;
+  if (packet) {
+    const manifest = await getManifest({ packet });
+    const manifestFiles = manifest.files?.filter((d) => d.type === type);
+    if (!manifestFiles?.length) throw new Error(`No files of type ${type} found in packet`);
+    if (manifestFiles?.length > 1) throw new Error(`Multiple files of type ${type} found in packet`);
+    filePath = manifestFiles[0].path;
+    const { files } = await getPacketFiles({ packet });
+    const handle = files.find((d) => d.path === filePath);
+    const buffer = await handle.buffer();
+    content = await buffer.toString();
+  } else {
+    content = await fsp.readFile(filename);
+    filePath = filename.split('/').pop();
+  }
+  if (filePath.slice(-5) === '.json' || filePath.slice(-6) === '.json5') {
+    try {
+      return JSON5.parse(content);
+    } catch (e) {
+      debug(`Erroring parsing json content from ${path}`, content);
+      throw e;
+    }
+  }
+  return content;
+}
+
+async function streamPacket({ packet, type }) {
+  if (!packet) throw new Error('no packet option specififed');
+  const manifest = await getManifest({ packet });
+  const manifestFiles = manifest.files?.filter((d) => d.type === type);
+  if (!manifestFiles?.length) throw new Error(`No files of type ${type} found in packet`);
+  if (manifestFiles?.length > 1) throw new Error(`Multiple files of type ${type} found in packet`);
+  const filePath = manifestFiles[0].path;
+  const { files } = await getPacketFiles({ packet });
+  const handle = files.find((d) => d.path === filePath);
+  return { stream: handle.stream(), path: filePath };
+}
+
+async function downloadFile({ packet, type = 'person' }) {
+  const { stream: fileStream, path: filePath } = await streamPacket({ packet, type });
+  const filename = await getTempFilename({ targetFilename: filePath.split('/').pop() });
+
+  return new Promise((resolve, reject) => {
+    fileStream.pipe(fs.createWriteStream(filename))
+      .on('error', reject)
+      .on('finish', () => {
+        resolve({ filename });
+      });
+  });
+}
+
+function bool(x, _defaultVal) {
+  const defaultVal = (_defaultVal === undefined) ? false : _defaultVal;
+  if (x === undefined || x === null || x === '') return defaultVal;
+  if (typeof x !== 'string') return !!x;
+  if (x === '1') return true; // 0 will return false, but '1' is true
+  const y = x.toLowerCase();
+  return !!(y.indexOf('y') + 1) || !!(y.indexOf('t') + 1);
+}
+
+module.exports = {
+  bool,
+  getTempFilename,
+  downloadFile,
+  getBatchTransform,
+  getDebatchTransform,
+  getFile,
+  getManifest,
+  getPacketFiles,
+  getTempDir,
+  streamPacket,
+};