@engine9/input-tools 2.0.9 → 2.0.10

package/file/FileUtilities.js

@@ -11,7 +11,7 @@ import languageEncoding from 'detect-file-encoding-and-language';
 import R2Worker from './R2.js';
 import S3Worker from './S3.js';
 import ParquetWorker from './Parquet.js';
-import { bool, getTempFilename, getStringArray, getTempDir, makeStrings, streamPacket, relativeDate } from './tools.js';
+import { bool, getTempFilename, getStringArray, getTempDir, getFilePostfix, makeStrings, streamPacket, relativeDate } from './tools.js';
 const fsp = fs.promises;
 const { Readable, Transform, PassThrough, Writable } = nodestream;
 const { pipeline } = promises;
@@ -669,6 +669,7 @@ Worker.prototype.analyze = async function ({ directory }) {
   let lastModified = null;
   let firstTime = null;
   let lastTime = null;
+  const postfixCounts = Object.create(null);
   const walk = async (dir) => {
     const entries = await fsp.readdir(dir, { withFileTypes: true });
     for (const ent of entries) {
@@ -678,6 +679,8 @@ Worker.prototype.analyze = async function ({ directory }) {
         await walk(fullPath);
       } else {
         fileCount += 1;
+        const postfix = getFilePostfix(fullPath);
+        postfixCounts[postfix] = (postfixCounts[postfix] || 0) + 1;
         const stats = await fsp.stat(fullPath);
         const mtime = stats.mtimeMs;
         const modifiedAt = new Date(stats.mtime).toISOString();
@@ -696,6 +699,7 @@ Worker.prototype.analyze = async function ({ directory }) {
   return {
     fileCount,
     directoryCount,
+    postfixCounts,
     firstModified: fileCount ? firstModified : null,
     lastModified: fileCount ? lastModified : null
   };

package/file/S3.js

@@ -2,7 +2,7 @@ import debug$0 from 'debug';
 import fs from 'node:fs';
 import withDb from 'mime-type/with-db';
 import clientS3 from '@aws-sdk/client-s3';
-import { getTempFilename, relativeDate } from './tools.js';
+import { getTempFilename, getFilePostfix, relativeDate } from './tools.js';
 const debug = debug$0('@engine9/input/S3');
 const { mimeType: mime } = withDb;
 const {
@@ -247,6 +247,7 @@ Worker.prototype.analyze = async function ({ directory }) {
   let lastModified = null;
   let firstTime = null;
   let lastTime = null;
+  const postfixCounts = Object.create(null);
   let ContinuationToken = undefined;
   do {
     const result = await s3Client.send(
@@ -270,6 +271,8 @@ Worker.prototype.analyze = async function ({ directory }) {
         continue;
       }
       fileCount++;
+      const postfix = getFilePostfix(objectKey);
+      postfixCounts[postfix] = (postfixCounts[postfix] || 0) + 1;
       const mtime = new Date(content.LastModified).getTime();
       const modifiedAt = new Date(content.LastModified).toISOString();
       const filename = `${this.prefix}://${Bucket}/${objectKey}`;
@@ -287,6 +290,7 @@ Worker.prototype.analyze = async function ({ directory }) {
   return {
     fileCount,
     directoryCount: dirsSeen.size,
+    postfixCounts,
     firstModified: fileCount ? firstModified : null,
     lastModified: fileCount ? lastModified : null
   };

package/file/tools.js

@@ -300,6 +300,18 @@ function makeStrings(o) {
     return a;
   }, {});
 }
+/** Basename postfix with leading dot, e.g. `.txt`, `.csv.gz`; empty string if none. */
+function getFilePostfix(filename) {
+  const base = path.basename(filename).toLowerCase();
+  if (!base || !base.includes('.')) return '';
+  if (base.endsWith('.gz')) {
+    const withoutGz = base.slice(0, -3);
+    const i = withoutGz.lastIndexOf('.');
+    if (i >= 0) return `${withoutGz.slice(i)}.gz`;
+    return '.gz';
+  }
+  return base.slice(base.lastIndexOf('.'));
+}
 function appendPostfix(filename, postfix) {
   const filenameParts = filename.split('/');
   const fileParts = filenameParts
@@ -331,6 +343,7 @@ export { getTempDir };
 export { getBatchTransform };
 export { getDebatchTransform };
 export { getFile };
+export { getFilePostfix };
 export { getManifest };
 export { getPacketFiles };
 export { getStringArray };
@@ -348,6 +361,7 @@ export default {
   getBatchTransform,
   getDebatchTransform,
   getFile,
+  getFilePostfix,
   getManifest,
   getPacketFiles,
   getStringArray,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9/input-tools",
-  "version": "2.0.9",
+  "version": "2.0.10",
   "type": "module",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",