@engine9/input-tools 2.0.11 → 2.0.12

package/file/FileUtilities.js

@@ -11,7 +11,17 @@ import languageEncoding from 'detect-file-encoding-and-language';
 import R2Worker from './R2.js';
 import S3Worker from './S3.js';
 import ParquetWorker from './Parquet.js';
-import { bool, getTempFilename, getStringArray, getTempDir, getFilePostfix, makeStrings, streamPacket, relativeDate } from './tools.js';
+import {
+  bool,
+  getTempFilename,
+  getStringArray,
+  getTempDir,
+  getFilePostfix,
+  makeStrings,
+  normalizeListDepth,
+  streamPacket,
+  relativeDate
+} from './tools.js';
 const fsp = fs.promises;
 const { Readable, Transform, PassThrough, Writable } = nodestream;
 const { pipeline } = promises;
@@ -623,15 +633,42 @@ Worker.prototype.json.metadata = {
     filename: { description: 'Get a javascript object from a file' }
   }
 };
-Worker.prototype.list = async function ({ directory, start: s, end: e }) {
+Worker.prototype.list = async function ({ directory, start: s, end: e, depth: depthOpt }) {
   if (!directory) throw new Error('directory is required');
   let start = null;
   let end = null;
   if (s) start = relativeDate(s);
   if (e) end = relativeDate(e);
+  const maxDepth = normalizeListDepth(depthOpt);
   if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
     const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    return worker.list({ directory, start, end });
+    return worker.list({ directory, start, end, depth: maxDepth });
+  }
+  if (maxDepth) {
+    const baseDir = path.resolve(directory);
+    const withModified = [];
+    const walk = async (dir, relParts) => {
+      const entries = await fsp.readdir(dir, { withFileTypes: true });
+      for (const ent of entries) {
+        const fullPath = path.join(dir, ent.name);
+        const segCount = relParts.length + 1;
+        if (segCount > maxDepth) continue;
+        const stats = await fsp.stat(fullPath);
+        if (start && stats.mtime < start.getTime()) continue;
+        if (end && stats.mtime > end.getTime()) continue;
+        const name = relParts.length ? `${relParts.join('/')}/${ent.name}` : ent.name;
+        withModified.push({
+          name,
+          type: ent.isDirectory() ? 'directory' : 'file',
+          modifiedAt: new Date(stats.mtime).toISOString()
+        });
+        if (ent.isDirectory() && segCount < maxDepth) {
+          await walk(fullPath, [...relParts, ent.name]);
+        }
+      }
+    };
+    await walk(baseDir, []);
+    return withModified;
   }
   const a = await fsp.readdir(directory, { withFileTypes: true });
   const withModified = [];
@@ -654,7 +691,11 @@ Worker.prototype.list = async function ({ directory, start: s, end: e }) {
 };
 Worker.prototype.list.metadata = {
   options: {
-    directory: { required: true }
+    directory: { required: true },
+    depth: {
+      description:
+        'If set, recursively list files and directories up to this path depth (relative to directory); omit for a single-level listing only'
+    }
   }
 };
 Worker.prototype.analyzeDirectory = async function ({ directory }) {

package/file/S3.js

@@ -2,7 +2,7 @@ import debug$0 from 'debug';
 import fs from 'node:fs';
 import withDb from 'mime-type/with-db';
 import clientS3 from '@aws-sdk/client-s3';
-import { getTempFilename, getFilePostfix, relativeDate } from './tools.js';
+import { getTempFilename, getFilePostfix, normalizeListDepth, relativeDate } from './tools.js';
 const debug = debug$0('@engine9/input/S3');
 const { mimeType: mime } = withDb;
 const {
@@ -187,50 +187,120 @@ Worker.prototype.write.metadata = {
     content: { description: 'Contents of file' }
   }
 };
-Worker.prototype.list = async function ({ directory, start, end, raw }) {
+Worker.prototype.list = async function ({ directory, start, end, raw, depth: depthOpt }) {
   if (!directory) throw new Error('directory is required');
   let dir = directory;
   while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
-  const { Bucket, Key: Prefix } = getParts(dir);
+  const { Bucket, Key: rootPrefix } = getParts(dir);
   const s3Client = this.getClient();
-  const command = new ListObjectsV2Command({
-    Bucket,
-    Prefix: `${Prefix}/`,
-    Delimiter: '/'
-  });
-  const { Contents: files, CommonPrefixes } = await s3Client.send(command);
-  if (raw) return files;
-  // debug('Prefixes:', { CommonPrefixes });
-  const output = []
-    .concat(
-      (CommonPrefixes || []).map((f) => ({
-        name: f.Prefix.slice(Prefix.length + 1, -1),
-        type: 'directory'
-      }))
-    )
-    .concat(
-      (files || [])
-        .filter(({ LastModified }) => {
-          if (start && new Date(LastModified) < start) {
-            return false;
-          } else if (end && new Date(LastModified) > end) {
-            return false;
-          } else {
-            return true;
-          }
-        })
-        .map(({ Key, Size, LastModified }) => ({
-          name: Key.slice(Prefix.length + 1),
-          type: 'file',
-          size: Size,
-          modifiedAt: new Date(LastModified).toISOString()
+  const maxDepth = normalizeListDepth(depthOpt);
+
+  const relToRoot = (keyOrPrefix) => {
+    const normalized = keyOrPrefix.replace(/\/$/, '');
+    if (!rootPrefix) return normalized;
+    if (normalized.length <= rootPrefix.length) return '';
+    return normalized.slice(rootPrefix.length + 1);
+  };
+
+  if (!maxDepth) {
+    const Prefix = rootPrefix;
+    const command = new ListObjectsV2Command({
+      Bucket,
+      Prefix: `${Prefix}/`,
+      Delimiter: '/'
+    });
+    const { Contents: files, CommonPrefixes } = await s3Client.send(command);
+    if (raw) return files;
+    const output = []
+      .concat(
+        (CommonPrefixes || []).map((f) => ({
+          name: f.Prefix.slice(Prefix.length + 1, -1),
+          type: 'directory'
         }))
-    );
+      )
+      .concat(
+        (files || [])
+          .filter(({ LastModified }) => {
+            if (start && new Date(LastModified) < start) {
+              return false;
+            } else if (end && new Date(LastModified) > end) {
+              return false;
+            } else {
+              return true;
+            }
+          })
+          .map(({ Key, Size, LastModified }) => ({
+            name: Key.slice(Prefix.length + 1),
+            type: 'file',
+            size: Size,
+            modifiedAt: new Date(LastModified).toISOString()
+          }))
+      );
+    return output;
+  }
+
+  if (raw) {
+    throw new Error('list raw output is not supported together with depth');
+  }
+
+  const output = [];
+
+  async function listLevel(currentPrefix) {
+    const prefixParam = currentPrefix === '' ? '' : `${currentPrefix}/`;
+    let ContinuationToken = undefined;
+    const allPrefixes = [];
+    const allFiles = [];
+    do {
+      const result = await s3Client.send(
+        new ListObjectsV2Command({
+          Bucket,
+          Prefix: prefixParam,
+          Delimiter: '/',
+          ContinuationToken
+        })
+      );
+      allPrefixes.push(...(result.CommonPrefixes || []));
+      allFiles.push(...(result.Contents || []));
+      ContinuationToken = result.IsTruncated ? result.NextContinuationToken : undefined;
+    } while (ContinuationToken);
+
+    for (const cp of allPrefixes) {
+      const subPrefix = cp.Prefix.replace(/\/$/, '');
+      const rel = relToRoot(cp.Prefix);
+      if (!rel) continue;
+      const segCount = rel.split('/').length;
+      if (segCount > maxDepth) continue;
+      output.push({ name: rel, type: 'directory' });
+      if (segCount < maxDepth) {
+        await listLevel(subPrefix);
+      }
+    }
+    for (const obj of allFiles) {
+      const { Key, Size, LastModified } = obj;
+      const rel = relToRoot(Key);
+      if (!rel) continue;
+      if (rel.split('/').length > maxDepth) continue;
+      if (start && new Date(LastModified) < start) continue;
+      if (end && new Date(LastModified) > end) continue;
+      output.push({
+        name: rel,
+        type: 'file',
+        size: Size,
+        modifiedAt: new Date(LastModified).toISOString()
+      });
+    }
+  }
+
+  await listLevel(rootPrefix);
   return output;
 };
 Worker.prototype.list.metadata = {
   options: {
-    directory: { required: true }
+    directory: { required: true },
+    depth: {
+      description:
+        'If set, recursively list objects and prefixes up to this key depth (relative to directory); omit for a single-level listing only'
+    }
   }
 };
 Worker.prototype.analyzeDirectory = async function ({ directory }) {

package/file/tools.js

@@ -335,6 +335,15 @@ function parseJSON5(o, defaultVal) {
   }
   return defaultVal || o;
 }
+/** @returns {number|undefined} positive integer max depth, or undefined if absent */
+function normalizeListDepth(depth) {
+  if (depth === undefined || depth === null || depth === '') return undefined;
+  const n = typeof depth === 'number' ? depth : Number.parseInt(String(depth), 10);
+  if (!Number.isFinite(n) || n < 1 || Math.floor(n) !== n) {
+    throw new Error('depth must be a positive integer');
+  }
+  return n;
+}
 export { appendPostfix };
 export { bool };
 export { downloadFile };
@@ -348,6 +357,7 @@ export { getManifest };
 export { getPacketFiles };
 export { getStringArray };
 export { makeStrings };
+export { normalizeListDepth };
 export { parseJSON5 };
 export { relativeDate };
 export { streamPacket };
@@ -366,6 +376,7 @@ export default {
   getPacketFiles,
   getStringArray,
   makeStrings,
+  normalizeListDepth,
   parseJSON5,
   relativeDate,
   streamPacket,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9/input-tools",
-  "version": "2.0.11",
+  "version": "2.0.12",
   "type": "module",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",