@engine9-io/input-tools 1.7.5 → 1.7.7

package/ForEachEntry.js

@@ -172,7 +172,7 @@ class ForEachEntry {
           batches += 1;
           records += batch?.length || 0;
 
-          debugThrottle(`Processed ${batches} batches for a total of ${records} records`);
+          debugThrottle(`Processed ${batches} batches for a total of ${records} outbound records`);
           cb();
         },
       }),

package/eslint.config.mjs

@@ -0,0 +1,13 @@
+import js from "@eslint/js";
+import globals from "globals";
+import { defineConfig } from "eslint/config";
+
+export default defineConfig([
+  { files: ["**/*.{js,mjs,cjs}"], plugins: { js }, extends: ["js/recommended"], languageOptions: {
+     globals: {
+      ...globals.node, // This includes 'process' and other Node.js globals
+      // globals.browser
+     }
+  } },
+  { files: ["**/*.js"], languageOptions: { sourceType: "commonjs" } },
+]);

package/file/FileUtilities.js

@@ -20,7 +20,7 @@ const S3Worker = require('./S3');
 const ParquetWorker = require('./Parquet');
 
 const {
-  bool, getStringArray, getTempDir, makeStrings, streamPacket,
+  bool, getStringArray, getTempDir, makeStrings, streamPacket,relativeDate
 } = require('./tools');
 
 function Worker({ accountId }) { this.accountId = accountId; }
@@ -153,7 +153,7 @@ Worker.prototype.detectEncoding.metadata = {
 Internal method to transform a file into a stream of objects.
 */
 Worker.prototype.fileToObjectStream = async function (options) {
-  const { filename, columns, limit: limitOption } = options;
+  const { filename, columns, limit: limitOption,format:formatOverride } = options;
 
   // handle stream item
   if (options.stream) {
@@ -203,14 +203,15 @@ Worker.prototype.fileToObjectStream = async function (options) {
   } else {
     stream.setEncoding(encoding);
   }
+  let format=formatOverride || postfix;
 
-  if (postfix === 'csv') {
+  if (format === 'csv') {
     const csvTransforms = this.csvToObjectTransforms({ ...options });
     transforms = transforms.concat(csvTransforms.transforms);
-  } else if (postfix === 'txt') {
+  } else if (format === 'txt') {
     const csvTransforms = this.csvToObjectTransforms({ ...options, delimiter: '\t' });
     transforms = transforms.concat(csvTransforms.transforms);
-  } else if (postfix === 'jsonl') {
+  } else if (format === 'jsonl') {
     /* Type of JSON that has the names in an array in the first record,
     and the values in JSON arrays thereafter
     */
@@ -609,17 +610,38 @@ Worker.prototype.json.metadata = {
   },
 };
 
-Worker.prototype.list = async function ({ directory }) {
+Worker.prototype.list = async function ({ directory, start:s, end:e }) {
   if (!directory) throw new Error('directory is required');
+  let start=null;
+  let end=null;
+  if (s) start=relativeDate(s);
+  if (e) end=relativeDate(e);
+  
   if (directory.startsWith('s3://') || directory.startsWith('r2://')) {
     const worker = new (directory.startsWith('r2://') ? R2Worker : S3Worker)(this);
-    return worker.list({ directory });
+    return worker.list({ directory, start, end });
   }
   const a = await fsp.readdir(directory, { withFileTypes: true });
-  return a.map((f) => ({
-    name: f.name,
-    type: f.isDirectory() ? 'directory' : 'file',
-  }));
+
+  const withModified=[];
+  for (const file of a) {
+      const fullPath = path.join(directory, file.name);
+      const stats = await fsp.stat(fullPath);
+      if (start && stats.mtime<start.getTime()){
+        //do not include
+      }else if (end && stats.mtime>end.getTime()){
+        //do nothing
+      }else{
+          withModified.push({
+            name:file.name,
+            type: file.isDirectory() ? 'directory' : 'file',
+            modifiedAt:new Date(stats.mtime).toISOString(),
+          });
+      }
+  }
+  
+  return withModified;
+  
 };
 Worker.prototype.list.metadata = {
   options: {

package/file/S3.js

@@ -1,6 +1,5 @@
 const debug = require('debug')('@engine9-io/input/S3');
 const fs = require('node:fs');
-// eslint-disable-next-line import/no-unresolved
 const { mimeType: mime } = require('mime-type/with-db');
 const {
   S3Client,
@@ -187,7 +186,7 @@ Worker.prototype.write.metadata = {
   },
 };
 
-Worker.prototype.list = async function ({ directory, raw }) {
+Worker.prototype.list = async function ({ directory, start,end,raw }) {
   if (!directory) throw new Error('directory is required');
   let dir = directory;
   while (dir.slice(-1) === '/') dir = dir.slice(0, -1);
@@ -206,7 +205,16 @@ Worker.prototype.list = async function ({ directory, raw }) {
     name: f.Prefix.slice(Prefix.length + 1, -1),
     type: 'directory',
   })))
-    .concat((files || []).map(({ Key, Size, LastModified }) => ({
+    .concat((files || [])
+    .filter(({LastModified})=>{
+      if (start && new Date(LastModified)<start){
+        return false;
+      }else if (end && new Date(LastModified)>end){
+        return false;
+      }else{
+        return true;
+      }
+    }).map(({ Key, Size, LastModified }) => ({
       name: Key.slice(Prefix.length + 1),
       type: 'file',
       size: Size,

package/file/tools.js

@@ -12,6 +12,15 @@ const { PassThrough } = require('node:stream');
 const progress = require('debug')('info:@engine9/input-tools');
 const unzipper = require('unzipper');
 
+const dayjs = require('dayjs');
+
+const {
+  S3Client,
+  HeadObjectCommand,
+  GetObjectCommand,
+} = require('@aws-sdk/client-s3');
+
+
 const {
   v7: uuidv7,
 } = require('uuid');
@@ -72,12 +81,6 @@ async function writeTempFile(options) {
   return { filename };
 }
 
-const {
-  S3Client,
-  HeadObjectCommand,
-  GetObjectCommand,
-} = require('@aws-sdk/client-s3');
-
 async function getPacketFiles({ packet }) {
   if (packet.indexOf('s3://') === 0) {
     const parts = packet.split('/');
@@ -128,6 +131,7 @@ async function getPacketFiles({ packet }) {
   return directory;
 }
 
+
 async function getManifest({ packet }) {
   if (!packet) throw new Error('no packet option specififed');
   const { files } = await getPacketFiles({ packet });
@@ -222,6 +226,12 @@ async function downloadFile({ packet, type = 'person' }) {
   });
 }
 
+function isValidDate(d) {
+  // we WANT to use isNaN, not the Number.isNaN -- we're checking the date type
+  // eslint-disable-next-line no-restricted-globals
+  return d instanceof Date && !isNaN(d);
+}
+
 function bool(x, _defaultVal) {
   const defaultVal = (_defaultVal === undefined) ? false : _defaultVal;
   if (x === undefined || x === null || x === '') return defaultVal;
@@ -240,6 +250,67 @@ function getStringArray(s, nonZeroLength) {
   if (nonZeroLength && a.length === 0) a = [0];
   return a;
 }
+function relativeDate(s, _initialDate) {
+  let initialDate = _initialDate;
+  if (!s || s === 'none') return null;
+  if (typeof s.getMonth === 'function') return s;
+  // We actually want a double equals here to test strings as well
+  // eslint-disable-next-line eqeqeq
+  if (parseInt(s, 10) == s) {
+    const r = new Date(parseInt(s, 10));
+    if (!isValidDate(r)) throw new Error(`Invalid integer date:${s}`);
+    return r;
+  }
+
+  if (initialDate) {
+    initialDate = new Date(initialDate);
+  } else {
+    initialDate = new Date();
+  }
+
+  let r = s.match(/^([+-]{1})([0-9]+)([YyMwdhms]{1})([.a-z]*)$/);
+
+  if (r) {
+    let period = null;
+    switch (r[3]) {
+      case 'Y':
+      case 'y': period = 'years'; break;
+
+      case 'M': period = 'months'; break;
+      case 'w': period = 'weeks'; break;
+      case 'd': period = 'days'; break;
+      case 'h': period = 'hours'; break;
+      case 'm': period = 'minutes'; break;
+      case 's': period = 'seconds'; break;
+      default: period = 'minutes'; break;
+    }
+
+    let d = dayjs(initialDate);
+
+    if (r[1] === '+') {
+      d = d.add(parseInt(r[2], 10), period);
+    } else {
+      d = d.subtract(parseInt(r[2], 10), period);
+    }
+    if (!isValidDate(d.toDate())) throw new Error(`Invalid date configuration:${r}`);
+    if (r[4]) {
+      const opts = r[4].split('.').filter(Boolean);
+      if (opts[0] === 'start') d = d.startOf(opts[1] || 'day');
+      else if (opts[0] === 'end') d = d.endOf(opts[1] || 'day');
+      else throw new Error(`Invalid relative date,unknown options:${r[4]}`);
+    }
+
+    return d.toDate();
+  }
+  if (s === 'now') {
+    r = dayjs(new Date()).toDate();
+    return r;
+  }
+  r = dayjs(new Date(s)).toDate();
+  if (!isValidDate(r)) throw new Error(`Invalid Date: ${s}`);
+  return r;
+}
+
 /*
   When comparing two objects, some may come from a file (thus strings), and some from
   a database or elsewhere (not strings), so for deduping make sure to make them all strings
@@ -263,6 +334,7 @@ module.exports = {
   getPacketFiles,
   getStringArray,
   makeStrings,
+  relativeDate,
   streamPacket,
   writeTempFile,
 };

package/index.js

@@ -21,6 +21,8 @@ const {
   downloadFile,
   getTempFilename,
   getTempDir,
+  isValidDate,
+  relativeDate,
   streamPacket,
   getPacketFiles,
   getBatchTransform,
@@ -54,73 +56,6 @@ handlebars.registerHelper('percent', (a, b) => `${((100 * a) / b).toFixed(2)}%`)
 
 handlebars.registerHelper('or', (a, b, c) => a || b || c);
 
-function isValidDate(d) {
-  // we WANT to use isNaN, not the Number.isNaN -- we're checking the date type
-  // eslint-disable-next-line no-restricted-globals
-  return d instanceof Date && !isNaN(d);
-}
-
-function relativeDate(s, _initialDate) {
-  let initialDate = _initialDate;
-  if (!s || s === 'none') return null;
-  if (typeof s.getMonth === 'function') return s;
-  // We actually want a double equals here to test strings as well
-  // eslint-disable-next-line eqeqeq
-  if (parseInt(s, 10) == s) {
-    const r = new Date(parseInt(s, 10));
-    if (!isValidDate(r)) throw new Error(`Invalid integer date:${s}`);
-    return r;
-  }
-
-  if (initialDate) {
-    initialDate = new Date(initialDate);
-  } else {
-    initialDate = new Date();
-  }
-
-  let r = s.match(/^([+-]{1})([0-9]+)([YyMwdhms]{1})([.a-z]*)$/);
-
-  if (r) {
-    let period = null;
-    switch (r[3]) {
-      case 'Y':
-      case 'y': period = 'years'; break;
-
-      case 'M': period = 'months'; break;
-      case 'w': period = 'weeks'; break;
-      case 'd': period = 'days'; break;
-      case 'h': period = 'hours'; break;
-      case 'm': period = 'minutes'; break;
-      case 's': period = 'seconds'; break;
-      default: period = 'minutes'; break;
-    }
-
-    let d = dayjs(initialDate);
-
-    if (r[1] === '+') {
-      d = d.add(parseInt(r[2], 10), period);
-    } else {
-      d = d.subtract(parseInt(r[2], 10), period);
-    }
-    if (!isValidDate(d.toDate())) throw new Error(`Invalid date configuration:${r}`);
-    if (r[4]) {
-      const opts = r[4].split('.').filter(Boolean);
-      if (opts[0] === 'start') d = d.startOf(opts[1] || 'day');
-      else if (opts[0] === 'end') d = d.endOf(opts[1] || 'day');
-      else throw new Error(`Invalid relative date,unknown options:${r[4]}`);
-    }
-
-    return d.toDate();
-  }
-  if (s === 'now') {
-    r = dayjs(new Date()).toDate();
-    return r;
-  }
-  r = dayjs(new Date(s)).toDate();
-  if (!isValidDate(r)) throw new Error(`Invalid Date: ${s}`);
-  return r;
-}
-
 async function list(_path) {
   const directory = await unzipper.Open.file(_path);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engine9-io/input-tools",
-  "version": "1.7.5",
+  "version": "1.7.7",
   "description": "Tools for dealing with Engine9 inputs",
   "main": "index.js",
   "scripts": {
@@ -10,10 +10,7 @@
   "author": "Engine9",
   "license": "GPL-3.0-or-later",
   "devDependencies": {
-    "eslint": "^8.57.0",
-    "eslint-config-airbnb-base": "^15.0.0",
-    "eslint-plugin-import": "^2.29.0",
-    "eslint-plugin-jsonc": "^2.15.1"
+    "eslint": "^9.33.0"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.723.0",

package/test/file.js

@@ -0,0 +1,18 @@
+const {
+  it,
+} = require('node:test');
+const assert = require('node:assert');
+const debug = require('debug')('files');
+
+const { FileUtilities } = require('../index');
+
+it('Should list a directory', async () => {
+  const futil=new FileUtilities({accountId:'test'});
+  let files=await futil.list({directory:'.'});
+  assert(files.length,"Should have some files");
+  debug(files);
+  let startTest=await futil.list({directory:'.',start:'2040-01-01'});
+  assert(startTest.length===0,"Should NOT have any files before future start date");
+  let endTest=await futil.list({directory:'.',end:'1900-01-01'});
+  assert(endTest.length===0,"Should NOT have any files before past end date");
+});

package/.eslintignore

@@ -1,5 +0,0 @@
-node_modules
-**/node_modules
-./node_modules/*
-./node_modules/**
-*.schema

package/.eslintrc.js

@@ -1,36 +0,0 @@
-module.exports = {
-  env: {
-    browser: true,
-    commonjs: true,
-    es2021: true,
-  },
-  extends: [
-    'airbnb-base',
-    'plugin:jsonc/base',
-    'plugin:jsonc/recommended-with-json5'
-  ],
-  overrides: [
-    {
-      env: {
-        node: true,
-      },
-      files: [
-        '.eslintrc.{js,cjs}',
-      ],
-      parserOptions: {
-        sourceType: 'script',
-      },
-      plugins: [
-        'json5',
-      ],
-    },
-  ],
-  parserOptions: {
-    ecmaVersion: 'latest',
-  },
-
-  rules: {
-    'func-names': 'off', // Anonymous functions have their useful cases
-    'no-param-reassign': [2, { props: false }], // We often assign props of an object in a function, and that's generally safe.
-  },
-};