marcattacks 2.2.5 → 2.2.7

package/dist/globber.js

@@ -0,0 +1,131 @@
+#!/usr/bin/env node
+import log4js from 'log4js';
+import { program } from 'commander';
+import path from "node:path";
+import dotenv from 'dotenv';
+import { createRequire } from 'node:module';
+import { sftpGlobFiles } from './stream/sftpstream.js';
+import { s3GlobFiles } from './stream/s3stream.js';
+import { fileGlobFiles } from './stream/filestream.js';
+const require = createRequire(import.meta.url);
+const pkg = require('../package.json');
+program.version(pkg.version)
+    .argument('<file>')
+    .option('-c,--config <config>', 'config .env', path.join(process.cwd(), '.env'))
+    .option('--key <keyfile>', 'private key file')
+    .option('--key-env <env>', 'private key environment variable')
+    .option('--log <format>', 'logging format')
+    .option('--info', 'output debugging messages')
+    .option('--debug', 'output more debugging messages')
+    .option('--trace', 'output much more debugging messages');
+program.parse(process.argv);
+const opts = program.opts();
+if (opts.log) {
+    let output = 'stderr';
+    let type = 0;
+    if (opts.log.indexOf('stdout') >= 0) {
+        output = 'stdout';
+    }
+    if (opts.log.indexOf('json') >= 0) {
+        type = 1;
+    }
+    if (type) {
+        configureJSONLogger(output);
+    }
+    else {
+        configureDefaultLogger(output);
+    }
+}
+else {
+    configureDefaultLogger('stderr');
+}
+const logger = log4js.getLogger();
+if (opts.info) {
+    logger.level = "info";
+}
+if (opts.debug) {
+    logger.level = "debug";
+}
+if (opts.trace) {
+    logger.level = "trace";
+}
+if (opts.config) {
+    dotenv.config({ path: opts.config, quiet: true });
+}
+main();
+function configureDefaultLogger(output) {
+    log4js.configure({
+        appenders: {
+            err: {
+                type: output,
+                layout: {
+                    type: "pattern",
+                    pattern: "%[%d %p %f{2} %m%]"
+                }
+            }
+        },
+        categories: {
+            default: { appenders: ["err"], level: "off", enableCallStack: true }
+        }
+    });
+}
+function configureJSONLogger(output) {
+    log4js.addLayout('json-pattern', (config) => {
+        return (logEvent) => {
+            return JSON.stringify({
+                timestamp: logEvent.startTime, // Similar to %d
+                level: logEvent.level.levelStr, // Similar to %p
+                category: logEvent.categoryName, // Similar to %c
+                message: logEvent.data.join(' '), // Similar to %m
+                context: logEvent.context, // Similar to %X (tokens)
+                pid: logEvent.pid // Similar to %z
+            });
+        };
+    });
+    log4js.configure({
+        appenders: {
+            err: {
+                type: output,
+                layout: {
+                    type: "json-pattern"
+                }
+            }
+        },
+        categories: {
+            default: { appenders: ["err"], level: "off", enableCallStack: true }
+        }
+    });
+}
+async function main() {
+    try {
+        const url = program.args[0];
+        let globs = [];
+        if (url?.startsWith("sftp")) {
+            globs = await sftpGlobFiles(new URL(url), program.opts());
+        }
+        else if (url?.startsWith("s3")) {
+            const s3url = new URL(url);
+            if (process.env.S3_ACCESS_KEY) {
+                s3url.username = process.env.S3_ACCESS_KEY;
+            }
+            if (process.env.S3_SECRET_KEY) {
+                s3url.password = process.env.S3_SECRET_KEY;
+            }
+            globs = await s3GlobFiles(s3url, program.opts());
+        }
+        else if (url?.startsWith("file")) {
+            globs = await fileGlobFiles(new URL(url));
+        }
+        else {
+            console.error(`${url} not supported`);
+        }
+        globs.forEach(g => {
+            console.log(g.href);
+        });
+    }
+    catch (e) {
+        logger.error("process crashed:", e);
+        process.exitCode = 4;
+    }
+}
+//# sourceMappingURL=globber.js.map

package/dist/globber.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"globber.js","sourceRoot":"","sources":["../src/globber.ts"],"names":[],"mappings":";AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAEvC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;KACvB,QAAQ,CAAC,QAAQ,CAAC;KAClB,MAAM,CAAC,sBAAsB,EAAC,aAAa,EAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;KAC7E,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;KAC7C,MAAM,CAAC,iBAAiB,EAAC,kCAAkC,CAAC;KAC5D,MAAM,CAAC,gBAAgB,EAAC,gBAAgB,CAAC;KACzC,MAAM,CAAC,QAAQ,EAAC,2BAA2B,CAAC;KAC5C,MAAM,CAAC,SAAS,EAAC,gCAAgC,CAAC;KAClD,MAAM,CAAC,SAAS,EAAC,qCAAqC,CAAC,CAAC;AAE7D,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAE5B,MAAM,IAAI,GAAK,OAAO,CAAC,IAAI,EAAE,CAAC;AAE9B,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;IACX,IAAI,MAAM,GAAG,QAAQ,CAAC;IACtB,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,GAAG,QAAQ,CAAC;IACtB,CAAC;IACD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,IAAI,GAAG,CAAC,CAAC;IACb,CAAC;IAED,IAAI,IAAI,EAAE,CAAC;QACP,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;SACI,CAAC;QACF,sBAAsB,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;AACL,CAAC;KACI,CAAC;IACF,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;IACZ,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;AAC1B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;AAC3B,CAAC;AAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACd,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAG,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,IAAI,EAAE,CAAC;AAEP,SAAS,sBAAsB,CAAC,MAAc;IAC1C,MAAM,CAAC,SAAS,CAAC;QACb,SAAS,EAAE;YACP,GAAG,EAAE;gBACD,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACJ,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,oBAAoB;iBAChC;aACJ;SACJ;QACD,UAAU,EAAE;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;SACxE;KACJ,CAAC,CAAC;AACP,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAc;IACvC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE;QACxC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAChB,OAAO,IAAI,CAAC,SAAS,CAAC;gBACtB,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAG,gBAAgB;gBAChD,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,gBAAgB;gBAChD,QAAQ,EAAE,QAAQ,CAAC,YAAY,EAAC,gBAAgB;gBAChD,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,gBAAgB;gBAClD,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAO,yBAAyB;gBACzD,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAe,gBAAgB;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,SAAS,CAAC;QACb,SAAS,EAAE;YACP,GAAG,EAAE;gBACD,IAAI,EAAE,MAAM;gBACZ,MAAM,EAAE;oBACJ,IAAI,EAAE,cAAc;iBACvB;aACJ;SACJ;QACD,UAAU,EAAE;YACR,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAG,eAAe,EAAE,IAAI,EAAE;SACxE;KACJ,CAAC,CAAC;AACP,CAAC;AAED,KAAK,UAAU,IAAI;IACf,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,KAAK,GAAW,EAAE,CAAC;QAEvB,IAAI,GAAG,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,EAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,CAAC;aACI,IAAI,GAAG,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAE3B,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;gBAC5B,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;YAC/C,CAAC;YAED,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;gBAC5B,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;YAC/C,CAAC;YAED,KAAK,GAAG,MAAM,WAAW,CAAC,KAAK,EAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACpD,CAAC;aACI,IAAI,GAAG,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9C,CAAC;aACI,CAAC;YACF,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,gBAAgB,CAAC,CAAC;QAC1C,CAAC;QAED,KAAK,CAAC,OAAO,CAAE,CAAC,CAAC,EAAE;YACf,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACP,CAAC;IACD,OAAO,CAAC,EAAE,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;QACpC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACzB,CAAC;AACL,CAAC"}

package/dist/output/json.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,wBAAsB,SAAS,CAAC,MAAM,EAAC,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,CAsC/D"}
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,wBAAsB,SAAS,CAAC,MAAM,EAAC,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,CAoC/D"}

package/dist/output/json.js

@@ -3,7 +3,6 @@ import log4js from 'log4js';
 const logger = log4js.getLogger();
 export async function transform(_param) {
     let isFirst = true;
-    let hasClosed = false;
     return new Transform({
         objectMode: true,
         transform(data, _encoding, callback) {
@@ -24,11 +23,10 @@ export async function transform(_param) {
             logger.trace(`adding ${output.length} bytes`);
             callback(null, output);
         },
-        final(callback) {
-            logger.debug('final reached');
-            if (!isFirst && !hasClosed) {
+        flush(callback) {
+            logger.debug('flush reached');
+            if (!isFirst) {
                 this.push("]");
-                hasClosed = true;
             }
             callback();
         }

package/dist/output/json.js.map

@@ -1 +1 @@
-{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAU;IACtC,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAC;gBACX,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,GAAG,CAAC;YAClB,CAAC;iBACI,CAAC;gBACF,MAAM,IAAI,GAAG,CAAC;YAClB,CAAC;YAED,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAE/B,OAAO,GAAG,KAAK,CAAC;YAEhB,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,SAAS,GAAG,IAAI,CAAC;YACrB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/output/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAU;IACtC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAC;gBACX,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,GAAG,CAAC;YAClB,CAAC;iBACI,CAAC;gBACF,MAAM,IAAI,GAAG,CAAC;YAClB,CAAC;YAED,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAE/B,OAAO,GAAG,KAAK,CAAC;YAEhB,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}

package/dist/output/rdf.js

@@ -38,8 +38,8 @@ export async function transform(_param) {
                 logger.warn(`[${counter}] is not a Record or a JSON-LD`);
             }
         },
-        final(callback) {
-            logger.debug('final reached');
+        flush(callback) {
+            logger.debug('flush reached');
             writer?.end();
             callback();
         }

package/dist/output/xml.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAMnC,wBAAsB,SAAS,CAAC,MAAM,EAAC,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,CAgE/D;AAED,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,EACzC,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,OAAO,CAAA;CAAE,GACnC,MAAM,CAqBR"}
+{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAMnC,wBAAsB,SAAS,CAAC,MAAM,EAAC,GAAG,GAAI,OAAO,CAAC,SAAS,CAAC,CA8D/D;AAED,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,EACzC,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,OAAO,CAAA;CAAE,GACnC,MAAM,CAqBR"}

package/dist/output/xml.js

@@ -4,7 +4,6 @@ import log4js from 'log4js';
 const logger = log4js.getLogger();
 export async function transform(_param) {
     let isFirst = true;
-    let hasClosed = false;
     return new Transform({
         objectMode: true,
         transform(data, _encoding, callback) {
@@ -45,14 +44,13 @@ export async function transform(_param) {
             logger.trace(`adding ${output.length} bytes`);
             callback(null, output);
         },
-        final(callback) {
-            logger.debug('final reached');
-            if (!isFirst && !hasClosed) {
+        flush(callback) {
+            logger.debug('flush reached');
+            if (!isFirst) {
                 logger.debug("flushing");
                 let output = "</marc:collection>\n";
                 logger.trace(`adding ${output.length} bytes`);
                 this.push(output);
-                hasClosed = true;
             }
             callback();
         }

package/dist/output/xml.js.map

@@ -1 +1 @@
-{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAU;IACtC,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,GAAG,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC5F,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;oBAChF,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,IAAI,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACxF,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;gBACzB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAClB,SAAS,GAAG,IAAI,CAAC;YACrB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,uDAAuD;IACvD,0DAA0D;IAC1D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,uDAAuD,EAAE,EAAE,CAAC,CAAC;IAE3E,6EAA6E;IAC7E,wDAAwD;IACxD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,0CAA0C,EAAE,OAAO,CAAC,CAAC;IAEnE,kBAAkB;IAClB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAElD,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}
+{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../src/output/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAG,cAAc,EAAC,MAAM,eAAe,CAAC;AAChF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAU;IACtC,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,GAAG,GAAgB,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAA;gBACV,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,8CAA8C,CAAC;gBACzD,MAAM,IAAI,mEAAmE,CAAC;gBAC9E,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;YAED,MAAM,IAAI,kBAAkB,CAAC;YAE7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,GAAG,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAA,CAAC;qBAChB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;oBACrB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,kBAAkB,SAAS,CAAC,KAAK,CAAC,kBAAkB,CAAC;gBACnE,CAAC;qBACI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxB,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAE,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC3C,MAAM,IAAI,6BAA6B,GAAG,KAAK,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC;gBAC5F,CAAC;qBACI,CAAC;oBACF,MAAM,IAAI,0BAA0B,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;oBAChF,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,EAAE;wBAClC,MAAM,IAAI,4BAA4B,IAAI,KAAK,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC;oBACxF,CAAC,CAAC,CAAC;oBACH,MAAM,IAAI,uBAAuB,CAAC;gBACtC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,mBAAmB,CAAC;YAE9B,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAE9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,MAAM,GAAG,sBAAsB,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAyC,EACzC,OAAoC;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,uDAAuD;IACvD,0DAA0D;IAC1D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,uDAAuD,EAAE,EAAE,CAAC,CAAC;IAE3E,6EAA6E;IAC7E,wDAAwD;IACxD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,0CAA0C,EAAE,OAAO,CAAC,CAAC;IAEnE,kBAAkB;IAClB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAElD,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}

package/dist/{filestream.d.ts → stream/filestream.d.ts}

@@ -1,4 +1,5 @@
 import type { Readable } from "stream";
 export declare function fileReadStream(url: URL): Promise<Readable>;
 export declare function fileLatestFile(url: URL): Promise<URL>;
+export declare function fileGlobFiles(url: URL): Promise<URL[]>;
 //# sourceMappingURL=filestream.d.ts.map

package/dist/stream/filestream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filestream.d.ts","sourceRoot":"","sources":["../../src/stream/filestream.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAKvC,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CAEjE;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,GAAI,OAAO,CAAC,GAAG,CAAC,CA+B5D;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAsC5D"}

package/dist/stream/filestream.js

@@ -0,0 +1,65 @@
+import fs from "fs";
+import log4js from 'log4js';
+const logger = log4js.getLogger();
+export async function fileReadStream(url) {
+    return fs.createReadStream(url);
+}
+export async function fileLatestFile(url) {
+    return new Promise((resolve) => {
+        logger.info(`trying to resolve ${url.href}`);
+        if (!url.href.includes("@latest:")) {
+            logger.info(`resolved as ${url.href}`);
+            resolve(url);
+            return;
+        }
+        const directory = url.pathname.replaceAll(/@latest:.*/g, "");
+        const extension = url.pathname.replaceAll(/.*@latest:/g, "");
+        logger.debug(`directory: ${directory} ; extension: ${extension}`);
+        fs.readdir(directory, (err, files) => {
+            if (err) {
+                throw new Error("Error finding latest file:", err);
+            }
+            for (let i = 0; i < files.length; i++) {
+                if (files[i]?.toLowerCase().endsWith(extension)) {
+                    logger.info(`resolved as file://${directory}${files[i]}`);
+                    resolve(new URL("file://" + directory + files[i]));
+                    return;
+                }
+            }
+            throw new Error("No latest file found!");
+        });
+    });
+}
+export async function fileGlobFiles(url) {
+    return new Promise((resolve, reject) => {
+        logger.info(`trying to glob files for ${url.href}`);
+        // Check if the URL follows the @glob: pattern
+        if (!url.href.includes("@glob:")) {
+            logger.info(`no glob pattern found, returning original URL in array`);
+            resolve([url]);
+            return;
+        }
+        // Extract directory path and extension
+        const directory = url.pathname.replaceAll(/@glob:.*/g, "");
+        const extension = url.pathname.replaceAll(/.*@glob:/g, "");
+        logger.debug(`directory: ${directory} ; extension: ${extension}`);
+        fs.readdir(directory, (err, files) => {
+            if (err) {
+                // Reject the promise if the directory cannot be read
+                return reject(new Error(`Error reading directory for glob: ${err.message}`));
+            }
+            const targetExt = extension.toLowerCase();
+            const matchedUrls = [];
+            // Filter all files in the directory that match the extension
+            for (const file of files) {
+                if (file.toLowerCase().endsWith(targetExt)) {
+                    // Reconstruct the file URL
+                    matchedUrls.push(new URL("file://" + directory + file));
+                }
+            }
+            logger.info(`glob resolved ${matchedUrls.length} files`);
+            resolve(matchedUrls);
+        });
+    });
+}
+//# sourceMappingURL=filestream.js.map

package/dist/stream/filestream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filestream.js","sourceRoot":"","sources":["../../src/stream/filestream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAQ;IACzC,OAAO,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAQ;IACzC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACjC,MAAM,CAAC,IAAI,CAAC,qBAAqB,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAE7C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,CAAC;YACb,OAAO;QACX,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;QAE5D,MAAM,CAAC,KAAK,CAAC,cAAc,SAAS,iBAAiB,SAAS,EAAE,CAAC,CAAC;QAElE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,GAAG,EAAC,KAAK,EAAE,EAAE;YACjC,IAAI,GAAG,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,4BAA4B,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;gBACzC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC9C,MAAM,CAAC,IAAI,CAAC,sBAAsB,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC1D,OAAO,CAAC,IAAI,GAAG,CAAC,SAAS,GAAG,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACnD,OAAO;gBACX,CAAC;YACF,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAQ;IACxC,OAAO,IAAI,OAAO,CAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,CAAC,IAAI,CAAC,4BAA4B,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAEpD,8CAA8C;QAC9C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACtE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACf,OAAO;QACX,CAAC;QAED,uCAAuC;QACvC,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAE3D,MAAM,CAAC,KAAK,CAAC,cAAc,SAAS,iBAAiB,SAAS,EAAE,CAAC,CAAC;QAElE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACjC,IAAI,GAAG,EAAE,CAAC;gBACN,qDAAqD;gBACrD,OAAO,MAAM,CAAC,IAAI,KAAK,CAAC,qCAAqC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjF,CAAC;YAED,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;YAC1C,MAAM,WAAW,GAAU,EAAE,CAAC;YAE9B,6DAA6D;YAC7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBACzC,2BAA2B;oBAC3B,WAAW,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC;gBAC5D,CAAC;YACL,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,iBAAiB,WAAW,CAAC,MAAM,QAAQ,CAAC,CAAC;YACzD,OAAO,CAAC,WAAW,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/stream/httpstream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"httpstream.d.ts","sourceRoot":"","sources":["../../src/stream/httpstream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAQlC,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAsCnE"}

package/dist/{httpstream.js → stream/httpstream.js}

@@ -9,13 +9,17 @@ export function httpReadStream(urlString) {
         try {
             const url = new URL(urlString);
             const client = url.protocol === 'http:' ? http : https;
+            logger.debug(`resolve ${url.href}`);
             const req = client.get(url, res => {
-                if (res.statusCode && res.statusCode >= 400) {
+                const statusCode = res.statusCode || 0;
+                logger.debug(`statusCode = ${statusCode}`);
+                if (statusCode >= 400) {
                     reject(new Error('HTTP ' + res.statusCode));
                     return;
                 }
-                // Follow redirects
-                if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+                // Follow redirects (without any sanity checks..i know)
+                if (statusCode >= 300 && statusCode < 400 && res.headers.location) {
+                    logger.info(`Redirect to ${res.headers.location}...`);
                     httpReadStream(res.headers.location).then(resolve).catch(reject);
                     return;
                 }

package/dist/stream/httpstream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"httpstream.js","sourceRoot":"","sources":["../../src/stream/httpstream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC5C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;YAEvD,MAAM,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAEpC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE;gBAC9B,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;gBAEvC,MAAM,CAAC,KAAK,CAAC,gBAAgB,UAAU,EAAE,CAAC,CAAC;gBAE3C,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;oBACpB,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;oBAC5C,OAAO;gBACX,CAAC;gBAED,uDAAuD;gBACvD,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;oBAChE,MAAM,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC;oBACtD,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBACjE,OAAO;gBACX,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACtB,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAC;gBAC5C,MAAM,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;YAC3C,MAAM,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC"}

package/dist/{s3stream.d.ts → stream/s3stream.d.ts}

@@ -6,4 +6,5 @@ export declare function s3WriteStream(url: URL, options: {
     partSize?: number;
 }): Promise<Writable>;
 export declare function s3LatestObject(url: URL, opts: any): Promise<URL>;
+export declare function s3GlobFiles(url: URL, opts: any): Promise<URL[]>;
 //# sourceMappingURL=s3stream.d.ts.map

package/dist/stream/s3stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3stream.d.ts","sourceRoot":"","sources":["../../src/stream/s3stream.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAc5C,wBAAsB,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CA0F3F;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;CAAC,GAAI,OAAO,CAAC,QAAQ,CAAC,CA+F1F;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAiFtE;AAED,wBAAsB,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAgFrE"}

package/dist/{s3stream.js → stream/s3stream.js}

@@ -1,6 +1,7 @@
 import { S3Client, GetObjectCommand, PutObjectCommand, UploadPartCommand, CreateMultipartUploadCommand, CompleteMultipartUploadCommand, paginateListObjectsV2 } from "@aws-sdk/client-s3";
 import { Readable, Writable } from "stream";
 import log4js from 'log4js';
+import { getCleanURL, getStrippedURL } from "../util/stream_helpers.js";
 const logger = log4js.getLogger();
 export async function s3ReadStream(url, options) {
     const config = parseURL(url);
@@ -175,13 +176,19 @@ export function s3WriteStream(url, options) {
     });
 }
 export async function s3LatestObject(url, opts) {
+    logger.info(`trying to resolve ${getCleanURL(url)}`);
     if (!url.href.includes("@latest:")) {
+        logger.info(`resolved as: ${getCleanURL(url)}`);
         return url;
     }
-    const bucket = url.pathname.replaceAll(/@latest:.*/g, "");
+    const bucket = url.pathname.replaceAll(/\/@latest:.*/g, "").replaceAll(/^\//g, "");
     const extension = url.pathname.replaceAll(/.*@latest:/g, "");
     const config = parseURL(url);
-    const s3Client = new S3Client(config);
+    delete config.key;
+    logger.debug(`s3 config:`, config);
+    logger.debug(`bucket:`, bucket);
+    logger.debug(`extension:`, extension);
+    const s3Client = makeClient(config);
     const paginatorConfig = {
         client: s3Client,
         pageSize: 1000
@@ -225,15 +232,81 @@ export async function s3LatestObject(url, opts) {
             url_parts.push(':');
             url_parts.push(url.port);
         }
-        url_parts.push(bucket + '/' + latestFile.Key);
-        logger.trace(`resolved as ${url_parts.join("")}`);
-        return new URL(url_parts.join(""));
+        url_parts.push('/' + bucket + '/' + latestFile.Key);
+        const result = new URL(url_parts.join(""));
+        logger.info(`resolved as: ${getCleanURL(result)}`);
+        return result;
     }
     catch (error) {
         console.error("Error finding latest S3 file:", error);
         throw error;
     }
 }
+export async function s3GlobFiles(url, opts) {
+    logger.info(`trying to glob files for ${getCleanURL(url)}`);
+    // Check if the URL follows the @glob: pattern
+    if (!url.href.includes("@glob:")) {
+        logger.info(`no glob pattern found, returning original URL in array`);
+        return [getStrippedURL(url)];
+    }
+    // Extract bucket name and extension from the pathname
+    const bucket = url.pathname.replaceAll(/\/@glob:.*/g, "").replaceAll(/^\//g, "");
+    const extension = url.pathname.replaceAll(/.*@glob:/g, "");
+    const config = parseURL(url);
+    logger.debug(`s3 config:`, config);
+    logger.debug(`bucket:`, bucket);
+    logger.debug(`extension:`, extension);
+    const s3Client = makeClient(config);
+    const paginatorConfig = {
+        client: s3Client,
+        pageSize: 1000
+    };
+    const commandInput = {
+        Bucket: bucket,
+    };
+    try {
+        const matchedUrls = [];
+        const targetExt = extension.toLowerCase();
+        // Iterate through all pages of the bucket using the V2 paginator
+        for await (const page of paginateListObjectsV2(paginatorConfig, commandInput)) {
+            const contents = page.Contents || [];
+            for (const obj of contents) {
+                if (obj.Key?.endsWith("/"))
+                    continue;
+                // Filter by extension match
+                if (obj.Key?.toLowerCase().endsWith(targetExt) || targetExt === '*') {
+                    // Reconstruct the URL for each matched object
+                    const url_parts = [];
+                    url_parts.push(url.protocol);
+                    url_parts.push('//');
+                    if (url.username) {
+                        url_parts.push(url.username);
+                        if (url.password) {
+                            url_parts.push(':');
+                            url_parts.push(url.password);
+                        }
+                        url_parts.push('@');
+                    }
+                    url_parts.push(url.hostname);
+                    if (url.port && !["80", "443"].includes(url.port)) {
+                        url_parts.push(':');
+                        url_parts.push(url.port);
+                    }
+                    // Note: bucket is already part of the path logic in parseURL
+                    url_parts.push(`/${bucket}/${obj.Key}`);
+                    const result = new URL(url_parts.join(""));
+                    matchedUrls.push(getStrippedURL(result));
+                }
+            }
+        }
+        logger.info(`glob resolved ${matchedUrls.length} files`);
+        return matchedUrls;
+    }
+    catch (error) {
+        logger.error("Error globbing S3 files:", error);
+        throw error;
+    }
+}
 function isNodeReadable(x) {
     return x && typeof x.pipe === "function" && typeof x.read === "function";
 }
@@ -262,8 +335,8 @@ function parseURL(url) {
     const config = {
         region: "us-east-1",
         endpoint: "http://localhost:3371",
-        bucket: "bbl",
-        key: "test.txt"
+        bucket: "",
+        key: ""
     };
     const scheme = url.protocol.startsWith("s3s") ? "https" : "http";
     config.endpoint = `${scheme}://${url.hostname}`;

package/dist/stream/s3stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3stream.js","sourceRoot":"","sources":["../../src/stream/s3stream.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EACR,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,4BAA4B,EAC5B,8BAA8B,EAC9B,qBAAqB,EAGxB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAC5C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAExE,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAUlC,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,GAAQ,EAAE,OAA2B;IACpE,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAE7B,MAAM,CAAC,KAAK,CAAC,YAAY,EAAC,MAAM,CAAC,CAAC;IAElC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,GAAG,GAAM,MAAM,CAAC,GAAG,CAAC;IAC1B,MAAM,KAAK,GAAI,OAAO,CAAC,KAAK,CAAC;IAC7B,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAE9B,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC;QAC3C,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG;QACR,KAAK,EAAE,KAAK;KACf,CAAC,CAAC,CAAC;IAEJ,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IAEtB,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC3D,CAAC;IAED,iEAAiE;IACjE,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,IAAgB,CAAC;IAC5B,CAAC;IAED,6EAA6E;IAC7E,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,oCAAoC;QACpC,gCAAgC;QAChC,IAAI,OAAQ,QAAgB,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YACtD,OAAQ,QAAgB,CAAC,OAAO,CAAC,IAAkC,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACR,sDAAsD;YACtD,MAAM,MAAM,GAAI,IAAmC,CAAC,SAAS,EAAE,CAAC;YAChE,MAAM,UAAU,GAAG,IAAI,QAAQ,CAAC;gBAC5B,IAAI;oBACA,0CAA0C;gBAC9C,CAAC;aACJ,CAAC,CAAC;YACH,CAAC,KAAK,IAAI,EAAE;gBACR,IAAI,CAAC;oBACL,OAAO,IAAI,EAAE,CAAC;wBACV,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;wBAC5C,IAAI,IAAI;4BAAE,MAAM;wBAChB,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;oBACxC,CAAC;oBACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACf,UAAU,CAAC,OAAO,CAAC,GAAY,CAAC,CAAC;gBACjC,CAAC;YACL,CAAC,CAAC,EAAE,CAAC;YACL,OAAO,UAAU,CAAC;QAClB,CAAC;IACL,CAAC;IAED,uCAAuC;IACvC,IAAI,OAAO,IAAI,KAAK,WAAW,IAAI,IAAI,YAAY,IAAI,EAAE,CAAC;QACtD,MAAM,MAAM,GAAI,IAAa,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,OAAQ,QAAgB,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAClD,OAAQ,QAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7C,CAAC;QACD,yBAAyB;QACzB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,IAAI,QAAQ,CAAC;YAChC,IAAI,KAAI,CAAC;SACR,CAAC,CAAC;QACH,CAAC,KAAK,IAAI,EAAE;YACZ,IAAI,CAAC;gBACD,OAAO,IAAI,EAAE,CAAC;oBACd,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAChB,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACpC,CAAC;gBACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,UAAU,CAAC,OAAO,CAAC,GAAY,CAAC,CAAC;YACrC,CAAC;QACD,CAAC,CAAC,EAAE,CAAC;QACL,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,+CAA+C;IAC/C,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAmD,CAAC,CAAC;IAC9E,CAAC;IAED,qBAAqB;IACrB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAQ,EAAE,OAA8B;IAClE,OAAO,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QAE7B,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;QACvB,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QAErD,IAAI,QAAQ,GAAkB,IAAI,CAAC;QACnC,IAAI,KAAK,GAA4D,EAAE,CAAC;QACxE,IAAI,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC;YACxB,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ;gBAClC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;gBAC/B,IAAI,CAAC;oBACD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;oBAExC,IAAI,MAAM,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;wBAC5B,MAAM,SAAS,EAAE,CAAC;oBACtB,CAAC;oBACD,QAAQ,EAAE,CAAC;gBACf,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,QAAQ,CAAC,GAAY,CAAC,CAAC;gBAC3B,CAAC;YACL,CAAC;YAED,KAAK,CAAC,KAAK,CAAC,QAAQ;gBAChB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzB,IAAI,CAAC;oBACD,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;oBAC7B,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;oBACtB,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAChC,MAAM,YAAY,EAAE,CAAC;oBACrB,QAAQ,EAAE,CAAC;gBACf,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,QAAQ,CAAC,GAAY,CAAC,CAAC;gBAC3B,CAAC;YACL,CAAC;SACJ,CAAC,CAAC;QAEH,KAAK,UAAU,YAAY;YACvB,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,4BAA4B,CAAC;oBACvD,MAAM,EAAE,MAAM;oBACd,GAAG,EAAE,GAAG;iBACX,CAAC,CAAC,CAAC;gBACJ,QAAQ,GAAG,GAAG,CAAC,QAAS,CAAC;YAC7B,CAAC;QACL,CAAC;QAED,KAAK,UAAU,SAAS,CAAC,MAAM,GAAG,KAAK;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM;gBAAE,OAAO;YAE3C,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAChC,MAAM,YAAY,EAAE,CAAC;YAErB,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAC3B,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,iBAAiB,CAAC;gBAC5C,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,GAAG;gBACR,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,QAAS;gBACnB,IAAI,EAAE,MAAM;aACf,CAAC,CAAC,CAAC;YAEJ,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC;YACvD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,UAAU,EAAE,CAAC;QACjB,CAAC;QAED,KAAK,UAAU,YAAY;YACvB,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,wCAAwC;gBACxC,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC;oBAC/B,MAAM,EAAE,MAAM;oBACd,GAAG,EAAE,GAAG;oBACR,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;iBACxB,CAAC,CAAC,CAAC;gBACJ,OAAO;YACX,CAAC;YAED,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,8BAA8B,CAAC;gBAC7C,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,GAAG;gBACR,QAAQ,EAAE,QAAS;gBACnB,eAAe,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE;aACpC,CAAC,CAAC,CAAC;QACR,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAQ,EAAE,IAAS;IACpD,MAAM,CAAC,IAAI,CAAC,qBAAqB,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAErD,IAAI,CAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,gBAAgB,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,OAAO,GAAG,CAAC;IACf,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,eAAe,EAAC,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,EAAC,EAAE,CAAC,CAAC;IACjF,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,EAAC,EAAE,CAAC,CAAC;IAE5D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC7B,OAAQ,MAA2B,CAAC,GAAG,CAAC;IAExC,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACnC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,CAAC,KAAK,CAAC,YAAY,EAAC,SAAS,CAAC,CAAC;IAErC,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAkB,CAAC,CAAC;IAEhD,MAAM,eAAe,GAAG;QACpB,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,IAAI;KACjB,CAAC;IAEF,MAAM,YAAY,GAAG;QACjB,MAAM,EAAE,MAAM;QACd,gCAAgC;KACnC,CAAC;IAEF,IAAI,CAAC;QACD,IAAI,UAAU,GAAmB,IAAI,CAAC;QACtC,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAE1C,0CAA0C;QAC1C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,qBAAqB,CAAC,eAAe,EAAE,YAAY,CAAC,EAAE,CAAC;YAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAErC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBACzB,sBAAsB;gBACtB,IAAI,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC7C,6CAA6C;oBAC7C,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,YAAa,GAAG,UAAU,CAAC,YAAa,CAAC,EAAE,CAAC;wBAChE,UAAU,GAAG,GAAG,CAAC;oBACrB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,sBAAsB,MAAM,IAAI,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,SAAS,GAAc,EAAE,CAAC;QAEhC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC7B,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;gBACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACnB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC7B,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,GAAG,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAI,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,gBAAgB,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAEnD,OAAO,MAAM,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;QACtD,MAAM,KAAK,CAAC;IAChB,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAQ,EAAE,IAAS;IACjD,MAAM,CAAC,IAAI,CAAC,4BAA4B,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAE5D,8CAA8C;IAC9C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;QACtE,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,sDAAsD;IACtD,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,EAAC,EAAE,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAE7B,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACnC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,CAAC,KAAK,CAAC,YAAY,EAAC,SAAS,CAAC,CAAC;IAErC,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAkB,CAAC,CAAC;IAEhD,MAAM,eAAe,GAAG;QACpB,MAAM,EAAE,QAAQ;QAChB,QAAQ,EAAE,IAAI;KACjB,CAAC;IAEF,MAAM,YAAY,GAAG;QACjB,MAAM,EAAE,MAAM;KACjB,CAAC;IAEF,IAAI,CAAC;QACD,MAAM,WAAW,GAAU,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAE1C,iEAAiE;QACjE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,qBAAqB,CAAC,eAAe,EAAE,YAAY,CAAC,EAAE,CAAC;YAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAErC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBACzB,IAAI,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACrC,4BAA4B;gBAC5B,IAAI,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;oBAClE,8CAA8C;oBAC9C,MAAM,SAAS,GAAa,EAAE,CAAC;oBAE/B,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBAC7B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAErB,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;wBACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBAC7B,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;4BACf,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;4BACpB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;wBACjC,CAAC;wBACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACxB,CAAC;oBAED,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;oBAE7B,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;wBAChD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACpB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC7B,CAAC;oBAED,6DAA6D;oBAC7D,SAAS,CAAC,IAAI,CAAC,IAAI,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBAExC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;oBAE3C,WAAW,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC7C,CAAC;YACL,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,iBAAiB,WAAW,CAAC,MAAM,QAAQ,CAAC,CAAC;QACzD,OAAO,WAAW,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,KAAK,CAAC;IAChB,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,CAAM;IAC1B,OAAO,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC;AAC7E,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAM;IAC5B,OAAO,OAAO,CAAC,EAAE,SAAS,KAAK,UAAU,CAAC;AAC9C,CAAC;AAED,SAAS,eAAe,CAAC,CAAM;IAC3B,OAAO,CAAC,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC;AAC9D,CAAC;AAED,SAAS,UAAU,CAAC,MAAgB;IAChC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,QAAQ,GAAoB;QAC9B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,MAAM,CAAC,MAAM;KACxB,CAAC;IAEF,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC/C,QAAQ,CAAC,WAAW,GAAG;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;SAC1C,CAAC;IACN,CAAC;IACD,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,GAAQ;IACtB,MAAM,MAAM,GAAc;QACtB,MAAM,EAAE,WAAW;QACnB,QAAQ,EAAE,uBAAuB;QACjC,MAAM,EAAE,EAAE;QACV,GAAG,EAAE,EAAE;KACV,CAAC;IAEF,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,CAAC,QAAQ,GAAG,GAAG,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAC;IAChD,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,CAAC,QAAQ,IAAI,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC;IACD,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACjD,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEzD,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC;IACtC,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACf,MAAM,CAAC,eAAe,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1C,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/dist/{sftpstream.d.ts → stream/sftpstream.d.ts}

@@ -9,4 +9,5 @@ export interface SftpConfig {
 export declare function sftpReadStream(url: URL, opts: any): Promise<Readable>;
 export declare function sftpWriteStream(url: URL, opts: any): Promise<Writable>;
 export declare function sftpLatestFile(url: URL, opts: any): Promise<URL>;
+export declare function sftpGlobFiles(url: URL, opts: any): Promise<URL[]>;
 //# sourceMappingURL=sftpstream.d.ts.map

package/dist/stream/sftpstream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sftpstream.d.ts","sourceRoot":"","sources":["../../src/stream/sftpstream.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAG,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAM7C,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC9B;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAmC3E;AAED,wBAAsB,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAiC5E;AAED,wBAAsB,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CA+EtE;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CA+EvE"}

package/dist/{sftpstream.js → stream/sftpstream.js}

@@ -58,9 +58,11 @@ export async function sftpWriteStream(url, opts) {
 }
 export async function sftpLatestFile(url, opts) {
     const config = makeSftpConfig(url, opts);
+    logger.info(`trying to resolve ${url.href}`);
     logger.debug(`sftp config:`, config);
     return new Promise((resolve, reject) => {
-        if (!url.pathname.match(/\/@latest:\w+$/)) {
+        if (!url.pathname.match(/\/@latest:\S+$/)) {
+            logger.info(`resolved as: ${url.href}`);
             resolve(url);
             return;
         }
@@ -108,7 +110,7 @@ export async function sftpLatestFile(url, opts) {
                         url_parts.push(url.port);
                     }
                     url_parts.push(latestPath);
-                    logger.trace(`resolved as: ${url_parts.join("")}`);
+                    logger.info(`resolved as: ${url_parts.join("")}`);
                     resolve(new URL(url_parts.join("")));
                 });
             });
@@ -117,6 +119,68 @@ export async function sftpLatestFile(url, opts) {
         conn.connect(config);
     });
 }
+export async function sftpGlobFiles(url, opts) {
+    const config = makeSftpConfig(url, opts);
+    logger.info(`trying to glob files for ${url.href}`);
+    return new Promise((resolve, reject) => {
+        // Check if the URL follows the @glob: pattern
+        if (!url.pathname.match(/\/@glob:\S+$/)) {
+            logger.info(`no glob pattern found, returning original URL in array`);
+            resolve([url]);
+            return;
+        }
+        const remoteDir = url.pathname.replace(/\/@glob.*/, "");
+        const extension = url.pathname.replace(/.*\/@glob:/, "");
+        const conn = new Client();
+        conn.on("ready", () => {
+            conn.sftp((err, sftp) => {
+                if (err) {
+                    conn.end();
+                    return reject(err);
+                }
+                sftp.readdir(remoteDir, (err, list) => {
+                    if (err) {
+                        conn.end();
+                        return reject(err);
+                    }
+                    if (!list || list.length === 0) {
+                        conn.end();
+                        return resolve([]); // Return empty array if no files exist
+                    }
+                    // Filter files by the extension provided after @glob:
+                    const matchedFiles = list.filter(f => f.filename.toLowerCase().endsWith(extension.toLowerCase()));
+                    const results = matchedFiles.map(file => {
+                        const filePath = `${remoteDir}/${file.filename}`;
+                        // Reconstruct the URL for each matched file
+                        const url_parts = [];
+                        url_parts.push(url.protocol);
+                        url_parts.push('//');
+                        if (url.username) {
+                            url_parts.push(url.username);
+                            if (url.password) {
+                                url_parts.push(':');
+                                url_parts.push(url.password);
+                            }
+                            url_parts.push('@');
+                        }
+                        url_parts.push(url.hostname);
+                        if (url.port && !["80", "443"].includes(url.port)) {
+                            url_parts.push(':');
+                            url_parts.push(url.port);
+                        }
+                        url_parts.push(filePath);
+                        return new URL(url_parts.join(""));
+                    });
+                    conn.end();
+                    logger.info(`glob resolved ${results.length} files`);
+                    resolve(results);
+                });
+            });
+        });
+        conn.on("error", (err) => reject(err));
+        conn.connect(config);
+    });
+}
 function makeSftpConfig(inputFile, opts) {
     let privateKey = undefined;
     if (opts.key) {