marcattacks 2.8.0 → 2.9.0

package/dist/util/stream_helpers.js

@@ -3,6 +3,12 @@ import log4js from 'log4js';
 import { createGunzip } from 'zlib';
 import tar from 'tar-stream';
 const logger = log4js.getLogger();
+/**
+ * How long to wait for a flush target that never emits 'finish'.
+ * process.stdout/stderr are never end()ed by pipeline(), so 'finish' will
+ * not fire for them; this fallback keeps the pipeline from hanging.
+ */
+const FLUSH_FALLBACK_MS = 2000;
 /**
  * Limits and skips objects in a Readable stream.
  * @param count Maximum number of objects to emit (optional)
@@ -11,7 +17,9 @@ const logger = log4js.getLogger();
 export function createCountableSkippedStream(count, skip = 0) {
     let skipped = 0;
     let pushed = 0;
-    return new Transform({
+    let limitReached = false;
+    let flushTarget;
+    const stream = new Transform({
         objectMode: true,
         transform(chunk, _encoding, callback) {
             if (skipped < skip) {
@@ -19,33 +27,78 @@ export function createCountableSkippedStream(count, skip = 0) {
                 logger.debug(`skipped: ${skipped}`);
                 return callback(); // Drop the chunk
             }
+            // Already past the limit: end the readable side and park upstream
+            // (no callback) so we stop reading the source. Mirrors the original
+            // control flow — the repeated push(null) matters for propagating EOF
+            // cleanly through a downstream worker pool.
             if (count !== undefined && pushed >= count) {
-                logger.debug("Limit reached, closing gracefully...");
                 this.push(null);
-                // Hacky but I do not know another way to close all streams and
-                // be able to let them flush their content
-                setTimeout(() => {
-                    logger.debug("delay finished, completing transform callback");
-                    this.destroy();
-                }, 2000);
+                closeGracefully();
                 return;
             }
             this.push(chunk);
             pushed++;
             logger.debug(`pushed: ${pushed}`);
+            // Hitting the limit: end the readable side so downstream flushes.
+            // We still call callback() below (the boundary chunk) so 'end'
+            // propagates; the *next* chunk parks the source via the guard above.
             if (count !== undefined && pushed === count) {
-                logger.debug("Limit reached, closing gracefully...");
-                this.push(null);
-                // Hacky but I do not know another way to close all streams and
-                // be able to let them flush their content
-                setTimeout(() => {
-                    logger.debug("delay finished, completing transform callback");
-                    this.destroy();
-                }, 2000);
+                closeGracefully();
             }
             callback();
         }
     });
+    stream.setFlushTarget = (target) => { flushTarget = target; };
+    /**
+     * Reached the record limit: end the downstream chain so the sink flushes,
+     * then destroy the pipeline once the sink has actually finished (rather
+     * than after a fixed delay). The destroy surfaces as ERR_STREAM_PREMATURE_
+     * CLOSE upstream, which the caller treats as a clean limiter stop.
+     */
+    function closeGracefully() {
+        if (limitReached)
+            return;
+        limitReached = true;
+        logger.debug("Limit reached, closing gracefully...");
+        stream.push(null); // EOF -> downstream transforms flush -> sink finishes
+        let torn = false;
+        let fallback;
+        const teardown = (why) => {
+            if (torn)
+                return;
+            torn = true;
+            // Clear the orphaned fallback timer; a pending timer would keep the
+            // event loop alive (and delay process exit) long after teardown.
+            if (fallback)
+                clearTimeout(fallback);
+            logger.debug(`sink ${why}, tearing down pipeline`);
+            stream.destroy();
+        };
+        // stdout/stderr are never end()ed by pipeline(), so they never emit
+        // 'finish'/'close'. Don't wait on events that won't come — fall straight
+        // through to the original fixed-timer teardown (this also keeps the
+        // default jsonata worker-pool case working, which the timer delay covers).
+        // Checking flushTarget inline (rather than via a helper bool) lets TS
+        // narrow it to a defined Writable for the real-sink branch below.
+        if (!flushTarget
+            || flushTarget === process.stdout
+            || flushTarget === process.stderr) {
+            setTimeout(() => teardown('timer (stdout/no target)'), FLUSH_FALLBACK_MS);
+            return;
+        }
+        if (flushTarget.writableFinished) {
+            teardown('already finished');
+            return;
+        }
+        // Real sink (S3, sftp, file): tear down the moment it has flushed,
+        // rather than after a fixed delay. The fallback timer only fires if the
+        // sink never finishes for some reason, so we don't hang.
+        flushTarget.once('finish', () => teardown('finished'));
+        flushTarget.once('close', () => teardown('closed'));
+        flushTarget.once('error', () => teardown('errored'));
+        fallback = setTimeout(() => teardown('flush fallback timer'), FLUSH_FALLBACK_MS);
+    }
+    return stream;
 }
 export function createVerboseStream(logEvery = 1000) {
     let recordNum = 0;

package/dist/util/stream_helpers.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream_helpers.js","sourceRoot":"","sources":["../../src/util/stream_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AACrE,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC;;;;GAIG;AACH,MAAM,UAAU,4BAA4B,CACxC,KAAc,EACd,OAAe,CAAC;IAEhB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAyB,EAAE,QAA2B;YACxE,IAAI,OAAO,GAAG,IAAI,EAAE,CAAC;gBACjB,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;gBACpC,OAAO,QAAQ,EAAE,CAAC,CAAC,iBAAiB;YACxC,CAAC;YAED,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;gBACzC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;gBAErD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEhB,+DAA+D;gBAC/D,0CAA0C;gBAC1C,UAAU,CAAC,GAAG,EAAE;oBACZ,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;oBAC9D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACnB,CAAC,EAAC,IAAI,CAAC,CAAC;gBAER,OAAO;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,MAAM,EAAE,CAAC;YAET,MAAM,CAAC,KAAK,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;YAElC,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;gBACrD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEhB,+DAA+D;gBAC/D,0CAA0C;gBAC1C,UAAU,CAAC,GAAG,EAAE;oBACZ,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;oBAC9D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACnB,CAAC,EAAC,IAAI,CAAC,CAAC;YACZ,CAAC;YAED,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AASD,MAAM,UAAU,mBAAmB,CAAC,WAAmB,IAAI;IACvD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC;QAC5B,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAyB,EAAE,QAA2B;YACxE,SAAS,EAAE,CAAC;YAEZ,MAAM,CAAC,KAAK,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,qBAAqB,aAAa,IAAI,CAAC,qBAAqB,UAAU,CAAC,CAAC;YAE7G,IAAI,SAAS,GAAG,QAAQ,KAAK,CAAC,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBAC9B,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,GAAC,QAAQ,CAAC;gBACxC,MAAM,KAAK,GAAG,SAAS,GAAC,QAAQ,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,qBAAqB,aAAa,IAAI,CAAC,qBAAqB,UAAU,CAAC,CAAC;gBAC7G,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YACtE,CAAC;YACD,QAAQ,CAAC,IAAI,EAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,CAAE,QAA2B;YAC9B,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,GAAC,IAAI,CAAC;YACpC,MAAM,KAAK,GAAG,SAAS,GAAC,QAAQ,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;YAC7G,SAAS,CAAC,QAAQ,GAAG,GAAG,EAAE;gBACtB,MAAM,CAAC,KAAK,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC;gBAChD,OAAO,SAAS,CAAC;YACrB,CAAC,CAAA;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAkB,CAAC;IAEpB,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB;IACpC,OAAO,YAAY,EAAE,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACtC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC;IAE9B,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;QAClC,UAAU,EAAE,IAAI;QAEhB,SAAS,CAAC,KAAU,EAAE,QAAgB,EAAE,QAA2B;YAC/D,4DAA4D;YAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAA0B,CAAC,CAAC;YAE/F,wEAAwE;YACxE,MAAM,eAAe,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAE9C,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnB,wBAAwB;gBACxB,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACJ,QAAQ,EAAE,CAAC;YACf,CAAC;QACL,CAAC;QACD,KAAK,CAAC,QAA2B;YAC7B,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACrC,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;QACzC,MAAM,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,IAAI,kBAAkB,CAAC,CAAC;QAEzD,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACxB,MAAM,CAAC,KAAK,CAAC,0BAA0B,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAClB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,KAAK,CAAC,gBAAgB,MAAM,CAAC,IAAI,iBAAiB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1E,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/C,IAAI,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACvB,MAAM,CAAC,KAAK,CAAC,uBAAuB,MAAM,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC1C,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;IACvD,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACxB,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;QACxC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,OAAO,eAAe,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,GAAQ;IAChC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IACzB,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IACzB,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,GAAQ;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,OAAO,CAAC,QAAQ,GAAG,EAAE,CAAC;IACtB,OAAO,CAAC,QAAQ,GAAG,EAAE,CAAC;IACtB,OAAO,OAAO,CAAC;AACnB,CAAC"}
+{"version":3,"file":"stream_helpers.js","sourceRoot":"","sources":["../../src/util/stream_helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAyC,MAAM,QAAQ,CAAC;AACpF,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAgBlC;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAE/B;;;;GAIG;AACH,MAAM,UAAU,4BAA4B,CACxC,KAAc,EACd,OAAe,CAAC;IAEhB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,WAAiC,CAAC;IAEtC,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAyB,EAAE,QAA2B;YACxE,IAAI,OAAO,GAAG,IAAI,EAAE,CAAC;gBACjB,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;gBACpC,OAAO,QAAQ,EAAE,CAAC,CAAC,iBAAiB;YACxC,CAAC;YAED,kEAAkE;YAClE,oEAAoE;YACpE,qEAAqE;YACrE,4CAA4C;YAC5C,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;gBACzC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChB,eAAe,EAAE,CAAC;gBAClB,OAAO;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjB,MAAM,EAAE,CAAC;YAET,MAAM,CAAC,KAAK,CAAC,WAAW,MAAM,EAAE,CAAC,CAAC;YAElC,kEAAkE;YAClE,+DAA+D;YAC/D,qEAAqE;YACrE,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;gBAC1C,eAAe,EAAE,CAAC;YACtB,CAAC;YAED,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAoB,CAAC;IAEtB,MAAM,CAAC,cAAc,GAAG,CAAC,MAAgB,EAAE,EAAE,GAAG,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAExE;;;;;OAKG;IACH,SAAS,eAAe;QACpB,IAAI,YAAY;YAAE,OAAO;QACzB,YAAY,GAAG,IAAI,CAAC;QAEpB,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,sDAAsD;QAEzE,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,IAAI,QAAmD,CAAC;QACxD,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE;YAC7B,IAAI,IAAI;gBAAE,OAAO;YACjB,IAAI,GAAG,IAAI,CAAC;YACZ,oEAAoE;YACpE,iEAAiE;YACjE,IAAI,QAAQ;gBAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;YACrC,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,yBAAyB,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC,CAAC;QAEF,oEAAoE;QACpE,yEAAyE;QACzE,oEAAoE;QACpE,2EAA2E;QAC3E,sEAAsE;QACtE,kEAAkE;QAClE,IAAI,CAAC,WAAW;eACT,WAAW,KAAK,OAAO,CAAC,MAAM;eAC9B,WAAW,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YACpC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,iBAAiB,CAAC,CAAC;YAC1E,OAAO;QACX,CAAC;QAED,IAAI,WAAW,CAAC,gBAAgB,EAAE,CAAC;YAC/B,QAAQ,CAAC,kBAAkB,CAAC,CAAC;YAC7B,OAAO;QACX,CAAC;QAED,mEAAmE;QACnE,wEAAwE;QACxE,yDAAyD;QACzD,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;QACvD,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpD,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;QACrD,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AASD,MAAM,UAAU,mBAAmB,CAAC,WAAmB,IAAI;IACvD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC;QAC5B,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAyB,EAAE,QAA2B;YACxE,SAAS,EAAE,CAAC;YAEZ,MAAM,CAAC,KAAK,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,qBAAqB,aAAa,IAAI,CAAC,qBAAqB,UAAU,CAAC,CAAC;YAE7G,IAAI,SAAS,GAAG,QAAQ,KAAK,CAAC,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBAC9B,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,GAAC,QAAQ,CAAC;gBACxC,MAAM,KAAK,GAAG,SAAS,GAAC,QAAQ,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,mBAAmB,IAAI,CAAC,qBAAqB,aAAa,IAAI,CAAC,qBAAqB,UAAU,CAAC,CAAC;gBAC7G,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YACtE,CAAC;YACD,QAAQ,CAAC,IAAI,EAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,CAAE,QAA2B;YAC9B,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,GAAC,IAAI,CAAC;YACpC,MAAM,KAAK,GAAG,SAAS,GAAC,QAAQ,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,eAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;YAC7G,SAAS,CAAC,QAAQ,GAAG,GAAG,EAAE;gBACtB,MAAM,CAAC,KAAK,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC;gBAChD,OAAO,SAAS,CAAC;YACrB,CAAC,CAAA;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAkB,CAAC;IAEpB,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB;IACpC,OAAO,YAAY,EAAE,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACtC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC;IAE9B,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;QAClC,UAAU,EAAE,IAAI;QAEhB,SAAS,CAAC,KAAU,EAAE,QAAgB,EAAE,QAA2B;YAC/D,4DAA4D;YAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAA0B,CAAC,CAAC;YAE/F,wEAAwE;YACxE,MAAM,eAAe,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAE9C,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnB,wBAAwB;gBACxB,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACJ,QAAQ,EAAE,CAAC;YACf,CAAC;QACL,CAAC;QACD,KAAK,CAAC,QAA2B;YAC7B,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACrC,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE;QACzC,MAAM,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,IAAI,kBAAkB,CAAC,CAAC;QAEzD,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACxB,MAAM,CAAC,KAAK,CAAC,0BAA0B,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAClB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,KAAK,CAAC,gBAAgB,MAAM,CAAC,IAAI,iBAAiB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1E,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/C,IAAI,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACvB,MAAM,CAAC,KAAK,CAAC,uBAAuB,MAAM,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;YACzD,IAAI,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC1C,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;IACvD,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACxB,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;QACxC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,OAAO,eAAe,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,GAAQ;IAChC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IACzB,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC;IACzB,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,GAAQ;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,OAAO,CAAC,QAAQ,GAAG,EAAE,CAAC;IACtB,OAAO,CAAC,QAAQ,GAAG,EAAE,CAAC;IACtB,OAAO,OAAO,CAAC;AACnB,CAAC"}

package/dist/util/xml_escape.d.ts

@@ -0,0 +1,5 @@
+export declare function needsEscape(s: string, forAttribute: boolean): boolean;
+export declare function escapeXML(value: string | number | null | undefined, options?: {
+    forAttribute?: boolean;
+}): string;
+//# sourceMappingURL=xml_escape.d.ts.map

package/dist/util/xml_escape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"xml_escape.d.ts","sourceRoot":"","sources":["../../src/util/xml_escape.ts"],"names":[],"mappings":"AAQA,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,GAAG,OAAO,CAYrE;AAED,wBAAgB,SAAS,CACrB,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,EACzC,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,OAAO,CAAA;CAAE,GACrC,MAAM,CAqCR"}

package/dist/util/xml_escape.js

@@ -0,0 +1,58 @@
+// XML text/attribute escaping shared by the MARCXML exporter (src/output/xml.ts)
+// and the marc_xml fix (src/fix/fixes.ts). Strips characters disallowed in XML
+// 1.0 and escapes the markup-significant ones.
+// True if the string contains any character escapeXML would change: & < >
+// (and " ' for attributes), disallowed XML 1.0 control chars, surrogates, or
+// non-characters. A single char-code scan; lets the common (clean) case skip
+// the 5-7 regex replaces below. No literal control chars in source.
+export function needsEscape(s, forAttribute) {
+    for (let i = 0; i < s.length; i++) {
+        const c = s.charCodeAt(i);
+        if (c === 38 || c === 60 || c === 62)
+            return true; // & < >
+        if (forAttribute && (c === 34 || c === 39))
+            return true; // " '
+        if (c <= 0x08 || c === 0x0B || c === 0x0C || (c >= 0x0E && c <= 0x1F))
+            return true; // ctrl
+        if (c >= 0x7F && c <= 0x9F)
+            return true;
+        if (c >= 0xD800 && c <= 0xDFFF)
+            return true; // surrogates
+        if (c >= 0xFDD0 && c <= 0xFDEF)
+            return true; // non-chars
+        if (c === 0xFFFE || c === 0xFFFF)
+            return true;
+    }
+    return false;
+}
+export function escapeXML(value, options) {
+    if (value === null || value === undefined)
+        return '';
+    let s = String(value);
+    // Fast path: most values are clean, so skip the regex replaces below.
+    if (!needsEscape(s, options?.forAttribute === true))
+        return s;
+    // STEP 1: Remove/replace invalid UTF-8 and disallowed XML characters
+    // Remove unpaired UTF-16 surrogates (invalid in JSON and problematic in XML)
+    s = s.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])/g, '\uFFFD'); // unpaired high surrogates
+    s = s.replace(/(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g, '\uFFFD'); // unpaired low surrogates
+    // Remove other disallowed XML 1.0 characters:
+    // Control chars: 0x00-0x08, 0x0B, 0x0C, 0x0E-0x1F, 0x7F-0x9F
+    // Non-characters: 0xFFFE, 0xFFFF, 0x1FFFE, 0x1FFFF, etc.
+    s = s.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\uFFFE\uFFFF]/g, '');
+    // Remove other non-characters (U+FFFE, U+FFFF in other planes)
+    // These can occur as U+FFFE, U+FFFF, U+1FFFE, U+1FFFF, etc.
+    s = s.replace(/[\uFDD0-\uFDEF]/g, '');
+    // STEP 2: Escape XML special characters
+    // IMPORTANT: Always escape & FIRST to avoid double-escaping
+    s = s.replace(/&/g, '&amp;');
+    // Escape other special characters
+    s = s.replace(/</g, '&lt;');
+    s = s.replace(/>/g, '&gt;');
+    if (options?.forAttribute) {
+        s = s.replace(/"/g, '&quot;');
+        s = s.replace(/'/g, '&apos;');
+    }
+    return s;
+}
+//# sourceMappingURL=xml_escape.js.map

package/dist/util/xml_escape.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"xml_escape.js","sourceRoot":"","sources":["../../src/util/xml_escape.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,+EAA+E;AAC/E,+CAA+C;AAE/C,0EAA0E;AAC1E,6EAA6E;AAC7E,6EAA6E;AAC7E,oEAAoE;AACpE,MAAM,UAAU,WAAW,CAAC,CAAS,EAAE,YAAqB;IACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC,CAAiB,QAAQ;QAC3E,IAAI,YAAY,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC,CAAW,MAAM;QACzE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,OAAO;QAC3F,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,aAAa;QAChF,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAuB,YAAY;QAC/E,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;IAClD,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,SAAS,CACrB,KAAyC,EACzC,OAAoC;IAEpC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAErD,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,sEAAsE;IACtE,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAE9D,qEAAqE;IAErE,6EAA6E;IAC7E,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC,CAAE,2BAA2B;IAC5F,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,sCAAsC,EAAE,QAAQ,CAAC,CAAC,CAAC,0BAA0B;IAE3F,8CAA8C;IAC9C,6DAA6D;IAC7D,yDAAyD;IACzD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,oEAAoE,EAAE,EAAE,CAAC,CAAC;IAExF,+DAA+D;IAC/D,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAEtC,wCAAwC;IACxC,4DAA4D;IAC5D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE7B,kCAAkC;IAClC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE5B,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QACxB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,CAAC;AACb,CAAC"}

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "marcattacks",
-  "version": "2.8.0",
-  "description": "A powerful streaming MARC21 to RDF converter with JSONata transformation and S3 support.",
+  "version": "2.9.0",
+  "description": "A powerful streaming MARC21 to RDF converter with JSONata, Catmandu Fix transformation and S3 support.",
   "keywords": [
     "marc21",
     "iso2709",
     "rdf",
     "bibframe",
     "jsonata",
+    "catmandu",
     "notation3",
     "linked-data",
     "marcxml",