marcattacks 2.0.0 → 2.2.0

package/dist/input/alephseq.js

@@ -1,86 +1,56 @@
-import { Readable } from "stream";
-import * as readline from 'node:readline';
+import { Transform } from "stream";
 import log4js from 'log4js';
 const logger = log4js.getLogger();
-export async function stream2readable(stream, _opts) {
+export async function transform(_opts) {
     let recordNum = 0;
-    let hasError = false;
-    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
-    let sourcePaused = false;
-    const readableStream = new Readable({
-        read() {
-            if (sourcePaused) {
-                logger.debug("backpressure off");
-                rl.resume();
-                sourcePaused = false;
-            }
-        },
-        destroy() {
-            stream.destroy();
-        },
-        objectMode: true
-    });
     let rec = [];
     let previd = "";
-    rl.on('line', (line) => {
-        if (hasError)
-            return;
-        logger.debug(line);
-        if (line.match(/^\w+\s[\x20-\x7E]{5}\sL\s.*/u)) {
-            // ok
-        }
-        else {
-            logger.error(`syntax error in record ${recordNum + 1}`);
-            hasError = true;
-            stream.destroy();
-            rl.close();
-            readableStream.destroy(new Error(String('parse error')));
-            return;
-        }
-        const [id, ...rest] = line.split(" ");
-        const data = rest.join(" ");
-        if (previd && previd !== id) {
-            const ok = readableStream.push({
-                record: rec
-            });
-            if (!ok) {
-                logger.debug("backpressure on");
-                rl.pause();
-                sourcePaused = true;
+    let tail = "";
+    return new Transform({
+        objectMode: true,
+        transform(chunk, _encoding, callback) {
+            const data = tail + chunk.toString();
+            const lines = data.split(/\r?\n/);
+            tail = lines.pop() || "";
+            for (const line of lines) {
+                if (line.length === 0)
+                    continue;
+                if (!line.match(/^\w+\s[\x20-\x7E]{5}\sL\s.*/u)) {
+                    const err = new Error(`syntax error in record ${recordNum + 1}`);
+                    logger.error(err.message);
+                    return callback(err);
+                }
+                const [id, ...rest] = line.split(" ");
+                const lineData = rest.join(" ");
+                if (previd && previd !== id) {
+                    this.push({ record: rec });
+                    rec = [];
+                    recordNum++;
+                }
+                const tag = lineData?.substring(0, 3);
+                const ind1 = lineData?.substring(3, 4);
+                const ind2 = lineData?.substring(4, 5);
+                const sf = lineData?.substring(8);
+                const parts = sf.split(/\$\$(.)/);
+                if (tag === 'FMT' || tag === 'LDR' || tag.startsWith("00")) {
+                    rec.push([tag, ind1, ind2, "_", ...parts]);
+                }
+                else {
+                    rec.push([tag, ind1, ind2, ...parts.slice(1)]);
+                }
+                previd = id;
             }
-            rec = [];
-            recordNum++;
-        }
-        const tag = data?.substring(0, 3);
-        const ind1 = data?.substring(3, 4);
-        const ind2 = data?.substring(4, 5);
-        const sf = data?.substring(8);
-        const parts = sf.split(/\$\$(.)/);
-        if (tag == 'FMT' || tag === 'LDR' || tag.startsWith("00")) {
-            rec.push([
-                tag, ind1, ind2
-            ].concat(["_"].concat(parts)));
-        }
-        else {
-            rec.push([
-                tag, ind1, ind2
-            ].concat(parts.slice(1)));
+            callback();
+        },
+        flush(callback) {
+            if (tail) {
+                logger.warn("ignoring partial chunk of data: ", tail);
+            }
+            if (rec.length > 0) {
+                this.push({ record: rec });
+            }
+            callback();
         }
-        previd = id;
-    });
-    rl.on('error', (error) => {
-        if (hasError)
-            return;
-        logger.error(`readline error ${error}`);
-    });
-    rl.on('close', () => {
-        if (hasError)
-            return;
-        readableStream.push({
-            record: rec
-        });
-        readableStream.push(null);
     });
-    return readableStream;
 }
 //# sourceMappingURL=alephseq.js.map

package/dist/input/alephseq.js.map

@@ -1 +1 @@
-{"version":3,"file":"alephseq.js","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AACzC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAC,CAAC,CAAC;IAE1E,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,EAAE,CAAC,MAAM,EAAE,CAAC;gBACZ,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,IAAI,GAAG,GAAgB,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAY,EAAE,CAAC;IAEzB,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,QAAQ;YAAE,OAAO;QAErB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEnB,IAAI,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;YAC7C,KAAK;QACT,CAAC;aACI,CAAC;YACF,MAAM,CAAC,KAAK,CAAC,0BAA0B,SAAS,GAAG,CAAC,EAAE,CAAC,CAAC;YACxD,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,cAAc,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACzD,OAAO;QACX,CAAC;QAED,MAAM,CAAC,EAAE,EAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5B,IAAI,MAAM,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC;gBAC3B,MAAM,EAAG,GAAG;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,GAAG,GAAG,EAAE,CAAC;YACT,SAAS,EAAE,CAAC;QAChB,CAAC;QAED,MAAM,GAAG,GAAI,IAAI,EAAE,SAAS,CAAC,CAAC,EAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,EAAE,SAAS,CAAC,CAAC,EAAC,CAAC,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,EAAE,SAAS,CAAC,CAAC,EAAC,CAAC,CAAC,CAAC;QAClC,MAAM,EAAE,GAAK,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAElC,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,IAAI,CAAC;gBACL,GAAG,EAAC,IAAI,EAAC,IAAI;aAChB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;aACI,CAAC;YACF,GAAG,CAAC,IAAI,CAAC;gBACL,GAAG,EAAC,IAAI,EAAC,IAAI;aAChB,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,GAAG,EAAG,CAAC;IACjB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;QACrB,IAAI,QAAQ;YAAE,OAAO;QACrB,MAAM,CAAC,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAChB,IAAI,QAAQ;YAAE,OAAO;QACrB,cAAc,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,GAAG;SACd,CAAC,CAAC;QACH,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
+{"version":3,"file":"alephseq.js","sourceRoot":"","sources":["../../src/input/alephseq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,GAAG,GAAe,EAAE,CAAC;IACzB,IAAI,MAAM,GAAW,EAAE,CAAC;IACxB,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAEhB,SAAS,CAAC,KAAU,EAAE,SAAiB,EAAE,QAA2B;YAChE,MAAM,IAAI,GAAG,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAElC,IAAI,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;YAEzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEhC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;oBAC9C,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,0BAA0B,SAAS,GAAG,CAAC,EAAE,CAAC,CAAC;oBACjE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBAC1B,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;gBAED,MAAM,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAEhC,IAAI,MAAM,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;oBAC1B,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC3B,GAAG,GAAG,EAAE,CAAC;oBACT,SAAS,EAAE,CAAC;gBAChB,CAAC;gBAED,MAAM,GAAG,GAAI,QAAQ,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvC,MAAM,IAAI,GAAG,QAAQ,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvC,MAAM,IAAI,GAAG,QAAQ,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvC,MAAM,EAAE,GAAK,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAElC,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzD,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC;gBAC/C,CAAC;qBAAM,CAAC;oBACJ,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnD,CAAC;gBAED,MAAM,GAAG,EAAG,CAAC;YACjB,CAAC;YAED,QAAQ,EAAE,CAAC;QACf,CAAC;QAED,KAAK,CAAC,QAA2B;YAC7B,IAAI,IAAI,EAAE,CAAC;gBACP,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE,IAAI,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjB,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAC/B,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}

package/dist/input/json.d.ts

@@ -1,3 +1,3 @@
-import { Readable } from "stream";
-export declare function stream2readable(stream: Readable, _opts: any): Promise<Readable>;
+import { Transform } from "stream";
+export declare function transform(_opts: any): Promise<Transform>;
 //# sourceMappingURL=json.d.ts.map

package/dist/input/json.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CAkCtF"}
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAG3D,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAsB9D"}

package/dist/input/json.js

@@ -1,34 +1,22 @@
-import { Readable } from "stream";
+import { Transform } from "stream";
 import streamArray from "stream-json/streamers/StreamArray.js";
-import log4js from 'log4js';
-const logger = log4js.getLogger();
-export async function stream2readable(stream, _opts) {
-    const pipeline = stream.pipe(streamArray.withParser());
-    let sourcePaused = false;
-    const readableStream = new Readable({
-        read() {
-            if (sourcePaused) {
-                logger.debug("backpressure off");
-                pipeline.resume();
-                sourcePaused = false;
-            }
+export async function transform(_opts) {
+    const jsonParser = streamArray.withParser();
+    const transformer = new Transform({
+        objectMode: true,
+        transform(chunk, encoding, callback) {
+            jsonParser.write(chunk, encoding);
+            callback();
         },
-        destroy() {
-            stream.destroy();
-        },
-        objectMode: true
-    });
-    pipeline.on('data', (data) => {
-        const ok = readableStream.push(data.value);
-        if (!ok) {
-            logger.debug("backpressure on");
-            pipeline.pause();
-            sourcePaused = true;
+        flush(callback) {
+            jsonParser.end();
+            callback();
         }
     });
-    pipeline.on('end', () => {
-        readableStream.push(null);
+    jsonParser.on('data', (data) => {
+        transformer.push(data.value);
     });
-    return readableStream;
+    jsonParser.on('error', (err) => transformer.destroy(err));
+    return transformer;
 }
 //# sourceMappingURL=json.js.map

package/dist/input/json.js.map

@@ -1 +1 @@
-{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,WAAW,MAAM,sCAAsC,CAAC;AAC/D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC;IAEvD,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;QAC9B,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE3C,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;YAC/B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/input/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,WAAW,MAAM,sCAAsC,CAAC;AAE/D,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,MAAM,UAAU,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC;IAE5C,MAAM,WAAW,GAAG,IAAI,SAAS,CAAC;QAC9B,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,QAAgB,EAAE,QAA2B;YAC/D,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClC,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,UAAU,CAAC,GAAG,EAAE,CAAC;YACjB,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;QAChC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAQ,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IAE/D,OAAO,WAAW,CAAC;AACvB,CAAC"}

package/dist/input/jsonl.d.ts

@@ -1,3 +1,3 @@
-import { Readable } from "stream";
-export declare function stream2readable(stream: Readable, _opts: any): Promise<Readable>;
+import { Transform } from "stream";
+export declare function transform(_opts: any): Promise<Transform>;
 //# sourceMappingURL=jsonl.d.ts.map

package/dist/input/jsonl.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CAyDtF"}
+{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAK3D,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CA+B9D"}

package/dist/input/jsonl.js

@@ -1,56 +1,37 @@
-import { Readable } from "stream";
-import * as readline from 'node:readline';
+import { Transform } from "stream";
 import log4js from 'log4js';
 const logger = log4js.getLogger();
-export async function stream2readable(stream, _opts) {
+export async function transform(_opts) {
     let recordNum = 0;
-    let hasError = false;
-    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
-    let sourcePaused = false;
-    const readableStream = new Readable({
-        read() {
-            if (sourcePaused) {
-                logger.debug("backpressure off");
-                rl.resume();
-                sourcePaused = false;
+    let tail = "";
+    return new Transform({
+        objectMode: true,
+        transform(chunk, _encoding, callback) {
+            const lines = (tail + chunk.toString()).split(/\r?\n/);
+            tail = lines.pop() || "";
+            for (const line of lines) {
+                if (!line.trim())
+                    continue;
+                try {
+                    this.push(JSON.parse(line));
+                    recordNum++;
+                }
+                catch (error) {
+                    logger.error(`JSON parse error at line ${recordNum + 1}: ${error}`);
+                    return callback(error instanceof Error ? error : new Error(String(error)));
+                }
             }
+            callback();
         },
-        destroy() {
-            stream.destroy();
-        },
-        objectMode: true
-    });
-    rl.on('line', (line) => {
-        if (hasError)
-            return;
-        recordNum++;
-        try {
-            const ok = readableStream.push(JSON.parse(line));
-            if (!ok) {
-                logger.debug("backpressure on");
-                rl.pause();
-                sourcePaused = true;
+        flush(callback) {
+            if (tail.trim()) {
+                try {
+                    this.push(JSON.parse(tail));
+                }
+                catch (e) { /* ignore trailing whitespace */ }
             }
-            recordNum++;
-        }
-        catch (error) {
-            hasError = true;
-            logger.error(`JSON parse error at line ${recordNum + 1}: ${error}`);
-            stream.destroy();
-            rl.close();
-            readableStream.destroy(error instanceof Error ? error : new Error(String(error)));
+            callback();
         }
     });
-    rl.on('error', (error) => {
-        if (hasError)
-            return;
-        logger.error(`readline error ${error}`);
-    });
-    rl.on('close', () => {
-        if (hasError)
-            return;
-        readableStream.push(null);
-    });
-    return readableStream;
 }
 //# sourceMappingURL=jsonl.js.map

package/dist/input/jsonl.js.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAA;AACzC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAC,CAAC,CAAC;IAE1E,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,EAAE,CAAC,MAAM,EAAE,CAAC;gBACZ,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QACnB,IAAI,QAAQ;YAAE,OAAO;QAErB,SAAS,EAAE,CAAC;QAEZ,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;YAEjD,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBAChC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,YAAY,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,SAAS,EAAE,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,4BAA4B,SAAS,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;YACpE,MAAM,CAAC,OAAO,EAAE,CAAC;YACjB,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,cAAc,CAAC,OAAO,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACtF,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;QACrB,IAAI,QAAQ;YAAE,OAAO;QACrB,MAAM,CAAC,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAChB,IAAI,QAAQ;YAAE,OAAO;QACrB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AAC1B,CAAC"}
+{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../src/input/jsonl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAiB,EAAG,QAA2B;YACjE,MAAM,KAAK,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACvD,IAAI,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;YAEzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBAAE,SAAS;gBAC3B,IAAI,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC5B,SAAS,EAAE,CAAC;gBAChB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,MAAM,CAAC,KAAK,CAAC,4BAA4B,SAAS,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;oBACpE,OAAO,QAAQ,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC/E,CAAC;YACL,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACd,IAAI,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gBAChC,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC,CAAC,gCAAgC,CAAC,CAAC;YACpD,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}

package/dist/input/marc.d.ts

@@ -1,3 +1,3 @@
-import { Readable } from "stream";
-export declare function stream2readable(stream: Readable, _opts: any): Promise<Readable>;
+import { Transform } from "stream";
+export declare function transform(_opts: any): Promise<Transform>;
 //# sourceMappingURL=marc.d.ts.map

package/dist/input/marc.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"marc.d.ts","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAMlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CA4EtF"}
+{"version":3,"file":"marc.d.ts","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAM3D,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAgD9D"}

package/dist/input/marc.js

@@ -1,30 +1,21 @@
-import { Readable } from "stream";
+import { Transform } from "stream";
 import { Marc } from "marcjs";
 import log4js from 'log4js';
 const logger = log4js.getLogger();
-export async function stream2readable(stream, _opts) {
-    let recordNum = 0;
-    let hasError = false;
-    let sourcePaused = false;
+export async function transform(_opts) {
     const parser = Marc.createStream('Iso2709', 'Parser');
-    const readableStream = new Readable({
-        read() {
-            if (sourcePaused) {
-                logger.debug("backpressure off");
-                parser.resume();
-                sourcePaused = false;
-            }
-        },
-        destroy() {
-            stream.destroy();
+    const transformer = new Transform({
+        objectMode: true,
+        transform(chunk, _encoding, callback) {
+            parser.write(chunk); //
+            callback();
         },
-        objectMode: true
+        flush(callback) {
+            parser.end(); //
+            callback();
+        }
     });
     parser.on('data', (record) => {
-        recordNum++;
-        if (recordNum % 1000 === 0) {
-            logger.info(`record: ${recordNum}`);
-        }
         let rec = [];
         rec.push(['LDR', ' ', ' ', '_', record.leader]);
         for (const field of record.fields) {
@@ -33,35 +24,25 @@ export async function stream2readable(stream, _opts) {
                 const ind1 = ' ';
                 const ind2 = ' ';
                 const data = field.slice(1);
-                rec.push([tag, ind1, ind2].concat(data));
+                rec.push([tag, ind1, ind2].concat(data)); //
             }
             else if (field.length > 3) {
                 const ind1 = field[1].charAt(0);
                 const ind2 = field[1].charAt(1);
                 const data = field.slice(2);
-                rec.push([tag, ind1, ind2].concat(data));
+                rec.push([tag, ind1, ind2].concat(data)); //
             }
             else {
-                logger.warn("marc error: ", field);
+                logger.warn("marc error: ", field); //
             }
         }
-        const ok = readableStream.push({
+        transformer.push({
             record: rec
         });
-        if (!ok) {
-            logger.debug("backpressure on");
-            parser.pause();
-            sourcePaused = true;
-        }
     });
-    parser.on('close', () => {
-        if (recordNum % 1000 === 0) {
-            logger.info(`record: ${recordNum}`);
-        }
-        readableStream.push(null);
-        logger.info(`processed ${recordNum} records`);
+    parser.on('error', (err) => {
+        transformer.destroy(err);
     });
-    stream.pipe(parser);
-    return readableStream;
+    return transformer;
 }
 //# sourceMappingURL=marc.js.map

package/dist/input/marc.js.map

@@ -1 +1 @@
-{"version":3,"file":"marc.js","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,KAAU;IAC9D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEtD,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,IAAI;YACA,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,MAAM,CAAC,MAAM,EAAE,CAAC;gBAChB,YAAY,GAAG,KAAK,CAAC;YACzB,CAAC;QACL,CAAC;QACD,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;QACD,UAAU,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE;QACzB,SAAS,EAAE,CAAC;QAEZ,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,GAAG,GAAgB,EAAE,CAAC;QAE1B,GAAG,CAAC,IAAI,CAAC,CAAE,KAAK,EAAG,GAAG,EAAG,GAAG,EAAG,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAEpD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,GAAG,GAAI,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,GAAG,CAAC;gBACjB,MAAM,IAAI,GAAG,GAAG,CAAC;gBACjB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAE5B,GAAG,CAAC,IAAI,CAAC,CAAE,GAAG,EAAG,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC;iBACI,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAE5B,GAAG,CAAC,IAAI,CAAC,CAAE,GAAG,EAAG,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,CAAC;iBACI,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QAED,MAAM,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC;YAC3B,MAAM,EAAG,GAAG;SACf,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,EAAE,CAAC;YACN,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAChC,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACpB,IAAI,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,aAAa,SAAS,UAAU,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEpB,OAAO,cAAc,CAAC;AAC1B,CAAC"}
+{"version":3,"file":"marc.js","sourceRoot":"","sources":["../../src/input/marc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEtD,MAAM,WAAW,GAAG,IAAI,SAAS,CAAC;QAC9B,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAiB,EAAE,QAA2B;YAChE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YACvB,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,KAAK,CAAC,QAA2B;YAC7B,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE;YAChB,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE;QACzB,IAAI,GAAG,GAAe,EAAE,CAAC;QAEzB,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAEhD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAErB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,GAAG,CAAC;gBACjB,MAAM,IAAI,GAAG,GAAG,CAAC;gBACjB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC5B,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;YAChD,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC5B,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;YAChD,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE;YAC1C,CAAC;QACL,CAAC;QAED,WAAW,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,GAAG;SACd,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACvB,CAAC"}

package/dist/input/rdf.d.ts

@@ -1,3 +1,3 @@
-import { Readable } from 'stream';
-export declare function stream2readable(stream: Readable, opts: any): Promise<Readable>;
+import { Transform } from 'stream';
+export declare function transform(opts: any): Promise<Transform>;
 //# sourceMappingURL=rdf.d.ts.map

package/dist/input/rdf.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rdf.d.ts","sourceRoot":"","sources":["../../src/input/rdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AASlC,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,GAAI,OAAO,CAAC,QAAQ,CAAC,CAcrF"}
+{"version":3,"file":"rdf.d.ts","sourceRoot":"","sources":["../../src/input/rdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAO3D,wBAAsB,SAAS,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CA8B7D"}

package/dist/input/rdf.js

@@ -1,18 +1,31 @@
-import { Readable } from 'stream';
-import { rdfParser } from "rdf-parse";
+import { Transform } from 'stream';
 import { parseStream } from "../util/rdf_parse.js";
 import log4js from 'log4js';
 const logger = log4js.getLogger();
-export async function stream2readable(stream, opts) {
-    const readableStream = new Readable({
+export async function transform(opts) {
+    const chunks = []; // Collect chunks in an array instead of a stream
+    return new Transform({
         objectMode: true,
-        destroy() {
-            stream.destroy();
+        transform(chunk, encoding, callback) {
+            logger.debug('chunk received');
+            chunks.push(chunk);
+            callback();
+        },
+        async flush(callback) {
+            try {
+                logger.debug('flush started');
+                const fullBuffer = Buffer.concat(chunks);
+                const { Readable } = await import('stream');
+                const finalStream = Readable.from(fullBuffer);
+                const record = await parseStream(finalStream, opts.path.href);
+                this.push(record);
+                callback();
+            }
+            catch (error) {
+                logger.error(`RDF parsing error: ${error}`);
+                callback(error instanceof Error ? error : new Error(String(error)));
+            }
         }
     });
-    const record = await parseStream(stream, opts.path.href);
-    readableStream.push(record);
-    readableStream.push(null);
-    return readableStream;
 }
 //# sourceMappingURL=rdf.js.map

package/dist/input/rdf.js.map

@@ -1 +1 @@
-{"version":3,"file":"rdf.js","sourceRoot":"","sources":["../../src/input/rdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAgB,EAAE,IAAS;IAC7D,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC;QAChC,UAAU,EAAE,IAAI;QAChB,OAAO;YACH,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,CAAC;KACJ,CAAC,CAAC;IAEH,MAAM,MAAM,GAAY,MAAM,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAElE,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE1B,OAAO,cAAc,CAAC;AAC1B,CAAC"}
+{"version":3,"file":"rdf.js","sourceRoot":"","sources":["../../src/input/rdf.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAS;IACrC,MAAM,MAAM,GAAU,EAAE,CAAC,CAAC,iDAAiD;IAE3E,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAEhB,SAAS,CAAC,KAAU,EAAE,QAAgB,EAAE,QAA2B;YAC/D,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,QAAQ,EAAE,CAAC;QACf,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,QAA2B;YACnC,IAAI,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBAC9B,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAEzC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC5C,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE9C,MAAM,MAAM,GAAW,MAAM,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEtE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAClB,QAAQ,EAAE,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;gBAC5C,QAAQ,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACxE,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}

package/dist/{transform/rdf.d.ts → input/tsv.d.ts}

@@ -1,3 +1,3 @@
 import { Transform } from "stream";
 export declare function transform(_opts: any): Promise<Transform>;
-//# sourceMappingURL=rdf.d.ts.map
+//# sourceMappingURL=tsv.d.ts.map

package/dist/input/tsv.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tsv.d.ts","sourceRoot":"","sources":["../../src/input/tsv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAK3D,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAgD9D"}

package/dist/input/tsv.js

@@ -0,0 +1,45 @@
+import { Transform } from "stream";
+import log4js from 'log4js';
+const logger = log4js.getLogger();
+export async function transform(_opts) {
+    let recordNum = 0;
+    let tail = "";
+    let keys;
+    return new Transform({
+        objectMode: true,
+        transform(chunk, _encoding, callback) {
+            const lines = (tail + chunk.toString()).split(/\r?\n/);
+            tail = lines.pop() || "";
+            for (const line of lines) {
+                recordNum++;
+                if (!line.trim())
+                    continue;
+                const fields = line.split("\t");
+                if (!keys) {
+                    keys = fields;
+                    continue;
+                }
+                if (keys.length != fields.length) {
+                    logger.error(`Error on line ${recordNum}, unexpected columns`);
+                    continue;
+                }
+                let data = {};
+                for (let i = 0; i < keys.length; i++) {
+                    data[keys[i]] = fields[i];
+                }
+                this.push(data);
+            }
+            callback();
+        },
+        flush(callback) {
+            if (tail.trim()) {
+                try {
+                    this.push(JSON.parse(tail));
+                }
+                catch (e) { /* ignore trailing whitespace */ }
+            }
+            callback();
+        }
+    });
+}
+//# sourceMappingURL=tsv.js.map

package/dist/input/tsv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tsv.js","sourceRoot":"","sources":["../../src/input/tsv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAU;IACtC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,IAAe,CAAC;IAEpB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,KAAU,EAAE,SAAiB,EAAG,QAA2B;YACjE,MAAM,KAAK,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACvD,IAAI,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;YAEzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,SAAS,EAAE,CAAC;gBAEZ,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBAAE,SAAS;gBAE3B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAEhC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACR,IAAI,GAAG,MAAM,CAAC;oBACd,SAAS;gBACb,CAAC;gBAED,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC/B,MAAM,CAAC,KAAK,CAAC,iBAAiB,SAAS,sBAAsB,CAAC,CAAC;oBAC/D,SAAS;gBACb,CAAC;gBAED,IAAI,IAAI,GAAS,EAAE,CAAC;gBAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAG,CAAC,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC/B,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;YAED,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACd,IAAI,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;gBAChC,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC,CAAC,gCAAgC,CAAC,CAAC;YACpD,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}

package/dist/input/xml.d.ts

@@ -1,3 +1,3 @@
-import { Readable } from 'stream';
-export declare function stream2readable(stream: Readable): Promise<Readable>;
+import { Transform } from 'stream';
+export declare function transform(_opts: any): Promise<Transform>;
 //# sourceMappingURL=xml.d.ts.map

package/dist/input/xml.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../src/input/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAU5C,wBAAsB,eAAe,CAAC,MAAM,EAAE,QAAQ,GAAI,OAAO,CAAC,QAAQ,CAAC,CAgJ1E"}
+{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../src/input/xml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAS3D,wBAAsB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAqE9D"}