marcattacks 2.5.0 → 2.7.0

package/dist/worker.js

@@ -0,0 +1,33 @@
+// Worker thread: runs a parallelizable map (e.g. jsonata, fix) on batches of
+// records. The heavy per-record work (jsonata evaluate / fix chain) runs here;
+// the main thread does I/O, parsing and serialization.
+//
+// Protocol:
+//   main -> worker : { seq, batch: record[] }
+//   worker -> main : { ready: true }                         (once, on startup)
+//                    { seq, mapped: (record|null)[] }        (null = rejected)
+//                    { seq, error: string }
+import { parentPort, workerData } from 'node:worker_threads';
+import { loadPlugin } from './plugin-loader.js';
+import { REJECT } from './fix/signal.js';
+const { map, param } = workerData;
+const mod = await loadPlugin(map, 'transform');
+if (typeof mod.createMapper !== 'function') {
+    throw new Error(`map '${map}' is not parallelizable (no createMapper)`);
+}
+const mapper = await mod.createMapper(param ?? {});
+parentPort.postMessage({ ready: true });
+parentPort.on('message', async (msg) => {
+    try {
+        const mapped = [];
+        for (const rec of msg.batch) {
+            const r = await mapper(rec);
+            mapped.push(r === REJECT ? null : r); // null marks a dropped record
+        }
+        parentPort.postMessage({ seq: msg.seq, mapped });
+    }
+    catch (err) {
+        parentPort.postMessage({ seq: msg.seq, error: err?.message ?? String(err) });
+    }
+});
+//# sourceMappingURL=worker.js.map

package/dist/worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,+EAA+E;AAC/E,uDAAuD;AACvD,EAAE;AACF,YAAY;AACZ,8CAA8C;AAC9C,gFAAgF;AAChF,+EAA+E;AAC/E,4CAA4C;AAC5C,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,UAAyC,CAAC;AAEjE,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;AAC/C,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;IACzC,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,2CAA2C,CAAC,CAAC;AAC5E,CAAC;AACD,MAAM,MAAM,GAAuB,MAAM,GAAG,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;AAEvE,UAAW,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAEzC,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,GAAkC,EAAE,EAAE;IACnE,IAAI,CAAC;QACD,MAAM,MAAM,GAAU,EAAE,CAAC;QACzB,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAG,8BAA8B;QAC1E,CAAC;QACD,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;IACtD,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAChB,UAAW,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;AACL,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "marcattacks",
-  "version": "2.5.0",
+  "version": "2.7.0",
   "description": "A powerful streaming MARC21 to RDF converter with JSONata transformation and S3 support.",
   "keywords": [
     "marc21",
@@ -36,6 +36,9 @@
     "demo:jsonl": "npx marcattacks --to jsonl data/sample.xml",
     "demo:rdf": "npx marcattacks --to rdf --map marc2rdf data/sample.xml",
     "demo:jsonld": "npx marcattacks --to jsonl --param fix=./demo/marc2rdf.jsonata data/sample.xml",
+    "demo:fix": "npx marcattacks --to jsonl --map fix --param fix=./demo/example.fix data/sample.xml",
+    "demo:fix:rdf": "npx marcattacks --to jsonl --map fix --param fix=./demo/marc2rdf.fix data/sample.xml",
+    "demo:workers": "npx marcattacks --to jsonl --param fix=./demo/marc2rdf.jsonata --workers 4 --info data/sample.xml",
     "demo:biblio:one": "npx marcattacks --from json --info ./data/biblio.json",
     "demo:biblio:all": "npx marcattacks --from jsonl --info https://biblio.ugent.be/exports/publications.json",
     "demo:parquet": " npx marcattacks ./data/sample.xml --to parquet --map avram --param schema=./demo/avram.schema --info --out ./data/avram.parquet",

package/dist/output/mulitipart.d.ts

@@ -1,7 +0,0 @@
-import { Transform } from 'stream';
-export interface OutputMultipartOptions {
-    header?: string;
-    delimiter?: string;
-}
-export declare function transform(opts?: OutputMultipartOptions): Promise<Transform>;
-//# sourceMappingURL=mulitipart.d.ts.map

package/dist/output/mulitipart.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"mulitipart.d.ts","sourceRoot":"","sources":["../../src/output/mulitipart.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,MAAM,WAAW,sBAAsB;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAsB,SAAS,CAAC,IAAI,GAAE,sBAA2B,GAAI,OAAO,CAAC,SAAS,CAAC,CAyDtF"}

package/dist/output/mulitipart.js

@@ -1,53 +0,0 @@
-import { Transform } from 'stream';
-import log4js from 'log4js';
-const logger = log4js.getLogger();
-export async function transform(opts = {}) {
-    let header = opts.header ? opts.header :
-        "Content-Type: multipart/mixed; boundary=\"marcattacks\"";
-    let delimiter = opts.delimiter ? opts.delimiter : "--marcattacks";
-    let isFirst = true;
-    return new Transform({
-        objectMode: true,
-        transform(data, _encoding, callback) {
-            if (Object.keys(data).length == 0) {
-                logger.debug('skipped empty record');
-                callback();
-                return;
-            }
-            let output = "";
-            if (isFirst) {
-                output += header + "\n\n";
-                isFirst = false;
-            }
-            else {
-                output += delimiter + "\n";
-            }
-            let fields = [];
-            let sortedKeys = Object.keys(data).sort();
-            sortedKeys.forEach(key => {
-                if (Array.isArray(data[key])) {
-                    fields.push(`ARRAY[${data[key].length}]`);
-                }
-                else if (typeof data[key] === 'object') {
-                    fields.push(`HASH[${Object.keys(data[key]).length}]`);
-                }
-                else {
-                    fields.push(data[key]);
-                }
-            });
-            output += fields.join(" ") + "\n";
-            logger.trace(`adding ${output.length} bytes`);
-            callback(null, output);
-        },
-        flush(callback) {
-            logger.debug('flush reached');
-            if (!isFirst) {
-                let output = delimiter + "--\n";
-                logger.trace(`adding ${output.length} bytes`);
-                this.push(output);
-            }
-            callback();
-        }
-    });
-}
-//# sourceMappingURL=mulitipart.js.map

package/dist/output/mulitipart.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"mulitipart.js","sourceRoot":"","sources":["../../src/output/mulitipart.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAOlC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAA+B,EAAE;IAC7D,IAAI,MAAM,GAAY,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC7C,yDAAyD,CAAC;IAC9D,IAAI,SAAS,GAAY,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC;IAE3E,IAAI,OAAO,GAAG,IAAI,CAAC;IAEnB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAC;gBACX,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAEhB,IAAI,OAAO,EAAE,CAAC;gBACV,MAAM,IAAI,MAAM,GAAG,MAAM,CAAC;gBAC1B,OAAO,GAAG,KAAK,CAAC;YACpB,CAAC;iBACI,CAAC;gBACF,MAAM,IAAI,SAAS,GAAG,IAAI,CAAC;YAC/B,CAAC;YAED,IAAI,MAAM,GAAc,EAAE,CAAC;YAE3B,IAAI,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YAE1C,UAAU,CAAC,OAAO,CAAE,GAAG,CAAC,EAAE;gBACtB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;oBAC3B,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC9C,CAAC;qBACI,IAAI,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC1D,CAAC;qBACI,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC3B,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;YAElC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,CAAC,QAAQ;YACV,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,IAAI,MAAM,GAAG,SAAS,GAAG,MAAM,CAAC;gBAChC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YACD,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}

package/dist/output/text.d.ts

@@ -1,6 +0,0 @@
-import { Transform } from 'stream';
-export interface OutputTextOptions {
-    newline?: "yes" | "no";
-}
-export declare function transform(opts?: OutputTextOptions): Promise<Transform>;
-//# sourceMappingURL=text.d.ts.map

package/dist/output/text.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../src/output/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAKnC,MAAM,WAAW,iBAAiB;IAC9B,OAAO,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;CAC1B;AAED,wBAAsB,SAAS,CAAC,IAAI,GAAE,iBAAsB,GAAI,OAAO,CAAC,SAAS,CAAC,CAkCjF"}

package/dist/output/text.js

@@ -1,34 +0,0 @@
-import { Transform } from 'stream';
-import log4js from 'log4js';
-const logger = log4js.getLogger();
-export async function transform(opts = {}) {
-    let newline = opts.newline ? opts.newline : "no";
-    return new Transform({
-        objectMode: true,
-        transform(data, _encoding, callback) {
-            if (Object.keys(data).length == 0) {
-                logger.debug('skipped empty record');
-                callback();
-                return;
-            }
-            let output = "";
-            let fields = [];
-            let sortedKeys = Object.keys(data).sort();
-            sortedKeys.forEach(key => {
-                if (Array.isArray(data[key])) {
-                    fields.push(`ARRAY[${data[key].length}]`);
-                }
-                else if (typeof data[key] === 'object') {
-                    fields.push(`HASH[${Object.keys(data[key]).length}]`);
-                }
-                else {
-                    fields.push(data[key]);
-                }
-            });
-            output += fields.join(delimiter) + "\n";
-            logger.trace(`adding ${output.length} bytes`);
-            callback(null, output);
-        }
-    });
-}
-//# sourceMappingURL=text.js.map

package/dist/output/text.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/output/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAMlC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAA0B,EAAE;IACxD,IAAI,OAAO,GAAY,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAE1D,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAC;gBACX,OAAO;YACX,CAAC;YAED,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,MAAM,GAAc,EAAE,CAAC;YAC3B,IAAI,UAAU,GAAc,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YAErD,UAAU,CAAC,OAAO,CAAE,GAAG,CAAC,EAAE;gBACtB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;oBAC3B,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC9C,CAAC;qBACI,IAAI,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC1D,CAAC;qBACI,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC3B,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;YAExC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,MAAM,QAAQ,CAAC,CAAC;YAC9C,QAAQ,CAAC,IAAI,EAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}