marcattacks 2.2.18 → 2.3.0

package/README.md

@@ -98,6 +98,7 @@ use `s3s://...` for using an SSL layer.
 - jsonl
 - marc (ISO2709)
 - rdf
+- csv
 - tsv
 - xml (MARCXML)
 
@@ -108,6 +109,7 @@ use `s3s://...` for using an SSL layer.
 - jsonl
 - parquet
 - rdf
+- csv
 - tsv
 - xml (MARCXML)
 

package/dist/input/csv.d.ts

@@ -0,0 +1,5 @@
+import { Transform } from "stream";
+export declare function transform(opts: {
+    delimiter?: string;
+}): Promise<Transform>;
+//# sourceMappingURL=csv.d.ts.map

package/dist/input/csv.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/input/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAG3D,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAYhF"}

package/dist/input/csv.js

@@ -0,0 +1,14 @@
+import { Transform } from "stream";
+import { parse } from "csv-parse";
+export async function transform(opts) {
+    const delimiter = opts['delimiter'] ?? ",";
+    const parser = parse({
+        delimiter,
+        columns: true, // Use first row as header keys
+        skip_empty_lines: true,
+        trim: false, // Match original behaviour — no implicit trimming
+        relax_column_count: false,
+    });
+    return parser;
+}
+//# sourceMappingURL=csv.js.map

package/dist/input/csv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/input/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,QAAQ,CAAC;AAC3D,OAAO,EAAE,KAAK,EAAmC,MAAM,WAAW,CAAC;AAEnE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAA4B;IACxD,MAAM,SAAS,GAAW,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC;IAEnD,MAAM,MAAM,GAAG,KAAK,CAAC;QACjB,SAAS;QACT,OAAO,EAAE,IAAI,EAAS,+BAA+B;QACrD,gBAAgB,EAAE,IAAI;QACtB,IAAI,EAAE,KAAK,EAAW,kDAAkD;QACxE,kBAAkB,EAAE,KAAK;KACF,CAAC,CAAC;IAE7B,OAAO,MAA8B,CAAC;AAC1C,CAAC"}

package/dist/output/csv.d.ts

@@ -0,0 +1,6 @@
+import { Transform } from 'stream';
+export declare function transform(opts: {
+    header: string;
+    delimiter?: string;
+}): Promise<Transform>;
+//# sourceMappingURL=csv.d.ts.map

package/dist/output/csv.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/output/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAMnC,wBAAsB,SAAS,CAAC,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAqDhG"}

package/dist/output/csv.js

@@ -0,0 +1,51 @@
+import { Transform } from 'stream';
+import { stringify } from 'csv-stringify';
+import log4js from 'log4js';
+const logger = log4js.getLogger();
+export async function transform(opts) {
+    const delimiter = opts['delimiter'] ?? ',';
+    let stringifier = null;
+    let sortedKeys;
+    return new Transform({
+        objectMode: true,
+        transform(data, _encoding, callback) {
+            if (Object.keys(data).length === 0) {
+                logger.debug('skipped empty record');
+                callback();
+                return;
+            }
+            // Initialise the stringifier on the first record
+            if (!stringifier) {
+                sortedKeys = Object.keys(data).sort();
+                stringifier = stringify({
+                    delimiter,
+                    columns: sortedKeys,
+                    header: opts.header !== 'no',
+                    cast: {
+                        object: (value) => Array.isArray(value)
+                            ? `ARRAY[${value.length}]`
+                            : `HASH[${Object.keys(value).length}]`,
+                    },
+                });
+                stringifier.on('data', (chunk) => this.push(chunk));
+                stringifier.on('error', (err) => this.destroy(err));
+            }
+            const record = {};
+            for (const key of sortedKeys) {
+                record[key] = data[key];
+            }
+            logger.trace(`writing record with ${sortedKeys.length} fields`);
+            stringifier.write(record, _encoding, callback);
+        },
+        flush(callback) {
+            if (stringifier) {
+                stringifier.end();
+                stringifier.once('finish', callback);
+            }
+            else {
+                callback();
+            }
+        },
+    });
+}
+//# sourceMappingURL=csv.js.map

package/dist/output/csv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/output/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACnC,OAAO,EAAE,SAAS,EAAuC,MAAM,eAAe,CAAC;AAC/E,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAA4C;IACxE,MAAM,SAAS,GAAW,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC;IAEnD,IAAI,WAAW,GAAwC,IAAI,CAAC;IAC5D,IAAI,UAAoB,CAAC;IAEzB,OAAO,IAAI,SAAS,CAAC;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,CAAC,IAAS,EAAE,SAAS,EAAE,QAAQ;YACpC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;gBACrC,QAAQ,EAAE,CAAC;gBACX,OAAO;YACX,CAAC;YAED,iDAAiD;YACjD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACf,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAEtC,WAAW,GAAG,SAAS,CAAC;oBACpB,SAAS;oBACT,OAAO,EAAE,UAAU;oBACnB,MAAM,EAAE,IAAI,CAAC,MAAM,KAAK,IAAI;oBAC5B,IAAI,EAAE;wBACF,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CACd,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;4BAChB,CAAC,CAAC,SAAS,KAAK,CAAC,MAAM,GAAG;4BAC1B,CAAC,CAAC,QAAQ,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG;qBACjD;iBAC0B,CAAC,CAAC;gBAEjC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACpD,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YACxD,CAAC;YAED,MAAM,MAAM,GAA4B,EAAE,CAAC;YAC3C,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC3B,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,uBAAuB,UAAU,CAAC,MAAM,SAAS,CAAC,CAAC;YAChE,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;QACnD,CAAC;QAED,KAAK,CAAC,QAAQ;YACV,IAAI,WAAW,EAAE,CAAC;gBACd,WAAW,CAAC,GAAG,EAAE,CAAC;gBAClB,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACJ,QAAQ,EAAE,CAAC;YACf,CAAC;QACL,CAAC;KACJ,CAAC,CAAC;AACP,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "marcattacks",
-  "version": "2.2.18",
+  "version": "2.3.0",
   "description": "A powerful streaming MARC21 to RDF converter with JSONata transformation and S3 support.",
   "keywords": [
     "marc21",
@@ -62,6 +62,8 @@
     "@dsnp/parquetjs": "^1.8.7",
     "@fast-csv/parse": "^5.0.5",
     "commander": "^14.0.2",
+    "csv-parse": "^6.2.1",
+    "csv-stringify": "^6.7.0",
     "dotenv": "^17.2.3",
     "fast-xml-parser": "^5.3.4",
     "jsonata": "^2.1.0",

package/dist/util/csv_parse.d.ts

@@ -1,3 +0,0 @@
-import { Readable } from "stream";
-export declare function parseStream(readable: Readable): Promise<Record<string, string>[]>;
-//# sourceMappingURL=csv_parse.d.ts.map

package/dist/util/csv_parse.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"csv_parse.d.ts","sourceRoot":"","sources":["../../src/util/csv_parse.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAGlC,wBAAsB,WAAW,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAqBvF"}

package/dist/util/csv_parse.js

@@ -1,22 +0,0 @@
-import { Readable } from "stream";
-import { parse } from '@fast-csv/parse';
-export async function parseStream(readable) {
-    return new Promise((resolve, reject) => {
-        const rows = [];
-        readable
-            .pipe(parse({
-            delimiter: "\t",
-            headers: true,
-            trim: true,
-            ignoreEmpty: true,
-        }))
-            .on("error", reject)
-            .on("data", (row) => {
-            rows.push(row);
-        })
-            .on("end", () => {
-            resolve(rows);
-        });
-    });
-}
-//# sourceMappingURL=csv_parse.js.map

package/dist/util/csv_parse.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"csv_parse.js","sourceRoot":"","sources":["../../src/util/csv_parse.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAkB;IAClD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,IAAI,GAA6B,EAAE,CAAC;QAE1C,QAAQ;aACL,IAAI,CACH,KAAK,CAAC;YACJ,SAAS,EAAE,IAAI;YACf,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,IAAI;YACV,WAAW,EAAE,IAAI;SAClB,CAAC,CACH;aACA,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;aACnB,EAAE,CAAC,MAAM,EAAE,CAAC,GAA2B,EAAE,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC,CAAC;aACD,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACd,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACL,CAAC"}