marcattacks 1.0.0

package/src/input/xml.ts

@@ -0,0 +1,125 @@
+import { Readable } from 'stream';
+import sax from 'sax';
+import log4js from 'log4js';
+
+type FieldAttribute = { [key: string]: string; } | { [key: string]: sax.QualifiedAttribute; };
+type SubfieldAttribute = { [key: string]: string; };
+type MARCType = 'leader' | 'control' | 'field' | 'subfield' | undefined;
+
+const logger = log4js.getLogger();
+
+export function stream2readable(stream: Readable) : Readable {
+    let recordNum = 0;
+
+    let sourcePaused = false;
+
+    const readableStream = new Readable({
+        read() {
+            if (sourcePaused) {
+                logger.debug("backpressure off");
+                stream.resume(); 
+                sourcePaused = false;
+            }
+        } ,
+        objectMode: true 
+    });
+
+    const parser = sax.createStream(true);
+
+    let record : string[][] = [];
+    let subfield : string[] = [];
+    let attrib : FieldAttribute = {};
+    let sattrib : SubfieldAttribute = {};
+    let type : MARCType;
+    let text : string = '';
+        
+    parser.on('opentag', (node: sax.Tag) => {
+        const localName = node.name.replaceAll(/^\w+:/g,'');
+
+        if (localName === 'collection') {
+            // Start collection...
+        }
+        else if (localName === 'record') {
+            // Start record...
+        }
+        else if (localName === 'leader') {
+            type = 'leader';
+        }
+        else if (localName == 'controlfield') {
+            type = 'control';
+            attrib = node.attributes
+        }
+        else if (localName === 'datafield') {
+            attrib = node.attributes;
+        }
+        else if (localName === 'subfield') {
+            sattrib = node.attributes;
+        }
+        else {
+            logger.error(`unknown tag: ${node.name}`);
+        }
+
+        text = '';
+    });
+
+    parser.on('text', (t: string) => {
+        text += t;
+    });
+
+    parser.on('closetag', (tag: string) => {
+        const localName = tag.replaceAll(/^\w+:/g,'');
+        if (localName === 'leader') {
+            record.push(['LDR',' ',' ','_',text]);
+        }
+        else if (localName == 'controlfield') {
+            let tag = attrib.tag as string;
+            record.push([tag,' ',' ','_',text]);
+        }
+        else if (localName === 'datafield') {
+            let tag  = attrib.tag as string;
+            let ind1 = attrib.ind1 as string;
+            let ind2 = attrib.ind2 as string;
+            record.push([tag,ind1,ind2].concat(subfield));
+            subfield = [];
+        }
+        else if (localName === 'subfield') {
+            let code = sattrib.code as string;
+            subfield = subfield.concat([code,text]);
+        }
+        if (localName === 'record') {
+            const ok = readableStream.push({ record });
+            
+            if (!ok) {
+                logger.debug("backpressure on");
+                stream.pause();
+                sourcePaused = true;
+            }
+
+            record = [];
+            recordNum++;
+
+            if (recordNum % 1000 === 0) {
+                logger.info(`record: ${recordNum}`);
+            }
+        }
+    }); 
+
+    parser.on("error", (err) => {
+        logger.error ("Parser error:", err);
+    });
+
+    parser.on('end', () => {
+        const ok = readableStream.push(null);
+        if (!ok) {
+            logger.debug("backpressure on");
+            stream.pause();
+            sourcePaused = true;
+        }
+        logger.info(`processed ${recordNum} records`);
+        readableStream.destroy();
+    });
+
+    stream.pipe(parser);
+
+    return readableStream;
+}   

package/src/marcmap.ts

@@ -0,0 +1,94 @@
+interface MARCMapOpts {
+    join_char?: string;
+}
+
+/**
+ * Given a marc row return the marc tag
+ */
+export function marctag(row: string[] | undefined) : string {
+    if (!row) {
+        return "";
+    }
+    return row[0] ? row[0] : "";
+}
+
+/**
+ * Given a marc row return an array with the indicators
+ */
+export function marcind(row: string[] | undefined) : string[] {
+    if (!row) {
+        return [" "," "];
+    }
+    let ind1 = row[1] ?? " ";
+    let ind2 = row[2] ?? " ";
+    return [ind1,ind2];
+}
+
+/**
+ * Given a marc and a callback function, call function(tag,row)
+ * for each tag
+ */
+export function marcForTag(rec: string[][] | undefined , fun: (tag:string, row:string[]) => void) : void {
+    if (!rec) return;
+    for (let i = 0 ; i < rec.length ; i++) {
+        let row = rec[i] ?? [];
+        let tag = row[0] ?? "---";
+        
+        if (tag !== undefined && row !== undefined) {
+            fun(tag,row);
+        }
+    }
+}
+
+/**
+ * Given a marc row and a callback function, call function(code,value)
+ * for each subfield
+ */
+export function marcForEachSub(row: string[] | undefined , fun: (code:string, value:string) => void) : void {
+    if (!row) return;
+    for (let i = 3 ; i < row.length ; i +=2) {
+        let code = row[i];
+        let value = row[i+1];
+        
+        if (code !== undefined && value !== undefined) {
+            fun(code, value);
+        }
+    }
+}
+
+/**
+ * Given an marc record and a field-path return a string[] with all matching values
+ */
+export function marcmap(record: string[][], find: string, opts: MARCMapOpts) : string[] {
+    const fullOpts = {
+        join_char: opts.join_char ?? " "
+    };
+
+    const results : string[] = [];
+
+    const tagName = find.substring(0,3);
+    const subMatch = find.substring(3) ? find.substring(3).split("").join("|") : ".*";
+    const subRegex = new RegExp(`^${subMatch}$`);
+    
+    for (const row of record) {
+        if (row[0] === tagName) {
+            results.push(marcsubfields(row,subRegex).join(fullOpts.join_char));
+        }
+    }
+    
+    return results;
+}
+
+/**
+ * Given a marc row and a code regex, return all matching values
+ */
+export function marcsubfields(row: string[], re: RegExp) : string[] {
+    const result : string[] = [];
+    for (let i = 3 ; i < row.length ; i += 2) {
+        if (row[i] !== undefined && row[i]?.match(re) && row[i+1] !== undefined) {
+            result.push(row[i+1] as string);
+        }
+
+    }
+    return result;
+}

package/src/output/alephseq.ts

@@ -0,0 +1,48 @@
+import { Readable, Writable } from 'stream';
+import { marcmap } from '../marcmap.js';
+import log4js from 'log4js';
+
+const logger = log4js.getLogger();
+
+export function readable2writable(readable: Readable, writable: Writable) : void {
+    readable.on('data', async (data: any) => {
+        let rec : string[][] = data['record'];
+
+        if (!rec) return;
+
+        let id = marcmap(rec,"001",{});
+
+        let output = `${id} FMT   L BK\n`;
+
+        for (let i = 0 ; i < rec.length ; i++) {
+            let tag  = rec[i]![0];
+            let ind1 = rec[i]![1];
+            let ind2 = rec[i]![2];
+            let sf = "";
+
+            for (let j = 3; j < rec[i]!.length ; j += 2) {
+                let code = rec[i]![j];
+                let val  = rec[i]![j+1];
+                if (tag!.match(/^FMT|LDR|00./g)) {
+                    sf += `${val}`;
+                }
+                else {
+                    sf += `\$\$${code}${val}`;
+                }
+            }
+
+            output += `${id} ${tag}${ind1}${ind2} L ${sf}\n`;
+        }
+
+        const ok = writable.write(output);
+
+        if (!ok) {
+            logger.debug("backpressure on");
+            readable.pause();
+            writable.once("drain", () => {
+                logger.debug("backpressure off");
+                readable.resume()
+            });
+        }
+    });
+}

package/src/output/json.ts

@@ -0,0 +1,38 @@
+import { Readable, Writable } from 'stream';
+import log4js from 'log4js';
+
+const logger = log4js.getLogger();
+
+export function readable2writable(readable: Readable, writable: Writable) : void {
+    let isFirst = true;
+
+    const ok = writable.write("[");
+
+    readable.on('data', (data: any) => {
+        let output = "";
+
+        if (!isFirst) {
+            output += ',';
+        }
+
+        output += JSON.stringify(data);
+
+        const ok = writable.write(output);
+
+        if (!ok) {
+            logger.debug("backpressure on");
+            readable.pause();
+            writable.once('drain' , () => {
+                logger.debug("backpressure off");
+                readable.resume();
+            });
+        }
+        
+        isFirst = false;
+    });
+
+    readable.on('close', () => {
+        writable.write("]");
+        writable.end();
+    });
+}

package/src/output/jsonl.ts

@@ -0,0 +1,23 @@
+import { Readable, Writable } from 'stream';
+import log4js from 'log4js';
+
+const logger = log4js.getLogger();
+
+export function readable2writable(readable: Readable, writable: Writable) : void {
+    readable.on('data', (data: any) => {
+        const ok = writable.write(JSON.stringify(data) + "\n");
+
+        if (!ok) {
+            logger.debug("backpressure on");
+            readable.pause();
+            writable.once('drain' , () => {
+                logger.debug("backpressure off");
+                readable.resume();
+            });
+        }
+    });
+
+    readable.on('close', () => {
+        writable.end();
+    });
+}

package/src/output/rdf.ts

@@ -0,0 +1,63 @@
+import { Readable, Writable } from 'stream';
+import N3 from 'n3';
+
+import log4js from 'log4js';
+
+const logger = log4js.getLogger();
+
+const { DataFactory } = N3;
+const { namedNode, literal, blankNode } = DataFactory;
+
+export function readable2writable(readable: Readable, writable: Writable) : void {
+    let writer : N3.Writer;
+
+    readable.on('data', (data: any)  => {   
+        let prefixes = data['prefixes'];
+
+        if (!writer) {
+            writer = new N3.Writer(writable, { end: false, prefixes });
+        }
+
+        let quads : any[] = data['quads'];
+
+        if (!quads) return;
+
+        for (let i = 0 ; i < quads.length ; i++) {
+            if (quads[i].subject && quads[i].predicate && quads[i].object) {
+                // ok
+            }
+            else return;
+            
+            let subject   = { type: 'NamedNode', value: '', ...quads[i].subject};
+            let predicate = { type: 'NamedNode', value: '', ...quads[i].predicate};
+            let object    = { type: 'NamedNode', value: '', ...quads[i].object};
+
+            let subjectValue = 
+                subject.type === 'NamedNode' ? namedNode(subject.value) 
+                : subject.type === 'BlankNode' ? blankNode(subject.value)
+                : namedNode(subject.value);
+            
+            let predicateValue = 
+                predicate.type === 'NamedNode' ? namedNode(predicate.value) 
+                : namedNode(predicate.value);
+            
+            let objectValue = 
+                object.type === 'NamedNode' ? namedNode(object.value) 
+                : object.type === 'BlankNode' ? blankNode(object.value)
+                : object.type === 'Literal' && object.as ? literal(object.value, namedNode(object.as))
+                : object.type === 'Literal' ? literal(object.value)
+                : namedNode(object.value);
+
+            writer.addQuad(
+                subjectValue,
+                predicateValue,
+                objectValue
+            );
+        }
+    });
+
+    readable.on('end', () => {
+        writer.end();
+        writable.end();
+    });
+}

package/src/output/xml.ts

@@ -0,0 +1,84 @@
+import { Readable, Writable } from 'stream';
+import { marcmap, marctag, marcind, marcsubfields , marcForEachSub} from '../marcmap.js';
+import log4js from 'log4js';
+
+const logger = log4js.getLogger();
+
+export function readable2writable(readable: Readable, writable: Writable) : void {
+    let isFirst = true;
+
+    writable.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+    writable.write("<marc:collection xmlns:marc=\"http://www.loc.gov/MARC21/slim\">\n");
+
+    readable.on('data', (data: any) => {
+        let rec : string[][] = data['record'];
+
+        if (!rec) return;
+
+        let output = " <marc:record>\n";
+
+        for (let i = 0 ; i < rec.length ; i++) {
+            let tag = marctag(rec[i]);
+            let ind = marcind(rec[i]); 
+            if (tag === 'FMT') {}
+            else if (tag === 'LDR') {
+                let value = marcsubfields(rec[i]!,/.*/)[0];
+                output += `  <marc:leader>${escapeXML(value)}</marc:leader>\n`;
+            }
+            else if (tag.match(/^00/)) {
+                let value = marcsubfields(rec[i]!,/.*/)[0];
+                output += `  <marc:controlfield tag="${tag}">${escapeXML(value)}</marc:controlfield>\n`;
+            }
+            else {
+                output += `  <marc:datafield tag="${tag}" ind1="${ind[0]}" ind2="${ind[1]}">\n`;
+                marcForEachSub(rec[i], (code,value) => {
+                    output += `    <marc:subfield code="${code}">${escapeXML(value)}</marc:subfield>\n`;
+                });
+                output += `  </marc:datafield>\n`;
+            }
+        }
+        output += " </marc:record>\n";
+
+        const ok = writable.write(output);
+
+        if (!ok) {
+            logger.debug("backpressure on");
+            readable.pause();
+            writable.once('drain', () => {
+                logger.debug("backpressure off");
+                readable.resume();
+            });
+        }
+    }); 
+
+    readable.on('end', () => {
+        writable.write("</marc:collection>\n");
+        writable.end();
+    });
+}
+
+export function escapeXML(
+  value: string | number | null | undefined,
+  options?: { forAttribute?: boolean }
+): string {
+    if (value === null || value === undefined) return '';
+
+    let s = String(value);
+
+    // Remove control chars that are disallowed in XML 1.0:
+    // keep tab (0x09), newline (0x0A), carriage return (0x0D)
+    s = s.replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\uFFFE\uFFFF]/g, '');
+
+    // Escape ampersand that are NOT part of a valid entity (avoid double-escape)
+    // Valid entity patterns: &name;  or  &#123;  or  &#x1A;
+    s = s.replace(/&(?!(?:[A-Za-z]+|#\d+|#x[0-9A-Fa-f]+);)/g, '&amp;');
+
+    // Escape the rest
+    s = s.replace(/</g, '&lt;').replace(/>/g, '&gt;');
+
+    if (options?.forAttribute) {
+        s = s.replace(/"/g, '&quot;').replace(/'/g, '&apos;');
+    }
+
+    return s;
+}

package/src/plugin-loader.ts

@@ -0,0 +1,27 @@
+import path from "path";
+
+export async function loadPlugin(
+    spec: string, type: "input" | "output" | "transform"
+) {
+  // spec can be:
+  //  - "./local/file.js"
+  //  - "/absolute/path/to/plugin.js"
+  //  - "package-plugin"
+  //  - "package-plugin/submodule"
+
+  try {
+    const resolved = path.resolve(spec);
+    return await import(resolved);
+  } catch (e1) {
+    try {
+      const resolved = new URL(`./${type}/${spec}.js`, import.meta.url).href;
+      return await import(resolved);
+    } catch (e2) {
+      const error = new Error(
+        `Cannot load plugin: ${spec}. Tried direct import and local plugin directory.`
+      );
+      error.cause = [e1, e2];
+      throw error;
+    }
+  }
+}