marcattacks 1.0.0

package/demo/demo.jsonata

@@ -0,0 +1,44 @@
+(
+    /******start helper functions****/
+    $strip := function ($str) {
+        $replace($str,/\s*[\,.:\/]$/,"")
+    };
+
+    $marcmap0 := function ($path) {
+        $marcmap($path)[0]
+    };
+
+    $typeLookupTable := {
+        "_": "schema:CreativeWork", /* default */
+        "book": "schema:Book",
+        "catalog": "schema:Book",
+        "correspondence": "schema:CreativeWork",
+        "dissertation": "schema:Thesis",
+        "ephemera": "schema:CreativeWork",
+        "image": "schema:VisualWork",
+        "manuscript": "schema:Book",
+        "map": "schema:Map",
+        "master": "schema:Thesis",
+        "periodical": "schema:Periodical",
+        "phd": "schema:Thesis"
+    };
+
+    $typeLookup := function ($val) {(
+        $v := $lookup($typeLookupTable,$val);
+        $v ? $v : $typeLookupTable._ 
+    )};
+    /******end helper functions******/
+
+    { 
+        "@id": "my:" & $marcmap0("001"),
+        "@type": $typeLookup($marcmap0("920a")),
+        "name": $marcmap("245ab") ~> $join(" ") ~> $strip() ,
+        "subject": $marcmap("500a") ~> $map(function ($val) {
+            { 
+                "@id": $genid(),
+                "@type": "schema:Subject",
+                "name": $val 
+            }
+        })
+    }
+)

package/dist/index.js

@@ -0,0 +1,150 @@
+#!/usr/bin/env node
+import log4js from 'log4js';
+import { program } from 'commander';
+import { loadPlugin } from './plugin-loader.js';
+import { sftpReadStream, sftpWriteStream, sftpLatestFile } from './sftpstream.js';
+import { httpReadStream } from './httpstream.js';
+import { Readable } from 'stream';
+import { pathToFileURL } from "node:url";
+import { SlowWritable } from './slow-writable.js';
+import path from "node:path";
+import fs from 'fs';
+import { s3ReaderStream, s3WriterStream } from './s3stream.js';
+log4js.configure({
+    appenders: {
+        err: {
+            type: "stderr",
+            layout: {
+                type: "pattern",
+                pattern: "%[%d %p %f{1} %m%]"
+            }
+        }
+    },
+    categories: {
+        default: { appenders: ["err"], level: "off", enableCallStack: true }
+    }
+});
+program.version('0.1.0')
+    .argument('<file>')
+    .option('-f,--from <from>', 'input type', 'xml')
+    .option('-t,--to <output>', 'output type', 'json')
+    .option('-m,--map <map>', 'data mapper', 'json')
+    .option('--fix <what>', 'jsonata')
+    .option('-o,--out <file>', 'output file')
+    .option('--key <keyfile>', 'private key file')
+    .option('--info', 'output debugging messages')
+    .option('--debug', 'output more debugging messages')
+    .option('--trace', 'output much more debugging messages');
+program.parse(process.argv);
+const opts = program.opts();
+const logger = log4js.getLogger();
+if (opts.info) {
+    logger.level = "info";
+}
+if (opts.debug) {
+    logger.level = "debug";
+}
+if (opts.trace) {
+    logger.level = "trace";
+}
+main();
+async function main() {
+    const url = program.args[0];
+    if (!url) {
+        console.error(`need an input file`);
+        process.exit(2);
+    }
+    let inputFile;
+    if (fs.existsSync(url)) {
+        const filePath = path.resolve(process.cwd(), url);
+        inputFile = pathToFileURL(filePath);
+    }
+    else {
+        inputFile = new URL(url);
+    }
+    logger.info(`using: ${inputFile}`);
+    let readableStream;
+    if (inputFile.protocol.startsWith("http")) {
+        readableStream = await httpReadStream(inputFile.toString());
+    }
+    else if (inputFile.protocol.startsWith("s3")) {
+        readableStream = await s3ReaderStream(inputFile, {});
+    }
+    else if (inputFile.protocol === 'sftp:') {
+        const config = makeSftpConfig(inputFile, opts);
+        let remotePath;
+        if (inputFile.pathname.match(/\/@latest:\w+$/)) {
+            const remoteDir = inputFile.pathname.replace(/\/@latest.*/, "");
+            const extension = inputFile.pathname.replace(/.*\/@latest:/, "");
+            remotePath = await sftpLatestFile(config, remoteDir, extension);
+        }
+        else {
+            remotePath = inputFile.pathname;
+        }
+        readableStream = await sftpReadStream(remotePath, config);
+    }
+    else {
+        readableStream = fs.createReadStream(inputFile);
+    }
+    let objectStream;
+    if (opts.from) {
+        const mod = await loadPlugin(opts.from, 'input');
+        objectStream = mod.stream2readable(readableStream);
+    }
+    else {
+        console.error(`Need --from`);
+        process.exit(1);
+    }
+    let resultStream = objectStream;
+    if (opts.map) {
+        const mod = await loadPlugin(opts.map, 'transform');
+        const transformer = await mod.transform(opts.fix);
+        resultStream = objectStream.pipe(transformer);
+    }
+    let outStream;
+    if (opts.out === '@slow') {
+        outStream = new SlowWritable({ delayMs: 100 });
+    }
+    else if (opts.out) {
+        if (opts.out.startsWith("sftp")) {
+            const url = new URL(opts.out);
+            const config = makeSftpConfig(url, opts);
+            logger.info(`put ${url}`);
+            outStream = await sftpWriteStream(url.href, config);
+        }
+        else if (opts.out.startsWith("s3")) {
+            const url = new URL(opts.out);
+            logger.info(`put ${url}`);
+            outStream = await s3WriterStream(url, {});
+        }
+        else {
+            outStream = fs.createWriteStream(opts.out, { encoding: 'utf-8' });
+        }
+    }
+    else {
+        outStream = process.stdout;
+    }
+    if (opts.to) {
+        const mod = await loadPlugin(opts.to, 'output');
+        mod.readable2writable(resultStream, outStream);
+    }
+}
+function makeSftpConfig(inputFile, opts) {
+    let privateKey = undefined;
+    if (opts.key) {
+        privateKey = fs.readFileSync(opts.key, { encoding: 'utf-8' });
+    }
+    let config = {
+        host: inputFile.hostname,
+        port: Number(inputFile.port) ?? 22,
+        username: inputFile.username
+    };
+    if (inputFile.password) {
+        config.password = inputFile.password;
+    }
+    if (privateKey) {
+        config.privateKey = privateKey;
+    }
+    return config;
+}
+//# sourceMappingURL=index.js.map

package/docker-compose.yaml

@@ -0,0 +1,37 @@
+services:
+  minio:
+    image: minio/minio:RELEASE.2025-04-22T22-12-26Z-cpuv1
+    command: server /data -address ":3371" --console-address ":3372"
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    volumes:
+      - minio-data:/data
+    ports:
+      - 3371:3371
+      - 3372:3372
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3371/minio/health/live"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+
+  mc:
+    image: minio/mc
+    depends_on:
+      - minio
+      # minio:
+      #   condition: service_healthy
+    restart: on-failure
+    entrypoint:
+      - sh
+      - -c
+      - |
+        sleep 2;
+        mc alias set docker http://minio:3371 minioadmin minioadmin;
+        mc mb docker/bbl --ignore-existing;
+  
+volumes:
+  minio-data:
+    driver: local

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "marcattacks",
+  "version": "1.0.0",
+  "main": "index.js",
+  "type": "module",
+  "author": "",
+  "repository": "https://codeberg.org/phochste/marcattacks.git",
+  "bin": {
+    "marcattacks": "./dist/index.js"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build:ts": "npx tsc",
+    "build:watch": "npx tsc -w",
+    "docker:build": "docker build . -t hochstenbach/marcattacks:v0.0.1",
+    "docker:run": "docker run --rm -v `pwd`/data:/app/data -it hochstenbach/marcattacks:v0.0.1 --to rdf --map rdf data/sample.xml",
+    "docker:push": "docker push hochstenbach/marcattacks:v0.0.1"
+  },
+  "keywords": [],
+  "license": "MIT",
+  "description": "",
+  "devDependencies": {
+    "@types/n3": "^1.26.1",
+    "@types/node": "^24.10.1",
+    "@types/sax": "^1.2.7",
+    "@types/ssh2-sftp-client": "^9.0.5",
+    "@types/stream-json": "^1.7.8",
+    "eslint": "^9.39.1",
+    "prettier": "^3.6.2",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.9.3"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.940.0",
+    "commander": "^14.0.2",
+    "fast-xml-parser": "^5.3.2",
+    "jsonata": "^2.1.0",
+    "log4js": "^6.9.1",
+    "n3": "^1.26.0",
+    "sax": "^1.4.3",
+    "ssh2-sftp-client": "^12.0.1",
+    "stream-chain": "^3.4.0",
+    "stream-json": "^1.9.1",
+    "uuid": "^13.0.0"
+  }
+}

package/plugin/demo.js

@@ -0,0 +1,12 @@
+import { Transform } from 'stream';
+
+export function transform(opts) {
+    return new Transform({
+        objectMode: true,
+        transform(data, encoding, callback) {
+            data['id'] = "brol";
+            data['record'] = [];
+            callback(null,data);
+        }
+    });
+}

package/src/httpstream.ts

@@ -0,0 +1,28 @@
+import { Readable } from 'stream';
+import * as http from 'http';
+import * as https from 'https';
+import { URL } from 'url';
+
+export function httpReadStream(urlString: string): Promise<Readable> {
+    return new Promise((resolve, reject) => {
+        const url = new URL(urlString);
+        const client = url.protocol === 'http:' ? http : https;
+
+        const req = client.get(url, res => {
+            if (res.statusCode && res.statusCode >= 400) {
+                reject(new Error('HTTP ' + res.statusCode));
+                return;
+            }
+
+            // Follow redirects
+            if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+                httpReadStream(res.headers.location).then(resolve).catch(reject);
+                return;
+            }
+
+            resolve(res);
+        });
+
+        req.on('error', reject);
+    });
+}

package/src/index.ts

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+
+import log4js from 'log4js';
+import { program } from 'commander';
+import { loadPlugin } from './plugin-loader.js';
+import { sftpReadStream , sftpWriteStream , sftpLatestFile , type SftpConfig } from './sftpstream.js';
+import { httpReadStream } from './httpstream.js';
+import { Readable } from 'stream';
+import { pathToFileURL } from "node:url";
+import type { Transform, Writable } from 'node:stream';
+import { SlowWritable } from './slow-writable.js';
+import path from "node:path";
+import fs from 'fs';
+import { s3ReaderStream, s3WriterStream } from './s3stream.js';
+
+log4js.configure({
+  appenders: {
+    err: { 
+        type: "stderr" ,
+        layout: {
+            type: "pattern",
+            pattern: "%[%d %p %f{1} %m%]"
+        }
+    }
+  },
+  categories: {
+    default: { appenders: ["err"], level: "off" , enableCallStack: true }
+  }
+});
+
+program.version('0.1.0')
+    .argument('<file>')
+    .option('-f,--from <from>','input type','xml')
+    .option('-t,--to <output>','output type','json')
+    .option('-m,--map <map>','data mapper','json')
+    .option('--fix <what>','jsonata')
+    .option('-o,--out <file>','output file')
+    .option('--key <keyfile>', 'private key file')
+    .option('--info','output debugging messages')
+    .option('--debug','output more debugging messages')
+    .option('--trace','output much more debugging messages');
+
+program.parse(process.argv);
+
+const opts   = program.opts();
+const logger = log4js.getLogger();
+
+if (opts.info) {
+    logger.level = "info";
+}
+
+if (opts.debug) {
+    logger.level = "debug";
+}
+
+if (opts.trace) {
+    logger.level = "trace";
+}
+
+main();
+
+async function main() : Promise<void> {
+    const url = program.args[0];
+
+    if (! url) {
+        console.error(`need an input file`);
+        process.exit(2);
+    }
+
+    let inputFile : URL;
+
+    if (fs.existsSync(url)) {
+        const filePath = path.resolve(process.cwd(), url);
+        inputFile = pathToFileURL(filePath);
+    }
+    else {
+        inputFile = new URL(url);
+    }
+
+    logger.info(`using: ${inputFile}`);
+
+    let readableStream;
+
+    if (inputFile.protocol.startsWith("http")) {
+        readableStream = await httpReadStream(inputFile.toString());
+    }
+    else if (inputFile.protocol.startsWith("s3")) {
+        readableStream = await s3ReaderStream(inputFile,{});
+    }
+    else if (inputFile.protocol === 'sftp:') {
+        const config = makeSftpConfig(inputFile,opts);
+
+        let remotePath;
+
+        if (inputFile.pathname.match(/\/@latest:\w+$/)) {
+            const remoteDir = inputFile.pathname.replace(/\/@latest.*/,"");
+            const extension = inputFile.pathname.replace(/.*\/@latest:/,"");
+            remotePath = await sftpLatestFile(config,remoteDir,extension);
+        }
+        else {
+            remotePath = inputFile.pathname;
+        }
+
+        readableStream = await sftpReadStream(remotePath, config);
+    }
+    else {
+        readableStream = fs.createReadStream(inputFile);
+    }
+
+    let objectStream : Readable;
+   
+    if (opts.from) {
+        const mod = await loadPlugin(opts.from,'input');
+        objectStream = mod.stream2readable(readableStream);
+    }
+    else {
+        console.error(`Need --from`);
+        process.exit(1);
+    }
+
+    let resultStream = objectStream;
+
+    if (opts.map) {
+        const mod = await loadPlugin(opts.map,'transform');
+        const transformer : Transform = await mod.transform(opts.fix);
+        resultStream = objectStream.pipe(transformer);
+    }
+
+    let outStream : Writable;
+
+    if (opts.out === '@slow') {
+        outStream = new SlowWritable({ delayMs: 100 });
+    }
+    else if (opts.out) {
+        if (opts.out.startsWith("sftp")) {
+            const url = new URL(opts.out);
+            const config = makeSftpConfig(url,opts);
+            logger.info(`put ${url}`);
+            outStream = await sftpWriteStream(url.href, config);
+        }
+        else if (opts.out.startsWith("s3")) {
+            const url = new URL(opts.out);
+            logger.info(`put ${url}`);
+            outStream = await s3WriterStream(url,{});
+        }
+        else {
+            outStream = fs.createWriteStream(opts.out, { encoding: 'utf-8'});
+        }
+    }
+    else {
+        outStream = process.stdout;
+    }
+
+    if (opts.to) {
+        const mod = await loadPlugin(opts.to,'output');
+        mod.readable2writable(resultStream, outStream);
+    }
+}
+
+function makeSftpConfig(inputFile: URL, opts: any) : SftpConfig {
+    let privateKey : string | undefined = undefined;
+
+    if (opts.key) {
+        privateKey = fs.readFileSync(opts.key,{ encoding: 'utf-8'});
+    }
+
+    let config: SftpConfig = {
+        host: inputFile.hostname,
+        port: Number(inputFile.port) ?? 22,
+        username: inputFile.username
+    };
+
+    if (inputFile.password) { config.password = inputFile.password }
+    if (privateKey) { config.privateKey = privateKey}
+
+    return config;
+}

package/src/input/alephseq.ts

@@ -0,0 +1,83 @@
+import { Readable } from "stream";
+import * as readline from 'node:readline'
+import log4js from 'log4js';
+
+const logger = log4js.getLogger();
+
+export function stream2readable(stream: Readable) : Readable {
+    let recordNum = 0;
+
+    const rl = readline.createInterface({input: stream, crlfDelay: Infinity});
+
+    let sourcePaused = false;
+
+    const readableStream = new Readable({
+        read() {
+            if (sourcePaused) {
+                logger.debug("backpressure off");
+                rl.resume(); 
+                sourcePaused = false;
+            }
+        } ,
+        objectMode: true 
+    });
+
+    let rec : string[][] = [];
+    let previd : string = "";
+
+    rl.on('line', (line) => {
+        const [id,...rest] = line.split(" ");
+        const data = rest.join(" ");
+
+        if (previd && previd !== id) {
+            const ok = readableStream.push({
+                record:  rec
+            });
+
+            if (!ok) {
+                logger.debug("backpressure on");
+                rl.pause();
+                sourcePaused = true; 
+            }
+            rec = [];
+            recordNum++;
+
+            if (recordNum % 1000 === 0) {
+                logger.info(`record: ${recordNum}`);
+            }
+        }
+
+        const tag  = data?.substring(0,3);
+        const ind1 = data?.substring(3,4);
+        const ind2 = data?.substring(4,5);
+        const sf   = data?.substring(8);
+        const parts = sf.split(/\$\$(.)/);
+
+        if (tag == 'FMT' || tag === 'LDR' || tag.startsWith("00")) {
+            rec.push([
+                tag,ind1,ind2
+            ].concat(["_"].concat(parts)));
+        }
+        else {
+            rec.push([
+                tag,ind1,ind2
+            ].concat(parts.slice(1)));
+        }
+        
+        previd = id!;
+    });
+
+    rl.on('close', () => {
+        readableStream.push({
+            record: rec
+        });
+        recordNum++;
+        if (recordNum % 1000 === 0) {
+            logger.info(`record: ${recordNum}`);
+        }
+        readableStream.push(null);
+        logger.info(`processed ${recordNum} records`);
+    });
+
+    return readableStream;
+}

package/src/input/json.ts

@@ -0,0 +1,47 @@
+import { Readable } from "stream";
+import streamArray from "stream-json/streamers/StreamArray.js";
+import log4js from 'log4js';
+
+const logger = log4js.getLogger();
+
+export function stream2readable(stream: Readable) : Readable {
+    let recordNum = 0;
+
+    const pipeline = stream.pipe(streamArray.withParser());
+
+    let sourcePaused = false;
+
+    const readableStream = new Readable({
+        read() {
+            if (sourcePaused) {
+                logger.debug("backpressure off");
+                pipeline.resume(); 
+                sourcePaused = false;
+            }
+        } ,
+        objectMode: true 
+    });
+
+    pipeline.on('data', (data: any) => {
+        const ok = readableStream.push(data.value);
+
+        if (!ok) {
+            logger.debug("backpressure on")
+            pipeline.pause();
+            sourcePaused = true;
+        }
+
+        recordNum++;
+
+        if (recordNum % 1000 === 0) {
+            logger.info(`record: ${recordNum}`);
+        }
+    });
+
+    pipeline.on('end', () => {
+        logger.info(`processed ${recordNum} records`);
+        readableStream.push(null);
+    });
+
+    return readableStream;
+}

package/src/input/jsonl.ts

@@ -0,0 +1,47 @@
+import { Readable } from "stream";
+import * as readline from 'node:readline'
+import log4js from 'log4js';
+
+const logger = log4js.getLogger();
+
+export function stream2readable(stream: Readable) : Readable {
+    let recordNum = 0;
+
+    const rl = readline.createInterface({input: stream, crlfDelay: Infinity});
+
+    let sourcePaused = false;
+
+    const readableStream = new Readable({
+        read() {
+            if (sourcePaused) {
+                logger.debug("backpressure off");
+                rl.resume(); 
+                sourcePaused = false;
+            }
+        } ,
+        objectMode: true 
+    });
+
+    rl.on('line', (line) => {
+        const ok = readableStream.push(JSON.parse(line));
+
+        if (!ok) {
+             logger.debug("backpressure on");
+             rl.pause();
+             sourcePaused = true; 
+        }
+
+        recordNum++;
+
+        if (recordNum % 1000 === 0) {
+            logger.info(`record: ${recordNum}`);
+        }
+    });
+
+    rl.on('close', () => {
+        readableStream.push(null);
+        logger.info(`processed ${recordNum} records`);
+    });
+
+    return readableStream;
+}