marcattacks 1.0.0 → 1.1.0

package/demo/demo.jsonata

@@ -1,44 +0,0 @@
-(
-    /******start helper functions****/
-    $strip := function ($str) {
-        $replace($str,/\s*[\,.:\/]$/,"")
-    };
-
-    $marcmap0 := function ($path) {
-        $marcmap($path)[0]
-    };
-
-    $typeLookupTable := {
-        "_": "schema:CreativeWork", /* default */
-        "book": "schema:Book",
-        "catalog": "schema:Book",
-        "correspondence": "schema:CreativeWork",
-        "dissertation": "schema:Thesis",
-        "ephemera": "schema:CreativeWork",
-        "image": "schema:VisualWork",
-        "manuscript": "schema:Book",
-        "map": "schema:Map",
-        "master": "schema:Thesis",
-        "periodical": "schema:Periodical",
-        "phd": "schema:Thesis"
-    };
-
-    $typeLookup := function ($val) {(
-        $v := $lookup($typeLookupTable,$val);
-        $v ? $v : $typeLookupTable._ 
-    )};
-    /******end helper functions******/
-
-    { 
-        "@id": "my:" & $marcmap0("001"),
-        "@type": $typeLookup($marcmap0("920a")),
-        "name": $marcmap("245ab") ~> $join(" ") ~> $strip() ,
-        "subject": $marcmap("500a") ~> $map(function ($val) {
-            { 
-                "@id": $genid(),
-                "@type": "schema:Subject",
-                "name": $val 
-            }
-        })
-    }
-)

package/docker-compose.yaml

@@ -1,37 +0,0 @@
-services:
-  minio:
-    image: minio/minio:RELEASE.2025-04-22T22-12-26Z-cpuv1
-    command: server /data -address ":3371" --console-address ":3372"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-    volumes:
-      - minio-data:/data
-    ports:
-      - 3371:3371
-      - 3372:3372
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:3371/minio/health/live"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-      start_period: 10s
-
-  mc:
-    image: minio/mc
-    depends_on:
-      - minio
-      # minio:
-      #   condition: service_healthy
-    restart: on-failure
-    entrypoint:
-      - sh
-      - -c
-      - |
-        sleep 2;
-        mc alias set docker http://minio:3371 minioadmin minioadmin;
-        mc mb docker/bbl --ignore-existing;
-  
-volumes:
-  minio-data:
-    driver: local

package/plugin/demo.js

@@ -1,12 +0,0 @@
-import { Transform } from 'stream';
-
-export function transform(opts) {
-    return new Transform({
-        objectMode: true,
-        transform(data, encoding, callback) {
-            data['id'] = "brol";
-            data['record'] = [];
-            callback(null,data);
-        }
-    });
-}

package/src/index.ts

@@ -1,177 +0,0 @@
-#!/usr/bin/env node
-
-import log4js from 'log4js';
-import { program } from 'commander';
-import { loadPlugin } from './plugin-loader.js';
-import { sftpReadStream , sftpWriteStream , sftpLatestFile , type SftpConfig } from './sftpstream.js';
-import { httpReadStream } from './httpstream.js';
-import { Readable } from 'stream';
-import { pathToFileURL } from "node:url";
-import type { Transform, Writable } from 'node:stream';
-import { SlowWritable } from './slow-writable.js';
-import path from "node:path";
-import fs from 'fs';
-import { s3ReaderStream, s3WriterStream } from './s3stream.js';
-
-log4js.configure({
-  appenders: {
-    err: { 
-        type: "stderr" ,
-        layout: {
-            type: "pattern",
-            pattern: "%[%d %p %f{1} %m%]"
-        }
-    }
-  },
-  categories: {
-    default: { appenders: ["err"], level: "off" , enableCallStack: true }
-  }
-});
-
-program.version('0.1.0')
-    .argument('<file>')
-    .option('-f,--from <from>','input type','xml')
-    .option('-t,--to <output>','output type','json')
-    .option('-m,--map <map>','data mapper','json')
-    .option('--fix <what>','jsonata')
-    .option('-o,--out <file>','output file')
-    .option('--key <keyfile>', 'private key file')
-    .option('--info','output debugging messages')
-    .option('--debug','output more debugging messages')
-    .option('--trace','output much more debugging messages');
-
-program.parse(process.argv);
-
-const opts   = program.opts();
-const logger = log4js.getLogger();
-
-if (opts.info) {
-    logger.level = "info";
-}
-
-if (opts.debug) {
-    logger.level = "debug";
-}
-
-if (opts.trace) {
-    logger.level = "trace";
-}
-
-main();
-
-async function main() : Promise<void> {
-    const url = program.args[0];
-
-    if (! url) {
-        console.error(`need an input file`);
-        process.exit(2);
-    }
-
-    let inputFile : URL;
-
-    if (fs.existsSync(url)) {
-        const filePath = path.resolve(process.cwd(), url);
-        inputFile = pathToFileURL(filePath);
-    }
-    else {
-        inputFile = new URL(url);
-    }
-
-    logger.info(`using: ${inputFile}`);
-
-    let readableStream;
-
-    if (inputFile.protocol.startsWith("http")) {
-        readableStream = await httpReadStream(inputFile.toString());
-    }
-    else if (inputFile.protocol.startsWith("s3")) {
-        readableStream = await s3ReaderStream(inputFile,{});
-    }
-    else if (inputFile.protocol === 'sftp:') {
-        const config = makeSftpConfig(inputFile,opts);
-
-        let remotePath;
-
-        if (inputFile.pathname.match(/\/@latest:\w+$/)) {
-            const remoteDir = inputFile.pathname.replace(/\/@latest.*/,"");
-            const extension = inputFile.pathname.replace(/.*\/@latest:/,"");
-            remotePath = await sftpLatestFile(config,remoteDir,extension);
-        }
-        else {
-            remotePath = inputFile.pathname;
-        }
-
-        readableStream = await sftpReadStream(remotePath, config);
-    }
-    else {
-        readableStream = fs.createReadStream(inputFile);
-    }
-
-    let objectStream : Readable;
-   
-    if (opts.from) {
-        const mod = await loadPlugin(opts.from,'input');
-        objectStream = mod.stream2readable(readableStream);
-    }
-    else {
-        console.error(`Need --from`);
-        process.exit(1);
-    }
-
-    let resultStream = objectStream;
-
-    if (opts.map) {
-        const mod = await loadPlugin(opts.map,'transform');
-        const transformer : Transform = await mod.transform(opts.fix);
-        resultStream = objectStream.pipe(transformer);
-    }
-
-    let outStream : Writable;
-
-    if (opts.out === '@slow') {
-        outStream = new SlowWritable({ delayMs: 100 });
-    }
-    else if (opts.out) {
-        if (opts.out.startsWith("sftp")) {
-            const url = new URL(opts.out);
-            const config = makeSftpConfig(url,opts);
-            logger.info(`put ${url}`);
-            outStream = await sftpWriteStream(url.href, config);
-        }
-        else if (opts.out.startsWith("s3")) {
-            const url = new URL(opts.out);
-            logger.info(`put ${url}`);
-            outStream = await s3WriterStream(url,{});
-        }
-        else {
-            outStream = fs.createWriteStream(opts.out, { encoding: 'utf-8'});
-        }
-    }
-    else {
-        outStream = process.stdout;
-    }
-
-    if (opts.to) {
-        const mod = await loadPlugin(opts.to,'output');
-        mod.readable2writable(resultStream, outStream);
-    }
-}
-
-function makeSftpConfig(inputFile: URL, opts: any) : SftpConfig {
-    let privateKey : string | undefined = undefined;
-
-    if (opts.key) {
-        privateKey = fs.readFileSync(opts.key,{ encoding: 'utf-8'});
-    }
-
-    let config: SftpConfig = {
-        host: inputFile.hostname,
-        port: Number(inputFile.port) ?? 22,
-        username: inputFile.username
-    };
-
-    if (inputFile.password) { config.password = inputFile.password }
-    if (privateKey) { config.privateKey = privateKey}
-
-    return config;
-}

package/src/output/rdf.ts

@@ -1,63 +0,0 @@
-import { Readable, Writable } from 'stream';
-import N3 from 'n3';
-
-import log4js from 'log4js';
-
-const logger = log4js.getLogger();
-
-const { DataFactory } = N3;
-const { namedNode, literal, blankNode } = DataFactory;
-
-export function readable2writable(readable: Readable, writable: Writable) : void {
-    let writer : N3.Writer;
-
-    readable.on('data', (data: any)  => {   
-        let prefixes = data['prefixes'];
-
-        if (!writer) {
-            writer = new N3.Writer(writable, { end: false, prefixes });
-        }
-
-        let quads : any[] = data['quads'];
-
-        if (!quads) return;
-
-        for (let i = 0 ; i < quads.length ; i++) {
-            if (quads[i].subject && quads[i].predicate && quads[i].object) {
-                // ok
-            }
-            else return;
-            
-            let subject   = { type: 'NamedNode', value: '', ...quads[i].subject};
-            let predicate = { type: 'NamedNode', value: '', ...quads[i].predicate};
-            let object    = { type: 'NamedNode', value: '', ...quads[i].object};
-
-            let subjectValue = 
-                subject.type === 'NamedNode' ? namedNode(subject.value) 
-                : subject.type === 'BlankNode' ? blankNode(subject.value)
-                : namedNode(subject.value);
-            
-            let predicateValue = 
-                predicate.type === 'NamedNode' ? namedNode(predicate.value) 
-                : namedNode(predicate.value);
-            
-            let objectValue = 
-                object.type === 'NamedNode' ? namedNode(object.value) 
-                : object.type === 'BlankNode' ? blankNode(object.value)
-                : object.type === 'Literal' && object.as ? literal(object.value, namedNode(object.as))
-                : object.type === 'Literal' ? literal(object.value)
-                : namedNode(object.value);
-
-            writer.addQuad(
-                subjectValue,
-                predicateValue,
-                objectValue
-            );
-        }
-    });
-
-    readable.on('end', () => {
-        writer.end();
-        writable.end();
-    });
-}

package/src/plugin-loader.ts

@@ -1,27 +0,0 @@
-import path from "path";
-
-export async function loadPlugin(
-    spec: string, type: "input" | "output" | "transform"
-) {
-  // spec can be:
-  //  - "./local/file.js"
-  //  - "/absolute/path/to/plugin.js"
-  //  - "package-plugin"
-  //  - "package-plugin/submodule"
-
-  try {
-    const resolved = path.resolve(spec);
-    return await import(resolved);
-  } catch (e1) {
-    try {
-      const resolved = new URL(`./${type}/${spec}.js`, import.meta.url).href;
-      return await import(resolved);
-    } catch (e2) {
-      const error = new Error(
-        `Cannot load plugin: ${spec}. Tried direct import and local plugin directory.`
-      );
-      error.cause = [e1, e2];
-      throw error;
-    }
-  }
-}

package/src/slow-writable.ts

@@ -1,165 +0,0 @@
-// slow-writable.ts
-import { Writable, type WritableOptions } from "stream";
-import log4js from 'log4js';
-
-const logger = log4js.getLogger();
-
-export interface SlowWritableOptions extends WritableOptions {
-  /**
-   * Delay per chunk (ms). Default 200 ms.
-   */
-  delayMs?: number;
-
-  /**
-   * Maximum number of concurrent "in-flight" asynchronous writes.
-   * While more writes may be queued, only up to this number will be processed in parallel.
-   * Default 1.
-   */
-  maxConcurrency?: number;
-
-  /**
-   * If set to a positive integer n, every nth chunk will produce an error (for testing).
-   * Default 0 (never error).
-   */
-  simulateErrorEveryN?: number;
-}
-
-/**
- * SlowWritable: a Writable stream that processes writes asynchronously
- * with an artificial delay and optional concurrency control.
- */
-export class SlowWritable extends Writable {
-  private delayMs: number;
-  private maxConcurrency: number;
-  private simulateErrorEveryN: number;
-  private inFlight = 0;
-  private queue: Array<{
-    chunk: any;
-    encoding: BufferEncoding;
-    callback: (err?: Error | null) => void;
-    seq: number;
-  }> = [];
-  private seqCounter = 0;
-  private destroyedFlag = false;
-
-  constructor(opts: SlowWritableOptions = {}) {
-    // Keep objectMode/encoding behavior from user but default to object mode false
-    const { delayMs = 200, maxConcurrency = 1, simulateErrorEveryN = 0, ...writableOpts } = opts;
-    super(writableOpts);
-    this.delayMs = delayMs;
-    this.maxConcurrency = Math.max(1, Math.floor(maxConcurrency));
-    this.simulateErrorEveryN = Math.max(0, Math.floor(simulateErrorEveryN));
-  }
-
-  // Node will call _write for each chunk
-  _write(chunk: any, encoding: BufferEncoding, callback: (err?: Error | null) => void): void {
-    if (this.destroyedFlag) {
-      callback(new Error("Stream is destroyed"));
-      return;
-    }
-
-    const seq = ++this.seqCounter;
-    this.queue.push({ chunk, encoding, callback, seq });
-    this.processQueue();
-  }
-
-  // Process queued writes honoring maxConcurrency
-  private processQueue(): void {
-    // If nothing to do or already at concurrency limit, return
-    while (this.inFlight < this.maxConcurrency && this.queue.length > 0 && !this.destroyedFlag) {
-      const item = this.queue.shift()!;
-      this.inFlight++;
-      this.performAsyncWrite(item)
-        .then(() => {
-          this.inFlight--;
-          // After finishing one, try to process more
-          // Use nextTick to avoid deep recursion
-          process.nextTick(() => this.processQueue());
-        })
-        .catch((err) => {
-          this.inFlight--;
-          // propagate error via callback; stream will emit 'error' as well
-          item.callback(err);
-          this.emit("error", err);
-          // continue processing queue
-          process.nextTick(() => this.processQueue());
-        });
-    }
-  }
-
-  // Simulate an asynchronous write that takes `delayMs` ms
-  private async performAsyncWrite(item: {
-    chunk: any;
-    encoding: BufferEncoding;
-    callback: (err?: Error | null) => void;
-    seq: number;
-  }): Promise<void> {
-    return new Promise((resolve, reject) => {
-      const maybeError =
-        this.simulateErrorEveryN > 0 && item.seq % this.simulateErrorEveryN === 0;
-
-      const timer = setTimeout(() => {
-        // simulate processing chunk here. For demonstration we just log.
-        // In real use, replace with actual async I/O.
-        // eslint-disable-next-line no-console
-        logger.info(`SlowWritable processed seq=${item.seq}`);
-
-        if (maybeError) {
-          const err = new Error(`Simulated error at seq ${item.seq}`);
-          item.callback(err);
-          reject(err);
-        } else {
-          item.callback();
-          resolve();
-        }
-      }, this.delayMs);
-
-      // If stream was destroyed meantime, cancel timer and callback with error
-      const onDestroy = () => {
-        clearTimeout(timer);
-        const err = new Error("Stream destroyed while writing");
-        try {
-          item.callback(err);
-        } catch (_) {
-          // ignore
-        }
-        reject(err);
-      };
-
-      // Ensure we don't leak listeners. If destroyedFlag becomes true quickly, call onDestroy.
-      if (this.destroyedFlag) {
-        onDestroy();
-      }
-    });
-  }
-
-  _final(callback: (err?: Error | null) => void): void {
-    // Wait until queue emptied and inFlight is zero
-    const check = () => {
-      if (this.destroyedFlag) {
-        callback(new Error("Stream destroyed before finalizing"));
-        return;
-      }
-      if (this.queue.length === 0 && this.inFlight === 0) {
-        callback();
-      } else {
-        setTimeout(check, 10);
-      }
-    };
-    check();
-  }
-
-  _destroy(err: Error | null, callback: (error?: Error | null) => void): void {
-    this.destroyedFlag = true;
-    // flush callbacks in queue with error
-    while (this.queue.length > 0) {
-      const item = this.queue.shift()!;
-      try {
-        item.callback(err ?? new Error("Stream destroyed"));
-      } catch (_) {
-        // ignore
-      }
-    }
-    callback(err);
-  }
-}

package/tsconfig.json

@@ -1,46 +0,0 @@
-{
-  // Visit https://aka.ms/tsconfig to read more about this file
-  "compilerOptions": {
-    // File Layout
-    "rootDir": "./src",
-    "outDir": "./dist",
-
-    // Environment Settings
-    // See also https://aka.ms/tsconfig/module
-    "module": "nodenext",
-    "target": "esnext",
-    "types": [
-      "node"
-    ],
-    // For nodejs:
-    // "lib": ["esnext"],
-    // "types": ["node"],
-    // and npm install -D @types/node
-
-    // Other Outputs
-    "sourceMap": true,
-    "declaration": true,
-    "declarationMap": true,
-
-    // Stricter Typechecking Options
-    "noUncheckedIndexedAccess": true,
-    "exactOptionalPropertyTypes": true,
-
-    // Style Options
-    // "noImplicitReturns": true,
-    // "noImplicitOverride": true,
-    // "noUnusedLocals": true,
-    // "noUnusedParameters": true,
-    // "noFallthroughCasesInSwitch": true,
-    // "noPropertyAccessFromIndexSignature": true,
-
-    // Recommended Options
-    "strict": true,
-    "jsx": "react-jsx",
-    "verbatimModuleSyntax": true,
-    "isolatedModules": true,
-    "noUncheckedSideEffectImports": true,
-    "moduleDetection": "force",
-    "skipLibCheck": true
-  }
-}