marcattacks 1.0.0

package/src/s3stream.ts

@@ -0,0 +1,266 @@
+import { 
+    S3Client, 
+    GetObjectCommand,
+    PutObjectCommand, 
+    UploadPartCommand, 
+    CreateMultipartUploadCommand, 
+    CompleteMultipartUploadCommand, 
+    type S3ClientConfig
+} from "@aws-sdk/client-s3";
+import { Readable, Writable } from "stream";
+import log4js from 'log4js';
+
+const logger = log4js.getLogger();
+type S3Config = {
+    region: string;
+    endpoint: string;
+    bucket: string;
+    key: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+};
+
+export async function s3ReaderStream(url: URL, options: { range?: string }): Promise<Readable> {
+    const config = parseURL(url);
+
+    logger.debug(`s3 config:`,config);
+
+    const bucket = config.bucket;
+    const key    = config.key;
+    const range  = options.range;
+    const s3 = makeClient(config);
+
+    const res = await s3.send(new GetObjectCommand({
+        Bucket: bucket,
+        Key: key,
+        Range: range,
+    }));
+
+    const body = res.Body;
+
+    if (!body) {
+        throw new Error("S3 GetObject returned an empty body");
+    }
+
+    // 1) If SDK returned a Node.js readable stream (typical in Node)
+    if (isNodeReadable(body)) {
+        return body as Readable;
+    }
+
+    // 2) If SDK returned a WHATWG ReadableStream (browser-ish or newer runtimes)
+    if (isReadableStream(body)) {
+        // Node.js v17+ has Readable.fromWeb
+        // Fallback: wrap async iterator
+        if (typeof (Readable as any).fromWeb === "function") {
+        return (Readable as any).fromWeb(body as ReadableStream<Uint8Array>);
+        } else {
+        // Convert using async iterator produced by the stream
+        const reader = (body as ReadableStream<Uint8Array>).getReader();
+        const nodeStream = new Readable({
+            read() {
+                // no-op. We'll push from async loop below
+            }
+        });
+        (async () => {
+            try {
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done) break;
+                nodeStream.push(Buffer.from(value));
+            }
+            nodeStream.push(null);
+            } catch (err) {
+            nodeStream.destroy(err as Error);
+            }
+        })();
+        return nodeStream;
+        }
+    }
+
+    // 3) If SDK returned a Blob (browsers)
+    if (typeof Blob !== "undefined" && body instanceof Blob) {
+        const stream = (body as Blob).stream();
+        if (typeof (Readable as any).fromWeb === "function") {
+        return (Readable as any).fromWeb(stream);
+        }
+        // fallback same as above
+        const reader = stream.getReader();
+        const nodeStream = new Readable({
+        read() {}
+        });
+        (async () => {
+        try {
+            while (true) {
+            const { done, value } = await reader.read();
+            if (done) break;
+            nodeStream.push(Buffer.from(value));
+            }
+            nodeStream.push(null);
+        } catch (err) {
+            nodeStream.destroy(err as Error);
+        }
+        })();
+        return nodeStream;
+    }
+
+    // 4) If it's an async iterable (some runtimes)
+    if (isAsyncIterable(body)) {
+        return Readable.from(body as AsyncIterable<Uint8Array | string | Buffer>);
+    }
+
+    // Unknown body shape
+    throw new Error("Unsupported S3 GetObject body type");
+}
+
+export function s3WriterStream(url: URL, options: { partSize?: number;}) : Promise<Writable> {
+    return new Promise<Writable>( (resolve) => {
+        const config = parseURL(url);
+
+        logger.debug(`s3 config:`, config);
+        const bucket = config.bucket;
+        const key = config.key;
+        const s3 = makeClient(config);
+        const partSize = options.partSize ?? 5 * 1024 * 1024;
+
+        let uploadId: string | null = null;
+        let parts: Array<{ ETag: string | undefined; PartNumber: number }> = [];
+        let buffer = Buffer.alloc(0);
+        let partNumber = 1;
+
+        const writer = new Writable({
+            async write(chunk, _encoding, callback) {
+                logger.debug("write chunk...");
+                try {
+                    buffer = Buffer.concat([buffer, chunk]);
+
+                    if (buffer.length >= partSize) {
+                        await flushPart();
+                    }
+                    callback();
+                } catch (err) {
+                    callback(err as Error);
+                }
+            },
+
+            async final(callback) {
+                logger.debug("final...");
+                try {
+                    logger.debug("flushPart...");
+                    await flushPart(true);
+                    logger.debug("finishUpload...");
+                    await finishUpload();
+                    callback();
+                } catch (err) {
+                    callback(err as Error);
+                }
+            }
+        });
+
+        async function ensureUpload() {
+            if (!uploadId) {
+                const res = await s3.send(new CreateMultipartUploadCommand({
+                    Bucket: bucket,
+                    Key: key
+                }));
+                uploadId = res.UploadId!;
+            }
+        }
+
+        async function flushPart(isLast = false) {
+            if (buffer.length === 0 && !isLast) return;
+
+            logger.debug("ensureUpload...");
+            await ensureUpload();
+
+            logger.debug("s3.send...");
+            const res = await s3.send(new UploadPartCommand({
+                Bucket: bucket,
+                Key: key,
+                PartNumber: partNumber,
+                UploadId: uploadId!,
+                Body: buffer
+            }));
+
+            parts.push({ ETag: res.ETag, PartNumber: partNumber });
+            buffer = Buffer.alloc(0);
+            partNumber++;
+        }
+
+        async function finishUpload() {
+            if (!uploadId) {
+                // No parts written, upload empty object
+                await s3.send(new PutObjectCommand({
+                    Bucket: bucket,
+                    Key: key,
+                    Body: Buffer.alloc(0)
+                }));
+                return;
+            }
+
+            await s3.send(new CompleteMultipartUploadCommand({
+                Bucket: bucket,
+                Key: key,
+                UploadId: uploadId!,
+                MultipartUpload: { Parts: parts }
+            }));
+        }
+
+        resolve(writer);
+    });
+}
+
+function isNodeReadable(x: any): x is Readable {
+    return x && typeof x.pipe === "function" && typeof x.read === "function";
+}
+
+function isReadableStream(x: any): x is ReadableStream {
+    return typeof x?.getReader === "function";
+}
+
+function isAsyncIterable(x: any): x is AsyncIterable<any> {
+    return x && typeof x[Symbol.asyncIterator] === "function";
+}
+
+function makeClient(config: S3Config) : S3Client {
+    logger.debug(config);
+    const myConfig : S3ClientConfig = {
+        endpoint: config.endpoint,
+        forcePathStyle: true,
+        region: config.region,
+    };
+
+    if (config.accessKeyId && config.secretAccessKey) {
+        myConfig.credentials = {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey
+        };
+    }
+    return new S3Client(myConfig);
+}
+
+function parseURL(url: URL) : S3Config {
+    const config : S3Config = {
+        region: "us-east-1",
+        endpoint: "http://localhost:3371",
+        bucket: "bbl",
+        key: "test.txt"
+    };
+    
+    const scheme = url.protocol.startsWith("s3s") ? "https" : "http";
+    config.endpoint = `${scheme}://${url.hostname}`;
+    if (url.port) {
+        config.endpoint += `:${url.port}`;
+    }
+    config.bucket = url.pathname.split("/")[1] ?? "";
+    config.key = url.pathname.split("/").splice(2).join("/");
+
+    if (url.username) {
+        config.accessKeyId = url.username;
+    }
+    
+    if (url.password) {
+        config.secretAccessKey = url.password;
+    }
+
+    return config;
+}

package/src/sftpstream.ts

@@ -0,0 +1,114 @@
+import { Client } from "ssh2";
+import { Readable , Writable } from "stream";
+
+export interface SftpConfig {
+  host: string;
+  port?: number;
+  username: string;
+  password?: string;
+  privateKey?: Buffer | string;
+}
+
+export async function sftpReadStream(remotePath: string, config: SftpConfig): Promise<Readable> {
+    return new Promise((resolve, reject) => {
+        const conn = new Client();
+
+        conn.on("ready", () => {
+            conn.sftp((err, sftp) => {
+                if (err) {
+                    conn.end();
+                    return reject(err);
+                }
+
+                const stream = sftp.createReadStream(remotePath);
+
+                // Close SSH connection when stream ends or errors
+                stream.on("close", () => conn.end());
+                stream.on("error", (err: any) => {
+                    conn.end();
+                    reject(err);
+                });
+
+                resolve(stream);
+            });
+        });
+
+        conn.on("error", (err) => reject(err));
+        conn.connect(config);
+    });
+}
+
+export async function sftpWriteStream(remotePath: string, config: SftpConfig): Promise<Writable> {
+    return new Promise((resolve, reject) => {
+        const conn = new Client();
+
+        conn.on("ready", () => {
+            conn.sftp((err, sftp) => {
+                if (err) {
+                    conn.end();
+                    return reject(err);
+                }
+
+                const stream = sftp.createWriteStream(remotePath, { encoding: "utf-8" });
+
+                // Close SSH connection when stream ends or errors
+                stream.on("close", () => conn.end());
+                stream.on("error", (err: any) => {
+                    conn.end();
+                    reject(err);
+                });
+
+                resolve(stream);
+            });
+        });
+
+        conn.on("error", (err) => reject(err));
+        conn.connect(config);
+    });
+}
+
+export async function sftpLatestFile(config: SftpConfig, remoteDir: string, extension: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const conn = new Client();
+
+        conn.on("ready", () => {
+            conn.sftp((err, sftp) => {
+                if (err) {
+                    conn.end();
+                    return reject(err);
+                }
+
+                sftp.readdir(remoteDir, (err, list) => {
+                    if (err) {
+                        conn.end();
+                        return reject(err);
+                    }
+
+                    if (!list || list.length === 0) {
+                        conn.end();
+                        return reject(new Error("No files found in directory"));
+                    }
+
+                    // Filter only .xml files
+                    const myFiles = list.filter(f => f.filename.toLowerCase().endsWith(extension));
+
+                    if (myFiles.length === 0) {
+                        conn.end();
+                        return reject(new Error(`No ${extension} files found in directory`));
+                    }
+
+                    const latest = myFiles.reduce((prev, curr) => 
+                        (prev.attrs.mtime > curr.attrs.mtime) ? prev : curr
+                    );
+
+                    const latestPath = `${remoteDir}/${latest.filename}`;
+                    conn.end();
+                    resolve(latestPath);
+                });
+            });
+        });
+
+        conn.on("error", (err) => reject(err));
+        conn.connect(config);
+    });
+}

package/src/slow-writable.ts

@@ -0,0 +1,165 @@
+// slow-writable.ts
+import { Writable, type WritableOptions } from "stream";
+import log4js from 'log4js';
+
+const logger = log4js.getLogger();
+
+export interface SlowWritableOptions extends WritableOptions {
+  /**
+   * Delay per chunk (ms). Default 200 ms.
+   */
+  delayMs?: number;
+
+  /**
+   * Maximum number of concurrent "in-flight" asynchronous writes.
+   * While more writes may be queued, only up to this number will be processed in parallel.
+   * Default 1.
+   */
+  maxConcurrency?: number;
+
+  /**
+   * If set to a positive integer n, every nth chunk will produce an error (for testing).
+   * Default 0 (never error).
+   */
+  simulateErrorEveryN?: number;
+}
+
+/**
+ * SlowWritable: a Writable stream that processes writes asynchronously
+ * with an artificial delay and optional concurrency control.
+ */
+export class SlowWritable extends Writable {
+  private delayMs: number;
+  private maxConcurrency: number;
+  private simulateErrorEveryN: number;
+  private inFlight = 0;
+  private queue: Array<{
+    chunk: any;
+    encoding: BufferEncoding;
+    callback: (err?: Error | null) => void;
+    seq: number;
+  }> = [];
+  private seqCounter = 0;
+  private destroyedFlag = false;
+
+  constructor(opts: SlowWritableOptions = {}) {
+    // Keep objectMode/encoding behavior from user but default to object mode false
+    const { delayMs = 200, maxConcurrency = 1, simulateErrorEveryN = 0, ...writableOpts } = opts;
+    super(writableOpts);
+    this.delayMs = delayMs;
+    this.maxConcurrency = Math.max(1, Math.floor(maxConcurrency));
+    this.simulateErrorEveryN = Math.max(0, Math.floor(simulateErrorEveryN));
+  }
+
+  // Node will call _write for each chunk
+  _write(chunk: any, encoding: BufferEncoding, callback: (err?: Error | null) => void): void {
+    if (this.destroyedFlag) {
+      callback(new Error("Stream is destroyed"));
+      return;
+    }
+
+    const seq = ++this.seqCounter;
+    this.queue.push({ chunk, encoding, callback, seq });
+    this.processQueue();
+  }
+
+  // Process queued writes honoring maxConcurrency
+  private processQueue(): void {
+    // If nothing to do or already at concurrency limit, return
+    while (this.inFlight < this.maxConcurrency && this.queue.length > 0 && !this.destroyedFlag) {
+      const item = this.queue.shift()!;
+      this.inFlight++;
+      this.performAsyncWrite(item)
+        .then(() => {
+          this.inFlight--;
+          // After finishing one, try to process more
+          // Use nextTick to avoid deep recursion
+          process.nextTick(() => this.processQueue());
+        })
+        .catch((err) => {
+          this.inFlight--;
+          // propagate error via callback; stream will emit 'error' as well
+          item.callback(err);
+          this.emit("error", err);
+          // continue processing queue
+          process.nextTick(() => this.processQueue());
+        });
+    }
+  }
+
+  // Simulate an asynchronous write that takes `delayMs` ms
+  private async performAsyncWrite(item: {
+    chunk: any;
+    encoding: BufferEncoding;
+    callback: (err?: Error | null) => void;
+    seq: number;
+  }): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const maybeError =
+        this.simulateErrorEveryN > 0 && item.seq % this.simulateErrorEveryN === 0;
+
+      const timer = setTimeout(() => {
+        // simulate processing chunk here. For demonstration we just log.
+        // In real use, replace with actual async I/O.
+        // eslint-disable-next-line no-console
+        logger.info(`SlowWritable processed seq=${item.seq}`);
+
+        if (maybeError) {
+          const err = new Error(`Simulated error at seq ${item.seq}`);
+          item.callback(err);
+          reject(err);
+        } else {
+          item.callback();
+          resolve();
+        }
+      }, this.delayMs);
+
+      // If stream was destroyed meantime, cancel timer and callback with error
+      const onDestroy = () => {
+        clearTimeout(timer);
+        const err = new Error("Stream destroyed while writing");
+        try {
+          item.callback(err);
+        } catch (_) {
+          // ignore
+        }
+        reject(err);
+      };
+
+      // Ensure we don't leak listeners. If destroyedFlag becomes true quickly, call onDestroy.
+      if (this.destroyedFlag) {
+        onDestroy();
+      }
+    });
+  }
+
+  _final(callback: (err?: Error | null) => void): void {
+    // Wait until queue emptied and inFlight is zero
+    const check = () => {
+      if (this.destroyedFlag) {
+        callback(new Error("Stream destroyed before finalizing"));
+        return;
+      }
+      if (this.queue.length === 0 && this.inFlight === 0) {
+        callback();
+      } else {
+        setTimeout(check, 10);
+      }
+    };
+    check();
+  }
+
+  _destroy(err: Error | null, callback: (error?: Error | null) => void): void {
+    this.destroyedFlag = true;
+    // flush callbacks in queue with error
+    while (this.queue.length > 0) {
+      const item = this.queue.shift()!;
+      try {
+        item.callback(err ?? new Error("Stream destroyed"));
+      } catch (_) {
+        // ignore
+      }
+    }
+    callback(err);
+  }
+}

package/src/transform/json.ts

@@ -0,0 +1,36 @@
+import { Transform } from "stream";
+import jsonata from "jsonata";
+import fs from "fs";
+import { marcmap } from '../marcmap.js';
+import { v4 as uuidv4 } from 'uuid';
+
+export async function transform(q: string) : Promise<Transform> {
+    let query = q;
+
+    if (q && fs.existsSync(q)) {
+        query = fs.readFileSync(q,{ encoding: 'utf-8'});
+    }
+
+    if (!q) {
+        query = '$';
+    }
+
+    return new Transform({
+        objectMode: true,
+        async transform(data: any, _encoding, callback) {
+            const expression = jsonata(query);
+            expression.registerFunction('marcmap', (code) => {
+                return marcmap(data['record'],code,{});
+            });
+            expression.registerFunction('genid', () => {
+                return genid();
+            });
+            data = await expression.evaluate(data);
+            callback(null,data);
+        }
+    });
+}
+
+function genid() : string {
+    return `genid:${uuidv4()}`;
+}