csv-to-json-streamer 1.0.0

package/index.js

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+import fs from "fs";
+import csv from "csv-parser";
+import { Transform, pipeline } from "stream";
+import { Command } from "commander";
+
+// 1. Your Custom Streams
+class ProgressLogger extends Transform {
+  constructor(logInterval = 10000) {
+    super({ objectMode: true });
+    this.rowCount = 0;
+    this.logInterval = logInterval;
+    this.startTime = Date.now();
+  }
+
+  _transform(chunk, encoding, callback) {
+    this.rowCount++;
+    if (this.rowCount % this.logInterval === 0) {
+      const elapsed = ((Date.now() - this.startTime) / 1000).toFixed(2);
+      console.log(`[Progress] Processed ${this.rowCount} rows in ${elapsed}s...`);
+    }
+    this.push(chunk);
+    callback();
+  }
+
+  _flush(callback) {
+    const totalTime = ((Date.now() - this.startTime) / 1000).toFixed(2);
+    console.log(`[Summary] Finished reading ${this.rowCount} total rows in ${totalTime}s.`);
+    callback();
+  }
+}
+
+class JSONFormatter extends Transform {
+  constructor() {
+    super({ writableObjectMode: true });
+    this.first = true;
+  }
+
+  _transform(obj, encoding, callback) {
+    if (this.first) {
+      this.push("[\n");
+      this.first = false;
+    } else {
+      this.push(",\n");
+    }
+    this.push(JSON.stringify(obj));
+    callback();
+  }
+
+  _flush(callback) {
+    if (!this.first) {
+      this.push("\n]");
+    } else {
+      this.push("[]");
+    }
+    callback();
+  }
+}
+
+// 2. Set up the CLI using Commander
+const program = new Command();
+
+program
+  .name('csv2json')
+  .description('Convert large CSV files to JSON efficiently using streams.')
+  .version('1.0.0')
+  .requiredOption('-i, --input <path>', 'Path to the input CSV file')
+  .option('-o, --output <path>', 'Path to the output JSON file', 'output.json')
+  .option('-l, --log-interval <number>', 'How often to log progress', '50000')
+  .action((options) => {
+    const inputFilePath = options.input;
+    const outputFilePath = options.output;
+    const logInterval = parseInt(options.logInterval, 10);
+
+    if (!fs.existsSync(inputFilePath)) {
+      console.error(`[Error] Input file not found: ${inputFilePath}`);
+      process.exit(1);
+    }
+
+    console.log(`[Start] Initiating pipeline from ${inputFilePath} to ${outputFilePath}...`);
+
+    pipeline(
+      fs.createReadStream(inputFilePath),
+      csv({
+        mapHeaders: ({ header }) => header.trim(),
+        mapValues: ({ value }) => value?.trim(),
+      }),
+      new ProgressLogger(logInterval),
+      new JSONFormatter(),
+      fs.createWriteStream(outputFilePath),
+      (err) => {
+        if (err) {
+          console.error("[Error] Pipeline failed:", err);
+          process.exit(1);
+        } else {
+          console.log(`[Success] Pipeline succeeded. Data written to ${outputFilePath}.`);
+          process.exit(0);
+        }
+      }
+    );
+  });
+
+// Parse the arguments triggered by the user
+program.parse(process.argv);

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "csv-to-json-streamer",
+  "version": "1.0.0",
+  "description": "A memory-efficient CLI tool to stream large CSV files into JSON.",
+  "main": "index.js",
+  "type": "module",
+  "bin": {
+    "csv2json": "index.js"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [
+    "cli",
+    "csv",
+    "json",
+    "stream",
+    "parser"
+  ],
+  "author": "Your Name",
+  "license": "MIT",
+  "dependencies": {
+    "commander": "^11.1.0",
+    "csv-parser": "^3.0.0"
+  }
+}