cisv 0.0.1

package/benchmark_cli_writer.sh

@@ -0,0 +1,280 @@
+#!/bin/bash
+
+# CISV Writer Benchmark Script
+# Compares CSV writing performance
+
+set -e
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+echo -e "${BLUE}=== CSV Writer Benchmark ===${NC}\n"
+
+# Check if cisv is built
+if [ ! -f "./cisv" ]; then
+    echo -e "${YELLOW}Building cisv CLI...${NC}"
+    make clean
+    make cli
+fi
+
+# Test file sizes
+declare -A test_sizes=(
+    ["small"]=1000
+    ["medium"]=100000
+    ["large"]=1000000
+    ["xlarge"]=10000000
+)
+
+# Function to benchmark a write command
+benchmark_write() {
+    local name=$1
+    local cmd=$2
+    local output=$3
+    local rows=$4
+
+    echo -e "${BLUE}--- $name ---${NC}"
+
+    # Get time with nanosecond precision
+    start=$(date +%s.%N)
+    eval "$cmd" > /dev/null 2>&1
+    end=$(date +%s.%N)
+
+    # Calculate elapsed time
+    elapsed=$(echo "$end - $start" | bc)
+
+    # Get file size
+    if [ -f "$output" ]; then
+        size=$(stat -c%s "$output" 2>/dev/null || stat -f%z "$output" 2>/dev/null)
+        size_mb=$(echo "scale=2; $size / 1048576" | bc)
+        throughput=$(echo "scale=2; $size_mb / $elapsed" | bc)
+        rows_per_sec=$(echo "scale=0; $rows / $elapsed" | bc)
+
+        echo "Time: ${elapsed} seconds"
+        echo "Size: ${size_mb} MB"
+        echo "Throughput: ${throughput} MB/s"
+        echo "Rows/sec: ${rows_per_sec}"
+
+        # Cleanup
+        rm -f "$output"
+    else
+        echo "Error: Output file not created"
+    fi
+    echo ""
+}
+
+# Install dependencies if needed
+install_deps() {
+    echo -e "${YELLOW}Checking/installing dependencies...${NC}"
+
+    # Check for Python pandas
+    if ! python3 -c "import pandas" 2>/dev/null; then
+        echo "Installing pandas..."
+        pip3 install pandas
+    fi
+
+    # Check for Ruby
+    if ! command -v ruby > /dev/null 2>&1; then
+        echo "Ruby not found, skipping Ruby benchmarks"
+    fi
+
+    # Build generate_csv tool if needed
+    if [ ! -f "./benchmark/generate_csv" ]; then
+        echo "Building C comparison tool..."
+        mkdir -p benchmark
+        cat > benchmark/generate_csv.c << 'EOF'
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+int main(int argc, char *argv[]) {
+    if (argc != 3) {
+        fprintf(stderr, "Usage: %s <rows> <output>\n", argv[0]);
+        return 1;
+    }
+
+    size_t rows = strtoull(argv[1], NULL, 10);
+    FILE *out = fopen(argv[2], "w");
+    if (!out) {
+        perror("fopen");
+        return 1;
+    }
+
+    // Write header
+    fprintf(out, "id,name,email,value,timestamp\n");
+
+    // Write rows
+    for (size_t i = 0; i < rows; i++) {
+        fprintf(out, "%zu,User_%zu,user%zu@example.com,%.2f,2024-01-01 00:00:00\n",
+                i + 1, i, i, i * 1.23);
+    }
+
+    fclose(out);
+    return 0;
+}
+EOF
+        gcc -O3 -o benchmark/generate_csv benchmark/generate_csv.c
+    fi
+
+    echo ""
+}
+
+# Python CSV writer
+create_python_writer() {
+    cat > benchmark/write_csv.py << 'EOF'
+import sys
+import csv
+from datetime import datetime
+
+def generate_csv(rows, output):
+    with open(output, 'w', newline='') as f:
+        writer = csv.writer(f)
+        writer.writerow(['id', 'name', 'email', 'value', 'timestamp'])
+
+        for i in range(rows):
+            writer.writerow([
+                i + 1,
+                f'User_{i}',
+                f'user{i}@example.com',
+                round(i * 1.23, 2),
+                '2024-01-01 00:00:00'
+            ])
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print(f"Usage: {sys.argv[0]} <rows> <output>")
+        sys.exit(1)
+
+    generate_csv(int(sys.argv[1]), sys.argv[2])
+EOF
+}
+
+# Ruby CSV writer
+create_ruby_writer() {
+    cat > benchmark/write_csv.rb << 'EOF'
+require 'csv'
+
+if ARGV.length != 2
+  puts "Usage: #{$0} <rows> <output>"
+  exit 1
+end
+
+rows = ARGV[0].to_i
+output = ARGV[1]
+
+CSV.open(output, 'w') do |csv|
+  csv << ['id', 'name', 'email', 'value', 'timestamp']
+
+  rows.times do |i|
+    csv << [
+      i + 1,
+      "User_#{i}",
+      "user#{i}@example.com",
+      (i * 1.23).round(2),
+      '2024-01-01 00:00:00'
+    ]
+  end
+end
+EOF
+}
+
+# Node.js CSV writer using fast-csv
+create_node_writer() {
+    cat > benchmark/write_csv.js << 'EOF'
+const fs = require('fs');
+const fastcsv = require('fast-csv');
+
+if (process.argv.length !== 4) {
+    console.error(`Usage: ${process.argv[1]} <rows> <output>`);
+    process.exit(1);
+}
+
+const rows = parseInt(process.argv[2]);
+const output = process.argv[3];
+
+const ws = fs.createWriteStream(output);
+const csvStream = fastcsv.format({ headers: true });
+
+csvStream.pipe(ws);
+
+for (let i = 0; i < rows; i++) {
+    csvStream.write({
+        id: i + 1,
+        name: `User_${i}`,
+        email: `user${i}@example.com`,
+        value: (i * 1.23).toFixed(2),
+        timestamp: '2024-01-01 00:00:00'
+    });
+}
+
+csvStream.end();
+EOF
+}
+
+# Setup
+install_deps
+create_python_writer
+create_ruby_writer
+create_node_writer
+
+# Run benchmarks for each size
+for size_name in small medium large xlarge; do
+    rows=${test_sizes[$size_name]}
+    echo -e "${GREEN}=== Testing $size_name ($rows rows) ===${NC}\n"
+
+    # CISV writer
+    benchmark_write "cisv write" \
+        "./cisv write -g $rows -o bench_cisv.csv -b" \
+        "bench_cisv.csv" \
+        "$rows"
+
+    # C fprintf baseline
+    benchmark_write "C fprintf" \
+        "./benchmark/generate_csv $rows bench_c.csv" \
+        "bench_c.csv" \
+        "$rows"
+
+    # Python csv module
+    benchmark_write "Python csv" \
+        "python3 benchmark/write_csv.py $rows bench_python.csv" \
+        "bench_python.csv" \
+        "$rows"
+
+    # Ruby CSV
+    if command -v ruby > /dev/null 2>&1; then
+        benchmark_write "Ruby CSV" \
+            "ruby benchmark/write_csv.rb $rows bench_ruby.csv" \
+            "bench_ruby.csv" \
+            "$rows"
+    fi
+
+    # Node.js fast-csv (if available)
+    if command -v node > /dev/null 2>&1 && [ -f "node_modules/fast-csv/index.js" ]; then
+        benchmark_write "Node.js fast-csv" \
+            "node benchmark/write_csv.js $rows bench_node.csv" \
+            "bench_node.csv" \
+            "$rows"
+    fi
+
+    # awk baseline
+    benchmark_write "awk" \
+        "awk 'BEGIN{print \"id,name,email,value,timestamp\"; for(i=1;i<=$rows;i++) printf \"%d,User_%d,user%d@example.com,%.2f,2024-01-01 00:00:00\\n\",i,i-1,i-1,(i-1)*1.23}' > bench_awk.csv" \
+        "bench_awk.csv" \
+        "$rows"
+
+    echo -e "${BLUE}$(printf '=%.0s' {1..60})${NC}\n"
+
+    # Only run small and medium for quick tests
+    if [[ "$1" == "--quick" && "$size_name" == "medium" ]]; then
+        break
+    fi
+done
+
+# Cleanup
+rm -f benchmark/write_csv.py benchmark/write_csv.rb benchmark/write_csv.js
+rm -f benchmark/generate_csv.c benchmark/generate_csv
+
+echo -e "${GREEN}Benchmark complete!${NC}"

package/binding.gyp

@@ -0,0 +1,57 @@
+{
+  "targets": [
+    {
+      "target_name": "cisv",
+      "sources": [
+        "src/cisv_addon.cc",
+        "src/cisv_parser.c",
+        "src/cisv_transformer.c"
+      ],
+      "include_dirs": [
+        "<!@(node -p \"require('node-addon-api').include\")",
+        "src/"
+      ],
+      "dependencies": [
+        "<!(node -p \"require('node-addon-api').gyp\")"
+      ],
+      "cflags!": [ "-fno-exceptions" ],
+      "cflags": ["-O3", "-mavx2"],
+      "cflags_cc!": [ "-fno-exceptions" ],
+      "defines": [
+        "NAPI_DISABLE_CPP_EXCEPTIONS",
+        "NAPI_VERSION=6"
+      ],
+      "conditions": [
+        ["OS=='linux'", {
+          "cflags": [
+            "-O3",
+            "-march=native",
+            "-mtune=native",
+            "-ffast-math"
+          ],
+          "cflags_cc": [
+            "-O3",
+            "-march=native",
+            "-mtune=native",
+            "-ffast-math"
+          ]
+        }],
+        ["OS=='mac'", {
+          "xcode_settings": {
+            "GCC_OPTIMIZATION_LEVEL": "3",
+            "OTHER_CFLAGS": [
+              "-march=native",
+              "-mtune=native",
+              "-ffast-math"
+            ],
+            "OTHER_CPLUSPLUSFLAGS": [
+              "-march=native",
+              "-mtune=native",
+              "-ffast-math"
+            ]
+          }
+        }]
+      ]
+    }
+  ]
+}

package/debug-addon.js

@@ -0,0 +1,64 @@
+// debug-addon.js
+console.log('=== Debugging Node.js Addon ===\n');
+
+try {
+    console.log('1. Attempting to load the addon...');
+    const addon = require('./build/Release/'); // or your main entry point
+    console.log('✓ Addon loaded successfully');
+    console.log('   Addon exports:', Object.keys(addon));
+
+    if (!addon.cisvParser) {
+        console.error('✗ cisvParser class not found in exports');
+        console.log('   Available exports:', addon);
+        process.exit(1);
+    }
+
+    console.log('\n2. Creating parser instance...');
+    const parser = new addon.cisvParser();
+    console.log('✓ Parser instance created');
+
+    console.log('\n3. Checking available methods...');
+    const prototype = Object.getPrototypeOf(parser);
+    const methods = Object.getOwnPropertyNames(prototype);
+    console.log('   Available methods:', methods);
+
+    console.log('\n4. Checking transform method specifically...');
+    console.log('   transform exists:', 'transform' in parser);
+    console.log('   transform type:', typeof parser.transform);
+
+    if (typeof parser.transform === 'function') {
+        console.log('✓ transform method is available and is a function');
+
+        console.log('\n5. Testing basic transform call...');
+        try {
+            // Test with minimal arguments
+            const result = parser.transform(0, 'uppercase');
+            console.log('✓ transform method call succeeded');
+            console.log('   Result:', result);
+        } catch (err) {
+            console.error('✗ transform method call failed:', err.message);
+            console.error('   Stack:', err.stack);
+        }
+    } else {
+        console.error('✗ transform method is not a function');
+        console.log('   Actual type:', typeof parser.transform);
+        console.log('   Value:', parser.transform);
+    }
+
+    console.log('\n6. Checking other methods...');
+    ['parseSync', 'write', 'end', 'getRows', 'transformField', 'transformRow', 'transformSchema'].forEach(method => {
+        console.log(`   ${method}: ${typeof parser[method]}`);
+    });
+
+} catch (error) {
+    console.error('✗ Failed to load or test addon:', error.message);
+    console.error('Full error:', error);
+
+    console.log('\nTroubleshooting steps:');
+    console.log('1. Check if addon compiled: ls -la build/Release/');
+    console.log('2. Try rebuilding: npm rebuild or node-gyp rebuild');
+    console.log('3. Check your index.js exports the right module');
+    console.log('4. Verify binding.gyp configuration');
+}
+
+console.log('\n=== Debug Complete ===');

package/examples/basic-parse.js

@@ -0,0 +1,65 @@
+'use strict';
+const { cisvParser } = require('../build/Release/cisv');
+const fs = require('fs');
+const path = require('path');
+
+const dataFilePath = path.join(__dirname, '../fixtures/data.csv');
+
+// Ensure the data.csv file exists before running
+if (!fs.existsSync(dataFilePath)) {
+    console.log('Creating a sample data.csv file...');
+    const sampleData = `id,name,email,city
+1,John Doe,john.doe@email.com,New York
+2,Jane Smith,"jane.smith@email.com",Los Angeles
+3,Peter Jones,peter.jones@email.com,"San Francisco"
+4,Mary Williams,"mary.w@email.com",Chicago`;
+    fs.writeFileSync(dataFilePath, sampleData);
+    console.log('Sample data.csv created.\n');
+}
+
+// Sync Parsing Example (for comparison)
+try {
+    const syncParser = new cisvParser();
+    // const dataFilePath = path.join(__dirname, '../data.csv');
+    const rows = syncParser.parseSync(dataFilePath);
+    console.log(`Sync parsing successful. Total rows found: ${rows.length}`);
+} catch (e) {
+    console.error('Sync parsing failed:', e);
+}
+
+console.log('\n' + '-'.repeat(40) + '\n');
+
+// Stream Parsing Example ---
+console.log('Starting stream parsing...');
+const streamParser = new cisvParser();
+
+fs.createReadStream(dataFilePath)
+  .on('data', chunk => {
+      try {
+          streamParser.write(chunk);
+      } catch (e) {
+          console.error('Error during stream write:', e);
+      }
+  })
+  .on('end', () => {
+    console.log('Stream finished.');
+
+    // Finalize the parsing process. This processes any remaining data.
+    streamParser.end();
+
+    // ✨ NOW, USE getRows() TO RETRIEVE THE RESULTS ✨
+    const allRows = streamParser.getRows();
+
+    console.log(`Total rows from stream: ${allRows.length}`);
+
+    if (allRows.length > 2) {
+        // As requested: get a specific line. Let's get the 3rd line (index 2).
+        const specificLine = allRows[2];
+        console.log('Getting a specific line (line 3):', specificLine);
+    } else {
+        console.log('Not enough rows to get the 3rd line.');
+    }
+  })
+  .on('error', (err) => {
+    console.error('Stream error:', err);
+  });

package/examples/large-file.js

@@ -0,0 +1,35 @@
+const { cisvParser } = require('../build/Release/cisv');
+const fs = require('fs');
+const path = require('path');
+
+// Process large CSV files with progress reporting
+function processLargeFile(filePath) {
+  const parser = new cisvParser();
+  const stats = fs.statSync(filePath);
+  const totalSize = stats.size;
+  let processed = 0;
+
+  const stream = fs.createReadStream(filePath);
+  const startTime = process.hrtime();
+
+  stream.on('data', (chunk) => {
+    processed += chunk.length;
+    parser.write(chunk);
+
+    // Report progress every MB
+    if (processed % (1024 * 1024) === 0) {
+      const percent = (processed / totalSize * 100).toFixed(1);
+      const [seconds] = process.hrtime(startTime);
+      console.log(`Processed: ${percent}% (${seconds}s)`);
+    }
+  });
+
+  stream.on('end', () => {
+    parser.end();
+    const rows = parser.getRows();
+    const [seconds] = process.hrtime(startTime);
+    console.log(`Completed! Processed ${rows.length} rows in ${seconds}s`);
+  });
+}
+
+processLargeFile(process.argv[2] || path.join(__dirname, '../fixtures/large.csv'));

package/examples/transform.js

@@ -0,0 +1,152 @@
+const { cisvParser, TransformType } = require('../build/Release/cisv');
+const fs = require('fs');
+const path = require('path');
+
+// Create test CSV if it doesn't exist
+const testFile = path.join(__dirname, 'transform_test.csv');
+if (!fs.existsSync(testFile)) {
+    const data = `id,name,email,amount,date
+1,john doe,JOHN@EXAMPLE.COM,1234.56,2024-01-01
+2,  jane smith  ,jane@EXAMPLE.com,2345.67,2024-01-02
+3,bob johnson,BOB@EXAMPLE.COM,3456.78,2024-01-03`;
+    fs.writeFileSync(testFile, data);
+}
+
+console.log('=== Native C Transform Examples ===\n');
+console.log('Available transform types:', TransformType);
+console.log('');
+
+// Example 1: Basic transforms using native C code
+console.log('1. Native C transforms:');
+const parser1 = new cisvParser();
+parser1.transform(1, 'uppercase')     // Name to uppercase (C implementation)
+       .transform(2, 'lowercase')     // Email to lowercase (C implementation)
+       .transform(3, 'to_float');     // Amount to float (C implementation)
+
+const rows1 = parser1.parseSync(testFile);
+console.log('Transformed with C code:', rows1[1]);
+
+// Example 2: Trim transform (C implementation)
+console.log('\n2. Native trim:');
+const parser2 = new cisvParser();
+parser2.transform(1, 'trim');
+
+const rows2 = parser2.parseSync(testFile);
+console.log('Original: "  jane smith  "');
+console.log('Trimmed with C:', `"${rows2[2][1]}"`);
+
+// Example 3: SHA256 hashing (C implementation)
+console.log('\n3. Native SHA256:');
+const parser3 = new cisvParser();
+parser3.transform(2, 'hash_sha256');
+
+const rows3 = parser3.parseSync(testFile);
+console.log('Hashed email (C implementation):', rows3[1][2]);
+
+// Example 4: Base64 encoding (C implementation)
+console.log('\n4. Native Base64:');
+const parser4 = new cisvParser();
+parser4.transform(1, 'base64_encode');
+
+const rows4 = parser4.parseSync(testFile);
+console.log('Base64 encoded names (C):', rows4.slice(1).map(row => row[1]));
+
+// Example 5: Multiple transforms on different fields
+console.log('\n5. Multiple native transforms:');
+const parser5 = new cisvParser();
+parser5.transform(0, 'to_int')        // ID to int
+       .transform(1, 'trim')           // Trim name
+       .transform(1, 'uppercase')      // Then uppercase
+       .transform(2, 'lowercase')      // Email lowercase
+       .transform(3, 'to_float');      // Amount to float
+
+const rows5 = parser5.parseSync(testFile);
+console.log('Multiple transforms:', rows5[1]);
+
+// Example 6: Performance test with native transforms
+console.log('\n6. Performance test (native C transforms):');
+
+// Generate larger test file
+const largeCsv = 'large_native_test.csv';
+const rowCount = 100000;
+console.log(`Generating ${rowCount} rows...`);
+
+let csvContent = 'id,name,email,amount\n';
+for (let i = 0; i < rowCount; i++) {
+    csvContent += `${i},  user ${i}  ,USER${i}@EXAMPLE.COM,${i * 1.23}\n`;
+}
+fs.writeFileSync(largeCsv, csvContent);
+
+// Test without transforms
+console.time('No transforms');
+const parserNoTransform = new cisvParser();
+parserNoTransform.parseSync(largeCsv);
+console.timeEnd('No transforms');
+
+// Test with C transforms
+console.time('With C transforms');
+const parserWithTransform = new cisvParser();
+parserWithTransform.transform(1, 'trim')
+                   .transform(1, 'uppercase')
+                   .transform(2, 'lowercase')
+                   .transform(3, 'to_float');
+parserWithTransform.parseSync(largeCsv);
+console.timeEnd('With C transforms');
+
+// Get stats
+const stats = parserWithTransform.getStats();
+console.log('Parse stats:', stats);
+
+// Example 7: Parse string content
+console.log('\n7. Parse string with transforms:');
+const csvString = `name,value
+test one,123
+TEST TWO,456`;
+
+const parser7 = new cisvParser();
+parser7.transform(0, 'uppercase')
+       .transform(1, 'to_int');
+
+const rows7 = parser7.parseString(csvString);
+console.log('Parsed string:', rows7);
+
+// Example 8: Streaming with transforms
+console.log('\n8. Streaming with transforms:');
+const parser8 = new cisvParser();
+parser8.transform(0, 'uppercase');
+
+parser8.write('name,value\n');
+parser8.write('test,123\n');
+parser8.write('another,456\n');
+parser8.end();
+
+const rows8 = parser8.getRows();
+console.log('Streamed rows:', rows8);
+
+// Example 9: Count rows (static method)
+console.log('\n9. Count rows:');
+const rowCount2 = cisvParser.countRows(testFile);
+console.log(`File has ${rowCount2} rows`);
+
+// Example 10: Clear and reuse parser
+console.log('\n10. Reuse parser:');
+const parser10 = new cisvParser();
+parser10.transform(0, 'uppercase');
+
+// First parse
+const firstParse = parser10.parseSync(testFile);
+console.log('First parse rows:', firstParse.length);
+
+// Clear and parse again with different transforms
+parser10.clear();
+parser10.clearTransforms();
+parser10.transform(1, 'lowercase');
+
+const secondParse = parser10.parseSync(testFile);
+console.log('Second parse rows:', secondParse.length);
+
+// Cleanup
+fs.unlinkSync(largeCsv);
+
+console.log('\n=== Native Transform Examples Complete ===');
+console.log('All transforms executed using optimized C code!');

package/examples/typescript.ts

@@ -0,0 +1,38 @@
+import { join } from 'path';
+import { createReadStream } from 'fs';
+
+declare module '../build/Release/cisv' {
+  export class cisvParser {
+    parseSync(path: string): string[][];
+    write(chunk: Buffer): void;
+    end(): void;
+    getRows(): string[][];
+  }
+}
+
+const { cisvParser } = require('../build/Release/cisv') as {
+  cisvParser: typeof cisvParser
+};
+
+async function processCSV(filePath: string) {
+  const parser = new cisvParser();
+  const stream = createReadStream(filePath);
+
+  return new Promise<string[][]>((resolve, reject) => {
+    stream.on('data', chunk => parser.write(chunk));
+    stream.on('end', () => {
+      parser.end();
+      resolve(parser.getRows());
+    });
+    stream.on('error', reject);
+  });
+}
+
+(async () => {
+  try {
+    const rows = await processCSV(join(__dirname, '../fixtures/data.csv'));
+    console.log(`Processed ${rows.length} rows`);
+  } catch (err) {
+    console.error('Error:', err);
+  }
+})();