cisv 0.0.1

package/.github/workflows/ci.yml

@@ -0,0 +1,158 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest] # macos-latest, windows-latest # skipped for now
+        node-version: [23.x, 24.x]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+    - run: npm install
+    - run: npm test
+    - name: Build
+      run: |
+        npm run build
+        npm run test:build
+
+  cli-build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest] # macos-latest ignored for now
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build CLI
+      run: |
+        npm install -g node-gyp
+        make clean
+        make build
+        make cli
+    - name: Test CLI basic functionality
+      run: |
+        ./cisv --version
+        ./cisv --help
+        echo "id,name,value" > test.csv
+        echo "1,test,100" >> test.csv
+        echo "2,demo,200" >> test.csv
+        ./cisv -c test.csv | grep -q "3" # including headers
+        ./cisv test.csv
+        ./cisv -s 0,2 test.csv
+        ./cisv --head 1 test.csv
+    - name: Upload CLI binary
+      uses: actions/upload-artifact@v4
+      with:
+        name: cisv-${{ matrix.os }}
+        path: cisv
+
+  memory-check:
+    runs-on: ubuntu-latest
+    needs: cli-build
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install memory checking tools
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y valgrind cppcheck clang-tools
+    - name: Build CLI with debug symbols
+      run: |
+        npm install -g node-gyp
+        make clean
+        make build
+        CFLAGS="-g -O0" make cli
+
+    - name: Run memory leak tests with Valgrind
+      run: |
+        # Run with valgrind leak check
+        valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose ./cisv --version 2>&1 | tee valgrind-version.log
+        # Test basic CSV parsing
+        echo "id,name,value" > memtest.csv
+        echo "1,test,100" >> memtest.csv
+        echo "2,demo,200" >> memtest.csv
+        valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes ./cisv memtest.csv 2>&1 | tee valgrind-basic.log
+        valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes ./cisv -c memtest.csv 2>&1 | tee valgrind-count.log
+        valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes ./cisv -s 0,2 memtest.csv 2>&1 | tee valgrind-select.log
+        # Check for memory leaks
+        ! grep -q "definitely lost" valgrind-*.log
+        ! grep -q "indirectly lost" valgrind-*.log
+
+    - name: Run memory deep leak tests on reader and writer
+      run: |
+        chmod +x test_select.sh
+        bash ./test_select.sh
+
+    - name: Run memory deep leak tests on transform
+      run: |
+        chmod +x test_transform.sh
+        bash ./test_transform.sh
+
+    - name: Run AddressSanitizer tests
+      run: |
+        make clean
+        CC=clang CFLAGS="-g -O0 -fsanitize=address -fno-omit-frame-pointer" LDFLAGS="-fsanitize=address" make cli
+        ./cisv --version
+        echo "id,name,value,x" > test_x.csv
+        echo "1,test,100,a" >> test_x.csv
+        echo "2,demo,200,b" >> test_x.csv
+        ./cisv -c test_x.csv
+        ./cisv -s 0,2,3 test_x.csv
+    - name: Upload memory check results
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: memory-check-logs
+        path: valgrind-*.log
+
+        #benchmark:
+        #  runs-on: ubuntu-latest
+        #  needs: [build, cli-build, memory-check]
+        #  steps:
+        #  - uses: actions/checkout@v3
+        #  - uses: actions/setup-node@v3
+        #    with:
+        #      node-version: 23.x
+        #  - name: Install benchmark dependencies
+        #    run: |
+        #      sudo apt-get update
+        #      sudo apt-get install -y miller bc time
+        #      pip3 install --upgrade pip
+        #      pip3 install 'babel>=2.9.0' 'csvkit>=1.1.0'
+        #  - name: Install Rust tools
+        #    uses: actions-rs/toolchain@v1
+        #    with:
+        #      toolchain: stable
+        #      override: true
+        #  - name: Install xsv and qsv
+        #    run: |
+        #      cargo install xsv
+        #      cargo install qsv
+        #  - name: Build cisv CLI
+        #    run: |
+        #      make clean
+        #      make cli
+        #  - name: Run Node.js benchmarks
+        #    run: |
+        #      npm install
+        #      npm run benchmark
+        #  - name: Run CLI benchmarks
+        #    run: |
+        #      make benchmark-cli
+        #  - name: Generate benchmark report
+        #    run: |
+        #      echo "## Benchmark Results" > benchmark-report.md
+        #      echo "Date: $(date)" >> benchmark-report.md
+        #      echo "Commit: ${{ github.sha }}" >> benchmark-report.md
+        #      echo "" >> benchmark-report.md
+        #      make benchmark-cli >> benchmark-report.md 2>&1
+        #  - name: Upload benchmark results
+        #    uses: actions/upload-artifact@v3
+        #    with:
+        #      name: benchmark-results
+        #      path: benchmark-report.md

package/.github/workflows/release.yml

@@ -0,0 +1,167 @@
+name: release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+permissions:
+  contents: write
+  packages: read
+env:
+  VERSION: ${{ github.ref_name }}
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest] # , macos-latest
+        arch: [amd64, arm64]
+        include:
+          - os: ubuntu-latest
+            pkg_os: ubuntu
+            pkg_type: deb
+         # - os: macos-latest
+         #   pkg_os: macos
+         #   pkg_type: tar.gz
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '22.x'
+
+      - name: Install build dependencies
+        shell: bash
+        run: |
+          npm install -g node-gyp node-addon-api
+          npm install
+
+      - name: Build CLI binary
+        shell: bash
+        run: |
+          # Set platform-specific variables
+          OS_NAME=$(echo "${{ matrix.os }}" | sed 's/-latest//')
+          ARCH="${{ matrix.arch }}"
+          EXT=""
+
+          # Build cisv with version info
+          npm install -g node-gyp
+          make clean
+          make build
+          CFLAGS="-DCISV_VERSION=\"${VERSION}\" -O3 -march=native" make cli
+
+          # Rename output file
+          OUTPUT="cisv-${{ matrix.pkg_os }}-${ARCH}${EXT}"
+          mv cisv${EXT} "$OUTPUT"
+          echo "OUTPUT_FILE=$OUTPUT" >> $GITHUB_ENV
+
+      - name: Build Node.js addon
+        shell: bash
+        run: |
+          npm run build
+          # Package node addon
+          ADDON_OUTPUT="cisv-node-${{ matrix.pkg_os }}-${VERSION}${ARCH}.node"
+          cp build/Release/cisv.node "$ADDON_OUTPUT"
+          echo "ADDON_FILE=$ADDON_OUTPUT" >> $GITHUB_ENV
+
+      - name: Package artifact
+        shell: bash
+        run: |
+          if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
+            sudo apt-get update && sudo apt-get install -y ruby-dev rubygems
+            sudo gem install fpm
+
+            # Create directory structure for package
+            mkdir -p package/usr/local/bin
+            mkdir -p package/usr/local/lib/node_modules/cisv
+
+            cp "$OUTPUT_FILE" package/usr/local/bin/cisv
+            chmod +x package/usr/local/bin/cisv
+
+            # Package with fpm
+            fpm -s dir -t deb -n cisv -v ${VERSION#v} \
+              --architecture ${{ matrix.arch }} \
+              --description "The fastest CSV parser" \
+              --url "https://github.com/${{ github.repository }}" \
+              --maintainer "${{ github.repository_owner }}" \
+              -C package \
+              usr/local/bin
+
+            PACKAGE=$(ls cisv_*_${{ matrix.arch }}.deb)
+          else
+            # Create tarball with both CLI and Node addon
+            tar czf cisv-${{ matrix.pkg_os }}-${{ matrix.arch }}.tar.gz "$OUTPUT_FILE" "$ADDON_FILE" README.md LICENSE
+            PACKAGE="cisv-${{ matrix.pkg_os }}-${{ matrix.arch }}.tar.gz"
+          fi
+          echo "PACKAGE_FILE=$PACKAGE" >> $GITHUB_ENV
+
+      - name: Test binary
+        shell: bash
+        run: |
+          # Basic smoke test
+          ./${{ env.OUTPUT_FILE }} --version
+          ./${{ env.OUTPUT_FILE }} --help
+
+          # Create test CSV
+          echo "id,name,value" > test.csv
+          echo "1,test,100" >> test.csv
+          echo "2,demo,200" >> test.csv
+
+          # Test basic functionality
+          ./${{ env.OUTPUT_FILE }} -c test.csv
+          ./${{ env.OUTPUT_FILE }} test.csv
+
+      - name: Upload Release Assets
+        uses: softprops/action-gh-release@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          files: |
+            ${{ env.OUTPUT_FILE }}
+            ${{ env.ADDON_FILE }}
+            ${{ env.PACKAGE_FILE }}
+          generate_release_notes: true
+
+      - name: Update Latest Tag
+        if: github.repository == 'Sanix-Darker/cisv'
+        uses: softprops/action-gh-release@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          tag_name: latest
+          name: latest
+          body: "latest release: ${{ env.VERSION }}"
+          files: |
+            ${{ env.OUTPUT_FILE }}
+            ${{ env.ADDON_FILE }}
+            ${{ env.PACKAGE_FILE }}
+
+  # TODO: to do later
+  publish-npm:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '23.x'
+          registry-url: 'https://registry.npmjs.org'
+
+          #- name: Update package version
+          #  run: |
+          #    npm version ${VERSION#v} --no-git-tag-version
+
+      - name: Build and test
+        run: |
+          npm install
+          npm run build
+          npm test
+
+      - name: Publish to npm
+        run: npm publish --access public
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package/Dockerfile

@@ -0,0 +1,63 @@
+# Dockerfile for CISV benchmarking with CPU/RAM isolation
+FROM ubuntu:22.04
+
+# Prevent interactive prompts
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    curl \
+    wget \
+    python3 \
+    python3-pip \
+    bc \
+    time \
+    valgrind \
+    clang \
+    libc6-dev \
+    miller \
+    ruby \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Node.js 20.x
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y nodejs
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install Rust CSV tools
+RUN cargo install xsv qsv
+
+# Install Python CSV tools
+RUN pip3 install --upgrade pip && \
+    pip3 install 'babel>=2.9.0' 'csvkit>=1.1.0' pandas
+
+# Install Node.js CSV tools (with legacy peer deps for compatibility)
+RUN npm install -g node-gyp node-addon-api fast-csv --legacy-peer-deps
+
+# Create benchmark user and directory
+RUN useradd -m -s /bin/bash benchmark
+WORKDIR /home/benchmark
+
+# Copy the project
+COPY --chown=benchmark:benchmark . /home/benchmark/cisv
+
+# Build cisv and dependencies
+WORKDIR /home/benchmark/cisv
+# Skip install-benchmark-deps since we already installed dependencies
+RUN make clean && cargo install qsv && make install-benchmark-deps && make cli
+
+# Create main benchmark runner script
+COPY ./benchmark_cli_writer.sh ./benchmark_cli_reader.sh ./run_benchmarks.sh /home/benchmark/
+RUN chmod +x /home/benchmark/run_benchmarks.sh
+
+# Switch to benchmark user
+USER benchmark
+WORKDIR /home/benchmark/cisv
+
+# Default command runs all benchmarks
+CMD ["/home/benchmark/run_benchmarks.sh", "all"]

package/LICENSE

@@ -0,0 +1,7 @@
+GNU GENERAL PUBLIC LICENSE
+Version 2, 29 Jul 2025
+
+Sanix-Darker CISV Project
+
+This project is licensed under the GNU General Public License, Version 2.0 (GPL-2.0).
+You may obtain a copy of the License at https://www.gnu.org/licenses/gpl-2.0.html.

package/Makefile

@@ -0,0 +1,160 @@
+CC ?= gcc
+CFLAGS ?= -O3 -march=native -pipe -fomit-frame-pointer -Wall -Wextra -std=c11 -flto -ffast-math -funroll-loops
+LDFLAGS ?= -flto -s
+NODE_GYP ?= node-gyp
+
+# CLI binary name
+CLI_BIN = cisv
+# Include both parser and writer
+CLI_SRC = src/cisv_parser.c src/cisv_writer.c src/cisv_transformer.c
+CLI_OBJ = $(CLI_SRC:.c=.o)
+
+# Build targets
+all: build cli
+
+install:
+	npm install -g node-gyp
+	npm install -g node-addon-api
+
+build:
+	npm install
+	$(NODE_GYP) configure build CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)"
+
+# Build CLI tool (cisv_parser.c contains main when CISV_CLI is defined)
+cli: $(CLI_BIN)
+
+$(CLI_BIN): $(CLI_OBJ)
+	$(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
+
+# Compile with CISV_CLI defined to include CLI code
+src/cisv_parser.o: src/cisv_parser.c
+	$(CC) $(CFLAGS) -DCISV_CLI -c -o src/cisv_parser.o src/cisv_parser.c
+
+src/cisv_writer.o: src/cisv_writer.c
+	$(CC) $(CFLAGS) -DCISV_CLI -c -o src/cisv_writer.o src/cisv_writer.c
+
+src/cisv_transformer.o: src/cisv_transformer.c
+	$(CC) $(CFLAGS) -DCISV_CLI -c -o src/cisv_transformer.o src/cisv_transformer.c
+
+# Install CLI tool to /usr/local/bin
+install-cli: cli
+	install -m 755 $(CLI_BIN) /usr/local/bin/$(CLI_BIN)
+
+# Install benchmark dependencies
+install-benchmark-deps:
+	@echo "Installing benchmark dependencies..."
+	@# Install Rust if not present
+	@if ! command -v cargo > /dev/null; then \
+		echo "Installing Rust..."; \
+		curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y; \
+		. $$HOME/.cargo/env; \
+	fi
+	#@# Install xsv
+	#@if ! command -v xsv > /dev/null; then \
+	#	echo "Installing xsv..."; \
+	#	cargo install xsv; \
+	#fi
+	@# Install qsv (faster fork of xsv)
+	#@if ! command -v qsv > /dev/null; then \
+	#	echo "Installing qsv..."; \
+	#	cargo install qsv; \
+	#fi
+	@# Build rust-csv example
+	@echo "Building rust-csv benchmark tool..."
+	@mkdir -p benchmark/rust-csv-bench
+	@cd benchmark/rust-csv-bench && \
+		if [ ! -f Cargo.toml ]; then \
+			cargo init --name csv-bench; \
+			echo 'csv = "1.3"' >> Cargo.toml; \
+		fi && \
+		echo 'use std::env;\nuse std::error::Error;\nuse csv::ReaderBuilder;\n\nfn main() -> Result<(), Box<dyn Error>> {\n    let args: Vec<String> = env::args().collect();\n    if args.len() < 2 { eprintln!("Usage: {} <file>", args[0]); std::process::exit(1); }\n    let mut rdr = ReaderBuilder::new().has_headers(true).from_path(&args[1])?;\n    let count = rdr.records().count();\n    println!("{}", count);\n    Ok(())\n}' > src/main.rs && \
+		cargo build --release
+	@# Fix csvkit Python 3.12 compatibility issue and install
+	@if command -v pip3 > /dev/null; then \
+		echo "Installing/fixing csvkit..."; \
+		pip3 install --upgrade pip; \
+		pip3 uninstall -y csvkit babel || true; \
+		pip3 install 'babel>=2.9.0' 'csvkit>=1.1.0'; \
+	fi
+
+debug:
+	$(NODE_GYP) configure build --debug
+
+clean:
+	$(NODE_GYP) clean
+	rm -rf build
+	rm -f $(CLI_BIN) $(CLI_OBJ)
+	rm -f src/cisv_cli.o  # Clean old object file if exists
+
+test: build
+	npm test
+
+# Benchmark comparing cisv CLI with other tools
+benchmark-cli: cli
+	@echo "=== Benchmarking CSV CLI tools ==="
+	@echo "Preparing test file..."
+	@python3 -c "import csv; w=csv.writer(open('bench_test.csv','w')); [w.writerow([i,f'name_{i}',f'email_{i}@test.com',f'city_{i}']) for i in range(1000000)]"
+	@echo ""
+	@echo "--- cisv (this project) ---"
+	@bash -c "TIMEFORMAT='real %3R'; time ./$(CLI_BIN) -c bench_test.csv 2>&1"
+	@if [ -f benchmark/rust-csv-bench/target/release/csv-bench ]; then \
+		echo ""; \
+		echo "--- rust-csv (Rust library) ---"; \
+		bash -c "TIMEFORMAT='real %3R'; time ./benchmark/rust-csv-bench/target/release/csv-bench bench_test.csv 2>&1"; \
+	fi
+	@if command -v xsv > /dev/null 2>&1; then \
+		echo ""; \
+		echo "--- xsv (Rust CLI) ---"; \
+		bash -c "TIMEFORMAT='real %3R'; time xsv count bench_test.csv 2>&1"; \
+	fi
+	@if command -v qsv > /dev/null 2>&1; then \
+		echo ""; \
+		echo "--- qsv (Rust CLI - faster xsv fork) ---"; \
+		bash -c "TIMEFORMAT='real %3R'; time qsv count bench_test.csv 2>&1"; \
+	fi
+	@if command -v wc > /dev/null 2>&1; then \
+		echo ""; \
+		echo "--- wc -l (baseline) ---"; \
+		bash -c "TIMEFORMAT='real %3R'; time wc -l bench_test.csv 2>&1"; \
+	fi
+	@if command -v csvstat > /dev/null 2>&1; then \
+		echo ""; \
+		echo "--- csvkit (Python) ---"; \
+		bash -c "TIMEFORMAT='real %3R'; time csvstat --count bench_test.csv 2>&1" || echo "csvkit failed - may need: pip3 install --upgrade babel csvkit"; \
+	fi
+	@if command -v mlr > /dev/null 2>&1; then \
+		echo ""; \
+		echo "--- Miller ---"; \
+		bash -c "TIMEFORMAT='real %3R'; time mlr --csv count bench_test.csv 2>&1"; \
+	fi
+	@echo ""
+	@echo "File size: " && ls -lh bench_test.csv | awk '{print $$5}'
+	@rm -f bench_test.csv
+
+benchmark: build
+	node benchmark/benchmark.js $(SAMPLE)
+
+perf: build
+	node test/performance.test.js
+
+coverage:
+	$(NODE_GYP) configure --coverage
+	$(MAKE) test
+
+package: clean build test
+	npm pack
+
+# Test writer functionality
+test-writer: cli
+	chmod +x test_writer.sh
+	./test_writer.sh
+
+# Benchmark writer performance
+benchmark-writer: cli
+	chmod +x benchmark_cli_writer.sh
+	./benchmark_cli_writer.sh
+
+# Run all benchmarks
+benchmark-all: benchmark-cli benchmark-writer
+
+.PHONY: all build cli clean test benchmark benchmark-cli benchmark-writer benchmark-all perf coverage package install-cli install-benchmark-deps test-writer