cisv 0.0.40 → 0.0.41

package/README.md

@@ -41,41 +41,41 @@ $ docker run --rm      \
 ## BENCHMARKS
 
 Benchmarks comparison with existing popular tools,
-cf pipeline you can check : (https://github.com/Sanix-Darker/cisv/actions/runs/17194915214/job/48775516036)
+cf pipeline you can check : (https://github.com/Sanix-Darker/cisv/actions/runs/17697547058/job/50298916576) a step "Publish to npm"
 
 ### SYNCHRONOUS RESULTS
 
 | Library            | Speed (MB/s) | Avg Time (ms) | Operations/sec |
 |--------------------|--------------|---------------|----------------|
-| cisv (sync)        | 30.04        | 0.02          | 64936          |
-| csv-parse (sync)   | 13.35        | 0.03          | 28870          |
-| papaparse (sync)   | 25.16        | 0.02          | 54406          |
+| cisv (sync)        | 45.58        | 0.01          | 98543          |
+| csv-parse (sync)   | 18.11        | 0.03          | 39155          |
+| papaparse (sync)   | 28.03        | 0.02          | 60596          |
 
 ### SYNCHRONOUS RESULTS (WITH DATA ACCESS)
 
 | Library            | Speed (MB/s) | Avg Time (ms) | Operations/sec |
 |--------------------|--------------|---------------|----------------|
-| cisv (sync)        | 31.24        | 0.01          | 67543          |
-| csv-parse (sync)   | 15.42        | 0.03          | 33335          |
-| papaparse (sync)   | 25.49        | 0.02          | 55107          |
+| cisv (sync)        | 46.80        | 0.01          | 101185         |
+| csv-parse (sync)   | 18.92        | 0.02          | 40900          |
+| papaparse (sync)   | 28.38        | 0.02          | 61363          |
 
 
 ### ASYNCHRONOUS RESULTS
 
 | Library                  | Speed (MB/s) | Avg Time (ms) | Operations/sec |
 |--------------------------|--------------|---------------|----------------|
-| cisv (async/stream)      | 61.31        | 0.01          | 132561         |
-| papaparse (async/stream) | 19.24        | 0.02          | 41603          |
-| neat-csv (async/promise) | 9.09         | 0.05          | 19655          |
+| cisv (async/stream)      | 70.07        | 0.01          | 151485         |
+| papaparse (async/stream) | 21.58        | 0.02          | 46646          |
+| neat-csv (async/promise) | 9.77         | 0.05          | 21126          |
 
 
 ### ASYNCHRONOUS RESULTS (WITH DATA ACCESS)
 
 | Library                  | Speed (MB/s) | Avg Time (ms) | Operations/sec |
 |--------------------------|--------------|---------------|----------------|
-| cisv (async/stream)      | 24.59        | 0.02          | 53160          |
-| papaparse (async/stream) | 21.86        | 0.02          | 47260          |
-| neat-csv (async/promise) | 9.38         | 0.05          | 20283          |
+| cisv (async/stream)      | 25.23        | 0.02          | 54545          |
+| papaparse (async/stream) | 22.49        | 0.02          | 48622          |
+| neat-csv (async/promise) | 9.91         | 0.05          | 21428          |
 
 ## INSTALLATION
 

package/cisv/cisv_addon.cc

@@ -174,6 +174,8 @@ public:
         // Initialize configuration with defaults
         cisv_config_init(&config_);
 
+        config_.max_row_size = 0;
+
         // Handle constructor options if provided
         if (info.Length() > 0 && info[0].IsObject()) {
             Napi::Object options = info[0].As<Napi::Object>();
@@ -261,7 +263,10 @@ public:
 
         // Numeric options
         if (options.Has("maxRowSize")) {
-            config_.max_row_size = options.Get("maxRowSize").As<Napi::Number>().Uint32Value();
+            Napi::Value val = options.Get("maxRowSize");
+            if (!val.IsNull() && !val.IsUndefined()) {
+                config_.max_row_size = val.As<Napi::Number>().Uint32Value();
+            }
         }
 
         if (options.Has("fromLine")) {

package/cisv/cisv_parser.c

@@ -6,7 +6,6 @@
 #include <errno.h>
 #include <time.h>
 #include <stdbool.h>
-#include <ctype.h>
 // NOTE: not dealing with windows for now, too much issues
 #include <sys/mman.h>
 #include <fcntl.h>
@@ -14,7 +13,6 @@
 #include <getopt.h>
 #include <sys/time.h>
 #include "cisv_parser.h"
-#include "cisv_simd.h"
 
 #ifdef __AVX512F__
 #include <immintrin.h>
@@ -24,8 +22,8 @@
 #include <immintrin.h>
 #endif
 
-#define RINGBUF_SIZE (1 << 20) // 1 MiB (we may adjust according to needs)
-// #define RINGBUF_SIZE (1 << 16) // 64kb (for memory safe reasons)
+#define RINGBUF_SIZE (256 * 1024)
+#define DIRECT_PARSE_THRESHOLD (64 * 1024)  // Parse directly if chunk > 64KB
 #define PREFETCH_DISTANCE 256
 
 struct cisv_parser {
@@ -494,6 +492,8 @@ static void parse_simd_chunk(cisv_parser *parser, const uint8_t *buffer, size_t
                 // Handle newline
                 if (is_newline) {
                     yield_row(parser);
+                    parser->current_row_size = 0;
+                    parser->row_start = special_pos + 1;
                 }
 
                 // Update state branchlessly
@@ -786,6 +786,7 @@ static void parse_simd_chunk(cisv_parser *parser, const uint8_t *buffer, size_t
         if (action & ACT_ROW) {
             yield_row(parser);
             parser->current_row_size = 0;
+            parser->row_start = cur + 1;
         }
 
         cur += 1 - ((action & ACT_REPROCESS) >> 2);
@@ -808,6 +809,7 @@ static int parse_memory(cisv_parser *parser, const uint8_t *buffer, size_t len)
         // Yield final row if there's content
         if (parser->field_start > parser->row_start || !parser->skip_empty_lines) {
             yield_row(parser);
+            parser->current_row_size = 0;
         }
     }
     return 0;
@@ -1025,24 +1027,36 @@ int cisv_parser_parse_file(cisv_parser *parser, const char *path) {
 }
 
 int cisv_parser_write(cisv_parser *parser, const uint8_t *chunk, size_t len) {
-    if (!parser || !chunk || len >= RINGBUF_SIZE) return -EINVAL;
+    if (!parser || !chunk) return -EINVAL;
 
-    // Branchless overflow handling
-    size_t overflow = (parser->head + len > RINGBUF_SIZE);
-    if (overflow) {
-        parse_memory(parser, parser->ring, parser->head);
-        parser->head = 0;
+    // For large chunks, bypass ring buffer entirely
+    if (len > DIRECT_PARSE_THRESHOLD) {
+        // Parse directly - this is actually FASTER for large data
+        return parse_memory(parser, chunk, len);
+    }
+
+    // Small chunks use ring buffer for efficiency
+    if (parser->head + len > RINGBUF_SIZE) {
+        // Flush current buffer
+        if (parser->head > 0) {
+            parse_memory(parser, parser->ring, parser->head);
+            parser->head = 0;
+        }
+
+        // If still too large, parse directly
+        if (len > RINGBUF_SIZE) {
+            return parse_memory(parser, chunk, len);
+        }
     }
 
     memcpy(parser->ring + parser->head, chunk, len);
     parser->head += len;
 
-    // Check for newline or buffer threshold
-    uint8_t has_newline = (memchr(chunk, '\n', len) != NULL);
-    uint8_t threshold = (parser->head > (RINGBUF_SIZE / 2));
-    if (has_newline | threshold) {
-        parse_memory(parser, parser->ring, parser->head);
+    // Process on newline or when buffer is getting full
+    if (memchr(chunk, '\n', len) || parser->head > (RINGBUF_SIZE * 3 / 4)) {
+        int result = parse_memory(parser, parser->ring, parser->head);
         parser->head = 0;
+        return result;
     }
     return 0;
 }

package/cisv/cisv_transformer.c

@@ -5,12 +5,15 @@
 #include <ctype.h>
 #include <stdio.h>
 
+#ifdef __AVX512F__
+#include <immintrin.h>
+#endif
+
 #ifdef __AVX2__
 #include <immintrin.h>
 #endif
 
 #define TRANSFORM_POOL_SIZE (1 << 20)  // 1MB default pool
-// #define TRANSFORM_POOL_SIZE (1 << 16)  // 64kb (for memory safe reasons)
 #define SIMD_ALIGNMENT 64
 
 // Create transform pipeline

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cisv",
-  "version": "0.0.40",
+  "version": "0.0.41",
   "description": "The fastest csv parser of the multiverse",
   "author": "sanix<s4nixd@gmail.com>",
   "main": "./build/Release/cisv.node",
@@ -9,7 +9,7 @@
     "install": "node-gyp rebuild",
     "build": "node-gyp rebuild",
     "test": "mocha ./tests/*.test.js && bash ./test_transform.sh",
-    "test:build": "npm run test",
+    "test:build": "npm run build && npm run test",
     "benchmark": "node benchmark/benchmark.js",
     "lint": "clang-format -i cisv/*.{cc,h}",
     "prepublishOnly": "npm run benchmark",