cisv 0.0.33 → 0.0.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -19,44 +19,63 @@ I wrote about basics in a blog post, you can read here :https://sanixdk.xyz/blog
19
19
  - SIMD accelerated with AVX-512/AVX2 auto-detection
20
20
  - Dynamic lookup tables for configurable parsing
21
21
 
22
+ ## CLI BENCHMARKS WITH DOCKER
23
+
24
+ ```bash
25
+ $ docker build -t cisv-benchmark .
26
+ ```
27
+
28
+ To run them... choosing some specs for the container to size resources, you can :
29
+
30
+ ```bash
31
+ $ docker run --rm \
32
+ --cpus="2.0" \
33
+ --memory="4g" \
34
+ --memory-swap="4g" \
35
+ --cpu-shares=1024 \
36
+ --security-opt \
37
+ seccomp=unconfined \
38
+ cisv-benchmark
39
+ ```
40
+
22
41
  ## BENCHMARKS
23
42
 
24
43
  Benchmarks comparison with existing popular tools,
25
- cf pipeline you can check : (https://github.com/Sanix-Darker/cisv/actions/runs/17194915214/job/48775516036)
44
+ cf pipeline you can check : (https://github.com/Sanix-Darker/cisv/actions/runs/17697547058/job/50298916576) a step "Publish to npm"
26
45
 
27
46
  ### SYNCHRONOUS RESULTS
28
47
 
29
48
  | Library | Speed (MB/s) | Avg Time (ms) | Operations/sec |
30
49
  |--------------------|--------------|---------------|----------------|
31
- | cisv (sync) | 30.04 | 0.02 | 64936 |
32
- | csv-parse (sync) | 13.35 | 0.03 | 28870 |
33
- | papaparse (sync) | 25.16 | 0.02 | 54406 |
50
+ | cisv (sync) | 45.58 | 0.01 | 98543 |
51
+ | csv-parse (sync) | 18.11 | 0.03 | 39155 |
52
+ | papaparse (sync) | 28.03 | 0.02 | 60596 |
34
53
 
35
54
  ### SYNCHRONOUS RESULTS (WITH DATA ACCESS)
36
55
 
37
56
  | Library | Speed (MB/s) | Avg Time (ms) | Operations/sec |
38
57
  |--------------------|--------------|---------------|----------------|
39
- | cisv (sync) | 31.24 | 0.01 | 67543 |
40
- | csv-parse (sync) | 15.42 | 0.03 | 33335 |
41
- | papaparse (sync) | 25.49 | 0.02 | 55107 |
58
+ | cisv (sync) | 46.80 | 0.01 | 101185 |
59
+ | csv-parse (sync) | 18.92 | 0.02 | 40900 |
60
+ | papaparse (sync) | 28.38 | 0.02 | 61363 |
42
61
 
43
62
 
44
63
  ### ASYNCHRONOUS RESULTS
45
64
 
46
65
  | Library | Speed (MB/s) | Avg Time (ms) | Operations/sec |
47
66
  |--------------------------|--------------|---------------|----------------|
48
- | cisv (async/stream) | 61.31 | 0.01 | 132561 |
49
- | papaparse (async/stream) | 19.24 | 0.02 | 41603 |
50
- | neat-csv (async/promise) | 9.09 | 0.05 | 19655 |
67
+ | cisv (async/stream) | 70.07 | 0.01 | 151485 |
68
+ | papaparse (async/stream) | 21.58 | 0.02 | 46646 |
69
+ | neat-csv (async/promise) | 9.77 | 0.05 | 21126 |
51
70
 
52
71
 
53
72
  ### ASYNCHRONOUS RESULTS (WITH DATA ACCESS)
54
73
 
55
74
  | Library | Speed (MB/s) | Avg Time (ms) | Operations/sec |
56
75
  |--------------------------|--------------|---------------|----------------|
57
- | cisv (async/stream) | 24.59 | 0.02 | 53160 |
58
- | papaparse (async/stream) | 21.86 | 0.02 | 47260 |
59
- | neat-csv (async/promise) | 9.38 | 0.05 | 20283 |
76
+ | cisv (async/stream) | 25.23 | 0.02 | 54545 |
77
+ | papaparse (async/stream) | 22.49 | 0.02 | 48622 |
78
+ | neat-csv (async/promise) | 9.91 | 0.05 | 21428 |
60
79
 
61
80
  ## INSTALLATION
62
81
 
@@ -174,6 +174,8 @@ public:
174
174
  // Initialize configuration with defaults
175
175
  cisv_config_init(&config_);
176
176
 
177
+ config_.max_row_size = 0;
178
+
177
179
  // Handle constructor options if provided
178
180
  if (info.Length() > 0 && info[0].IsObject()) {
179
181
  Napi::Object options = info[0].As<Napi::Object>();
@@ -261,7 +263,10 @@ public:
261
263
 
262
264
  // Numeric options
263
265
  if (options.Has("maxRowSize")) {
264
- config_.max_row_size = options.Get("maxRowSize").As<Napi::Number>().Uint32Value();
266
+ Napi::Value val = options.Get("maxRowSize");
267
+ if (!val.IsNull() && !val.IsUndefined()) {
268
+ config_.max_row_size = val.As<Napi::Number>().Uint32Value();
269
+ }
265
270
  }
266
271
 
267
272
  if (options.Has("fromLine")) {
@@ -6,7 +6,6 @@
6
6
  #include <errno.h>
7
7
  #include <time.h>
8
8
  #include <stdbool.h>
9
- #include <ctype.h>
10
9
  // NOTE: not dealing with windows for now, too much issues
11
10
  #include <sys/mman.h>
12
11
  #include <fcntl.h>
@@ -14,10 +13,17 @@
14
13
  #include <getopt.h>
15
14
  #include <sys/time.h>
16
15
  #include "cisv_parser.h"
17
- #include "cisv_simd.h"
18
16
 
19
- #define RINGBUF_SIZE (1 << 20) // 1 MiB (we may adjust according to needs)
20
- // #define RINGBUF_SIZE (1 << 16) // 64kb (for memory safe reasons)
17
+ #ifdef __AVX512F__
18
+ #include <immintrin.h>
19
+ #endif
20
+
21
+ #ifdef __AVX2__
22
+ #include <immintrin.h>
23
+ #endif
24
+
25
+ #define RINGBUF_SIZE (256 * 1024)
26
+ #define DIRECT_PARSE_THRESHOLD (64 * 1024) // Parse directly if chunk > 64KB
21
27
  #define PREFETCH_DISTANCE 256
22
28
 
23
29
  struct cisv_parser {
@@ -95,133 +101,285 @@ void cisv_config_init(cisv_config *config) {
95
101
  static void init_tables(cisv_parser *parser) {
96
102
  if (parser->tables_initialized) return;
97
103
 
98
- // Allocate tables if not already allocated
104
+ // Allocate both tables in one allocation for better cache locality
99
105
  if (!parser->state_table) {
100
- parser->state_table = calloc(4 * 256, sizeof(uint8_t));
101
- parser->action_table = calloc(4 * 256, sizeof(uint8_t));
102
- if (!parser->state_table || !parser->action_table) {
103
- return; // Handle allocation failure gracefully
104
- }
106
+ parser->state_table = aligned_alloc(64, 8 * 256); // Align to cache line
107
+ if (!parser->state_table) return;
108
+ parser->action_table = parser->state_table + (4 * 256);
109
+ memset(parser->state_table, 0, 8 * 256);
105
110
  }
106
111
 
107
- // Get table pointers for easier access
108
- uint8_t (*state_table)[256] = (uint8_t (*)[256])parser->state_table;
109
- uint8_t (*action_table)[256] = (uint8_t (*)[256])parser->action_table;
112
+ uint8_t (*st)[256] = (uint8_t (*)[256])parser->state_table;
113
+ uint8_t (*at)[256] = (uint8_t (*)[256])parser->action_table;
114
+
115
+ // Unroll initialization loops for better performance
116
+ // Pre-calculate commonly used values
117
+ const uint8_t q = parser->quote;
118
+ const uint8_t d = parser->delimiter;
119
+ const uint8_t e = parser->escape;
120
+ const uint8_t c = parser->comment;
121
+
122
+ // Initialize with SIMD where possible
123
+ #ifdef __AVX2__
124
+ __m256i unquoted_state = _mm256_set1_epi8(S_UNQUOTED);
125
+ __m256i quoted_state = _mm256_set1_epi8(S_QUOTED);
126
+ __m256i comment_state = _mm256_set1_epi8(S_COMMENT);
127
+
128
+ for (int i = 0; i < 256; i += 32) {
129
+ _mm256_store_si256((__m256i*)&st[S_UNQUOTED][i], unquoted_state);
130
+ _mm256_store_si256((__m256i*)&st[S_QUOTED][i], quoted_state);
131
+ _mm256_store_si256((__m256i*)&st[S_COMMENT][i], comment_state);
132
+ }
133
+ #else
134
+ memset(st[S_UNQUOTED], S_UNQUOTED, 256);
135
+ memset(st[S_QUOTED], S_QUOTED, 256);
136
+ memset(st[S_COMMENT], S_COMMENT, 256);
137
+ #endif
110
138
 
111
- // Initialize state transitions
112
- for (int c = 0; c < 256; c++) {
113
- // S_UNQUOTED transitions
114
- state_table[S_UNQUOTED][c] = S_UNQUOTED;
115
- if (c == parser->quote) {
116
- state_table[S_UNQUOTED][c] = S_QUOTED;
117
- } else if (parser->comment && c == parser->comment) {
118
- state_table[S_UNQUOTED][c] = S_COMMENT;
119
- }
139
+ // Set special transitions
140
+ st[S_UNQUOTED][q] = S_QUOTED;
141
+ if (c) st[S_UNQUOTED][c] = S_COMMENT;
120
142
 
121
- // S_QUOTED transitions
122
- state_table[S_QUOTED][c] = S_QUOTED;
123
- if (parser->escape && c == parser->escape) {
124
- state_table[S_QUOTED][c] = S_QUOTE_ESC;
125
- } else if (c == parser->quote) {
126
- state_table[S_QUOTED][c] = S_QUOTE_ESC;
127
- }
143
+ if (e) {
144
+ st[S_QUOTED][e] = S_QUOTE_ESC;
145
+ memset(st[S_QUOTE_ESC], S_QUOTED, 256);
146
+ } else {
147
+ st[S_QUOTED][q] = S_QUOTE_ESC;
148
+ memset(st[S_QUOTE_ESC], S_UNQUOTED, 256);
149
+ st[S_QUOTE_ESC][q] = S_QUOTED;
150
+ }
128
151
 
129
- // S_QUOTE_ESC transitions
130
- if (parser->escape) {
131
- // With explicit escape character, always return to quoted state
132
- state_table[S_QUOTE_ESC][c] = S_QUOTED;
133
- } else {
134
- // RFC4180-style: "" becomes a literal quote
135
- if (c == parser->quote) {
136
- state_table[S_QUOTE_ESC][c] = S_QUOTED;
137
- } else {
138
- state_table[S_QUOTE_ESC][c] = S_UNQUOTED;
139
- }
140
- }
152
+ st[S_COMMENT]['\n'] = S_UNQUOTED;
141
153
 
142
- // S_COMMENT transitions - stay in comment until newline
143
- state_table[S_COMMENT][c] = S_COMMENT;
144
- if (c == '\n') {
145
- state_table[S_COMMENT][c] = S_UNQUOTED;
154
+ // Initialize actions with minimal branches
155
+ memset(at, ACT_NONE, 4 * 256);
156
+ at[S_UNQUOTED][d] = ACT_FIELD;
157
+ at[S_UNQUOTED]['\n'] = ACT_FIELD | ACT_ROW;
158
+ at[S_UNQUOTED]['\r'] = ACT_FIELD;
159
+
160
+ if (!e) {
161
+ // Vectorize the action table initialization
162
+ for (int i = 0; i < 256; i++) {
163
+ at[S_QUOTE_ESC][i] = (i != q) ? ACT_REPROCESS : ACT_NONE;
146
164
  }
147
165
  }
148
166
 
149
- // Initialize action table
150
- memset(action_table, ACT_NONE, 4 * 256);
167
+ // Use SIMD for comment actions
168
+ #ifdef __AVX2__
169
+ __m256i skip_act = _mm256_set1_epi8(ACT_SKIP);
170
+ for (int i = 0; i < 256; i += 32) {
171
+ _mm256_store_si256((__m256i*)&at[S_COMMENT][i], skip_act);
172
+ }
173
+ #else
174
+ memset(at[S_COMMENT], ACT_SKIP, 256);
175
+ #endif
176
+ at[S_COMMENT]['\n'] = ACT_ROW;
151
177
 
152
- // S_UNQUOTED actions
153
- action_table[S_UNQUOTED][(uint8_t)parser->delimiter] = ACT_FIELD;
154
- action_table[S_UNQUOTED]['\n'] = ACT_FIELD | ACT_ROW;
155
- action_table[S_UNQUOTED]['\r'] = ACT_FIELD; // Handle CRLF
178
+ parser->tables_initialized = 1;
179
+ }
156
180
 
157
- // S_QUOTE_ESC actions
158
- if (!parser->escape) {
159
- // RFC4180-style: reprocess non-quote characters
160
- for (int c = 0; c < 256; c++) {
161
- if (c != parser->quote) {
162
- action_table[S_QUOTE_ESC][c] = ACT_REPROCESS;
181
+ // SIMD-optimized whitespace detection lookup table
182
+ // Ultra-fast trimming with AVX512/AVX2
183
+ static inline const uint8_t* trim_start(const uint8_t *start, const uint8_t *end) {
184
+ size_t len = end - start;
185
+
186
+ #ifdef __AVX512F__
187
+ if (len >= 64) {
188
+ const __m512i max_ws = _mm512_set1_epi8(32);
189
+
190
+ while (len >= 64) {
191
+ __m512i chunk = _mm512_loadu_si512(start);
192
+ __mmask64 is_ws = _mm512_cmple_epu8_mask(chunk, max_ws);
193
+
194
+ if (is_ws != 0xFFFFFFFFFFFFFFFFULL) {
195
+ return start + __builtin_ctzll(~is_ws);
163
196
  }
197
+ start += 64;
198
+ len -= 64;
164
199
  }
165
200
  }
201
+ #elif defined(__AVX2__)
202
+ if (len >= 32) {
203
+ const __m256i max_ws = _mm256_set1_epi8(32);
204
+
205
+ while (len >= 32) {
206
+ __m256i chunk = _mm256_loadu_si256((__m256i*)start);
207
+ __m256i cmp = _mm256_cmpgt_epi8(chunk, max_ws);
208
+ uint32_t mask = _mm256_movemask_epi8(cmp);
166
209
 
167
- // S_COMMENT actions - skip everything except newline
168
- for (int c = 0; c < 256; c++) {
169
- action_table[S_COMMENT][c] = ACT_SKIP;
210
+ if (mask) {
211
+ return start + __builtin_ctz(mask);
212
+ }
213
+ start += 32;
214
+ len -= 32;
215
+ }
170
216
  }
171
- action_table[S_COMMENT]['\n'] = ACT_ROW;
217
+ #endif
172
218
 
173
- parser->tables_initialized = 1;
174
- }
219
+ // Unrolled 8-byte processing
220
+ while (len >= 8) {
221
+ uint64_t v = *(uint64_t*)start;
222
+ uint64_t has_non_ws = ((v & 0xE0E0E0E0E0E0E0E0ULL) != 0) |
223
+ ((v & 0x1F1F1F1F1F1F1F1FULL) > 0x0D0D0D0D0D0D0D0DULL);
224
+ if (has_non_ws) {
225
+ for (int i = 0; i < 8; i++) {
226
+ if ((uint8_t)(v >> (i*8)) > 32) return start + i;
227
+ }
228
+ }
229
+ start += 8;
230
+ len -= 8;
231
+ }
175
232
 
176
- static inline const uint8_t* trim_start(const uint8_t *start, const uint8_t *end) {
177
- while (start < end && isspace(*start)) start++;
178
- return start;
179
- }
233
+ // 4-byte processing
234
+ if (len >= 4) {
235
+ uint32_t v = *(uint32_t*)start;
236
+ for (int i = 0; i < 4; i++) {
237
+ uint8_t c = (v >> (i*8)) & 0xFF;
238
+ if (c > 32) return start + i;
239
+ }
240
+ start += 4;
241
+ len -= 4;
242
+ }
243
+
244
+ // Remainder
245
+ switch(len) {
246
+ case 3: if (*start > 32) return start; start++;
247
+ /* fallthrough */
248
+ case 2: if (*start > 32) return start; start++;
249
+ /* fallthrough */
250
+ case 1: if (*start > 32) return start; start++;
251
+ }
180
252
 
181
- static inline const uint8_t* trim_end(const uint8_t *start, const uint8_t *end) {
182
- while (end > start && isspace(*(end - 1))) end--;
183
253
  return end;
184
254
  }
185
255
 
186
- static inline void yield_field(cisv_parser *parser, const uint8_t *start, const uint8_t *end) {
187
- // Apply trimming if configured
188
- if (parser->trim) {
189
- start = trim_start(start, end);
190
- end = trim_end(start, end);
256
+ static inline const uint8_t* trim_end(const uint8_t *start, const uint8_t *end) {
257
+ size_t len = end - start;
258
+
259
+ #ifdef __AVX512F__
260
+ while (len >= 64) {
261
+ const uint8_t *check = end - 64;
262
+ __m512i chunk = _mm512_loadu_si512(check);
263
+ const __m512i max_ws = _mm512_set1_epi8(32);
264
+ __mmask64 is_non_ws = _mm512_cmpgt_epu8_mask(chunk, max_ws);
265
+
266
+ if (is_non_ws) {
267
+ int last_non_ws = 63 - __builtin_clzll(is_non_ws);
268
+ return check + last_non_ws + 1;
269
+ }
270
+ end -= 64;
271
+ len -= 64;
272
+ }
273
+ #elif defined(__AVX2__)
274
+ while (len >= 32) {
275
+ const uint8_t *check = end - 32;
276
+ __m256i chunk = _mm256_loadu_si256((__m256i*)check);
277
+ const __m256i max_ws = _mm256_set1_epi8(32);
278
+ __m256i cmp = _mm256_cmpgt_epi8(chunk, max_ws);
279
+ uint32_t mask = _mm256_movemask_epi8(cmp);
280
+
281
+ if (mask) {
282
+ int last_non_ws = 31 - __builtin_clz(mask);
283
+ return check + last_non_ws + 1;
284
+ }
285
+ end -= 32;
286
+ len -= 32;
191
287
  }
288
+ #endif
289
+
290
+ // Unrolled 8-byte processing
291
+ while (len >= 8) {
292
+ const uint8_t *check = end - 8;
293
+ uint64_t v = *(uint64_t*)check;
192
294
 
193
- // Branchless check: multiply callback by validity flag
194
- size_t valid = (parser->fcb != NULL) & (start != NULL) & (end != NULL) & (end >= start);
195
- if (valid) {
196
- parser->fcb(parser->user, (const char *)start, (size_t)(end - start));
295
+ for (int i = 7; i >= 0; i--) {
296
+ if ((uint8_t)(v >> (i*8)) > 32) return check + i + 1;
297
+ }
298
+ end -= 8;
299
+ len -= 8;
197
300
  }
198
- }
199
301
 
200
- static inline void yield_row(cisv_parser *parser) {
201
- // Check if we should skip empty lines
202
- if (parser->skip_empty_lines && parser->field_start == parser->row_start) {
203
- parser->row_start = parser->field_start;
204
- return;
302
+ // 4-byte processing
303
+ if (len >= 4) {
304
+ const uint8_t *check = end - 4;
305
+ uint32_t v = *(uint32_t*)check;
306
+ for (int i = 3; i >= 0; i--) {
307
+ if ((uint8_t)(v >> (i*8)) > 32) return check + i + 1;
308
+ }
309
+ end -= 4;
310
+ len -= 4;
205
311
  }
206
312
 
207
- // Check line range
208
- if (parser->current_line < parser->from_line) {
209
- parser->current_line++;
210
- parser->row_start = parser->field_start;
211
- return;
313
+ // Remainder
314
+ while (len-- > 0) {
315
+ if (*(--end) > 32) return end + 1;
212
316
  }
213
317
 
214
- if (parser->to_line > 0 && parser->current_line > parser->to_line) {
215
- return;
318
+ return start;
319
+ }
320
+
321
+ // yield_field with prefetching and branchless code
322
+ static inline void yield_field(cisv_parser *parser, const uint8_t *start, const uint8_t *end) {
323
+ // Prefetch parser structure for next access
324
+ __builtin_prefetch(parser, 0, 3);
325
+
326
+ // Branchless trimming using conditional move
327
+ const uint8_t *s = start;
328
+ const uint8_t *e = end;
329
+
330
+ // Use conditional assignment instead of branch
331
+ const uint8_t *trimmed_s = trim_start(s, e);
332
+ const uint8_t *trimmed_e = trim_end(trimmed_s, e);
333
+
334
+ // Branchless selection: if trim is 0, use original, if 1, use trimmed
335
+ uintptr_t mask = -(uintptr_t)parser->trim;
336
+ s = (const uint8_t*)(((uintptr_t)trimmed_s & mask) | ((uintptr_t)s & ~mask));
337
+ e = (const uint8_t*)(((uintptr_t)trimmed_e & mask) | ((uintptr_t)e & ~mask));
338
+
339
+ // Combine all conditions into single branch
340
+ uintptr_t fcb_addr = (uintptr_t)parser->fcb;
341
+ uintptr_t valid_mask = -(fcb_addr != 0);
342
+ valid_mask &= -(s != 0);
343
+ valid_mask &= -(e != 0);
344
+ valid_mask &= -(e >= s);
345
+
346
+ // Single branch for callback execution
347
+ if (valid_mask) {
348
+ // Prefetch user data for callback
349
+ __builtin_prefetch(parser->user, 0, 1);
350
+ parser->fcb(parser->user, (const char *)s, (size_t)(e - s));
216
351
  }
352
+ }
217
353
 
218
- if (parser->rcb) {
354
+ // yield_row with reduced branches
355
+ static inline void yield_row(cisv_parser *parser) {
356
+ // Prefetch frequently accessed memory
357
+ __builtin_prefetch(&parser->current_line, 1, 3);
358
+ __builtin_prefetch(&parser->row_start, 1, 3);
359
+
360
+ // Compute all conditions upfront
361
+ int is_empty_line = (parser->field_start == parser->row_start);
362
+ int skip_empty = parser->skip_empty_lines & is_empty_line;
363
+ int before_range = (parser->current_line < parser->from_line);
364
+ int after_range = (parser->to_line > 0) & (parser->current_line > parser->to_line);
365
+ int in_range = !before_range & !after_range;
366
+
367
+ // Branchless increment of current_line (always happens except when after range)
368
+ parser->current_line += !after_range;
369
+
370
+ // Branchless update of row_start (happens except when after range)
371
+ uintptr_t new_row_start = (uintptr_t)parser->field_start;
372
+ uintptr_t old_row_start = (uintptr_t)parser->row_start;
373
+ parser->row_start = (uint8_t*)((old_row_start & -after_range) | (new_row_start & ~(-after_range)));
374
+
375
+ // Branchless reset of row_size
376
+ parser->current_row_size &= after_range;
377
+
378
+ // Single branch for callback (most common case last for better prediction)
379
+ if ((!skip_empty) & in_range & (parser->rcb != NULL)) {
380
+ __builtin_prefetch(parser->user, 0, 1);
219
381
  parser->rcb(parser->user);
220
382
  }
221
-
222
- parser->current_line++;
223
- parser->row_start = parser->field_start;
224
- parser->current_row_size = 0;
225
383
  }
226
384
 
227
385
  static inline void handle_error(cisv_parser *parser, const char *msg) {
@@ -334,6 +492,8 @@ static void parse_simd_chunk(cisv_parser *parser, const uint8_t *buffer, size_t
334
492
  // Handle newline
335
493
  if (is_newline) {
336
494
  yield_row(parser);
495
+ parser->current_row_size = 0;
496
+ parser->row_start = special_pos + 1;
337
497
  }
338
498
 
339
499
  // Update state branchlessly
@@ -626,6 +786,7 @@ static void parse_simd_chunk(cisv_parser *parser, const uint8_t *buffer, size_t
626
786
  if (action & ACT_ROW) {
627
787
  yield_row(parser);
628
788
  parser->current_row_size = 0;
789
+ parser->row_start = cur + 1;
629
790
  }
630
791
 
631
792
  cur += 1 - ((action & ACT_REPROCESS) >> 2);
@@ -648,6 +809,7 @@ static int parse_memory(cisv_parser *parser, const uint8_t *buffer, size_t len)
648
809
  // Yield final row if there's content
649
810
  if (parser->field_start > parser->row_start || !parser->skip_empty_lines) {
650
811
  yield_row(parser);
812
+ parser->current_row_size = 0;
651
813
  }
652
814
  }
653
815
  return 0;
@@ -865,24 +1027,36 @@ int cisv_parser_parse_file(cisv_parser *parser, const char *path) {
865
1027
  }
866
1028
 
867
1029
  int cisv_parser_write(cisv_parser *parser, const uint8_t *chunk, size_t len) {
868
- if (!parser || !chunk || len >= RINGBUF_SIZE) return -EINVAL;
1030
+ if (!parser || !chunk) return -EINVAL;
869
1031
 
870
- // Branchless overflow handling
871
- size_t overflow = (parser->head + len > RINGBUF_SIZE);
872
- if (overflow) {
873
- parse_memory(parser, parser->ring, parser->head);
874
- parser->head = 0;
1032
+ // For large chunks, bypass ring buffer entirely
1033
+ if (len > DIRECT_PARSE_THRESHOLD) {
1034
+ // Parse directly - this is actually FASTER for large data
1035
+ return parse_memory(parser, chunk, len);
1036
+ }
1037
+
1038
+ // Small chunks use ring buffer for efficiency
1039
+ if (parser->head + len > RINGBUF_SIZE) {
1040
+ // Flush current buffer
1041
+ if (parser->head > 0) {
1042
+ parse_memory(parser, parser->ring, parser->head);
1043
+ parser->head = 0;
1044
+ }
1045
+
1046
+ // If still too large, parse directly
1047
+ if (len > RINGBUF_SIZE) {
1048
+ return parse_memory(parser, chunk, len);
1049
+ }
875
1050
  }
876
1051
 
877
1052
  memcpy(parser->ring + parser->head, chunk, len);
878
1053
  parser->head += len;
879
1054
 
880
- // Check for newline or buffer threshold
881
- uint8_t has_newline = (memchr(chunk, '\n', len) != NULL);
882
- uint8_t threshold = (parser->head > (RINGBUF_SIZE / 2));
883
- if (has_newline | threshold) {
884
- parse_memory(parser, parser->ring, parser->head);
1055
+ // Process on newline or when buffer is getting full
1056
+ if (memchr(chunk, '\n', len) || parser->head > (RINGBUF_SIZE * 3 / 4)) {
1057
+ int result = parse_memory(parser, parser->ring, parser->head);
885
1058
  parser->head = 0;
1059
+ return result;
886
1060
  }
887
1061
  return 0;
888
1062
  }
@@ -5,12 +5,15 @@
5
5
  #include <ctype.h>
6
6
  #include <stdio.h>
7
7
 
8
+ #ifdef __AVX512F__
9
+ #include <immintrin.h>
10
+ #endif
11
+
8
12
  #ifdef __AVX2__
9
13
  #include <immintrin.h>
10
14
  #endif
11
15
 
12
16
  #define TRANSFORM_POOL_SIZE (1 << 20) // 1MB default pool
13
- // #define TRANSFORM_POOL_SIZE (1 << 16) // 64kb (for memory safe reasons)
14
17
  #define SIMD_ALIGNMENT 64
15
18
 
16
19
  // Create transform pipeline
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "cisv",
3
- "version": "0.0.33",
3
+ "version": "0.0.41",
4
4
  "description": "The fastest csv parser of the multiverse",
5
5
  "author": "sanix<s4nixd@gmail.com>",
6
6
  "main": "./build/Release/cisv.node",
@@ -9,7 +9,7 @@
9
9
  "install": "node-gyp rebuild",
10
10
  "build": "node-gyp rebuild",
11
11
  "test": "mocha ./tests/*.test.js && bash ./test_transform.sh",
12
- "test:build": "npm run test",
12
+ "test:build": "npm run build && npm run test",
13
13
  "benchmark": "node benchmark/benchmark.js",
14
14
  "lint": "clang-format -i cisv/*.{cc,h}",
15
15
  "prepublishOnly": "npm run benchmark",