cisv 0.0.33 → 0.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -13
- package/cisv/cisv_addon.cc +6 -1
- package/cisv/cisv_parser.c +281 -107
- package/cisv/cisv_transformer.c +4 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -19,44 +19,63 @@ I wrote about basics in a blog post, you can read here :https://sanixdk.xyz/blog
|
|
|
19
19
|
- SIMD accelerated with AVX-512/AVX2 auto-detection
|
|
20
20
|
- Dynamic lookup tables for configurable parsing
|
|
21
21
|
|
|
22
|
+
## CLI BENCHMARKS WITH DOCKER
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
$ docker build -t cisv-benchmark .
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
To run them... choosing some specs for the container to size resources, you can :
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
$ docker run --rm \
|
|
32
|
+
--cpus="2.0" \
|
|
33
|
+
--memory="4g" \
|
|
34
|
+
--memory-swap="4g" \
|
|
35
|
+
--cpu-shares=1024 \
|
|
36
|
+
--security-opt \
|
|
37
|
+
seccomp=unconfined \
|
|
38
|
+
cisv-benchmark
|
|
39
|
+
```
|
|
40
|
+
|
|
22
41
|
## BENCHMARKS
|
|
23
42
|
|
|
24
43
|
Benchmarks comparison with existing popular tools,
|
|
25
|
-
cf pipeline you can check : (https://github.com/Sanix-Darker/cisv/actions/runs/
|
|
44
|
+
cf pipeline you can check : (https://github.com/Sanix-Darker/cisv/actions/runs/17697547058/job/50298916576) a step "Publish to npm"
|
|
26
45
|
|
|
27
46
|
### SYNCHRONOUS RESULTS
|
|
28
47
|
|
|
29
48
|
| Library | Speed (MB/s) | Avg Time (ms) | Operations/sec |
|
|
30
49
|
|--------------------|--------------|---------------|----------------|
|
|
31
|
-
| cisv (sync) |
|
|
32
|
-
| csv-parse (sync) |
|
|
33
|
-
| papaparse (sync) |
|
|
50
|
+
| cisv (sync) | 45.58 | 0.01 | 98543 |
|
|
51
|
+
| csv-parse (sync) | 18.11 | 0.03 | 39155 |
|
|
52
|
+
| papaparse (sync) | 28.03 | 0.02 | 60596 |
|
|
34
53
|
|
|
35
54
|
### SYNCHRONOUS RESULTS (WITH DATA ACCESS)
|
|
36
55
|
|
|
37
56
|
| Library | Speed (MB/s) | Avg Time (ms) | Operations/sec |
|
|
38
57
|
|--------------------|--------------|---------------|----------------|
|
|
39
|
-
| cisv (sync) |
|
|
40
|
-
| csv-parse (sync) |
|
|
41
|
-
| papaparse (sync) |
|
|
58
|
+
| cisv (sync) | 46.80 | 0.01 | 101185 |
|
|
59
|
+
| csv-parse (sync) | 18.92 | 0.02 | 40900 |
|
|
60
|
+
| papaparse (sync) | 28.38 | 0.02 | 61363 |
|
|
42
61
|
|
|
43
62
|
|
|
44
63
|
### ASYNCHRONOUS RESULTS
|
|
45
64
|
|
|
46
65
|
| Library | Speed (MB/s) | Avg Time (ms) | Operations/sec |
|
|
47
66
|
|--------------------------|--------------|---------------|----------------|
|
|
48
|
-
| cisv (async/stream) |
|
|
49
|
-
| papaparse (async/stream) |
|
|
50
|
-
| neat-csv (async/promise) | 9.
|
|
67
|
+
| cisv (async/stream) | 70.07 | 0.01 | 151485 |
|
|
68
|
+
| papaparse (async/stream) | 21.58 | 0.02 | 46646 |
|
|
69
|
+
| neat-csv (async/promise) | 9.77 | 0.05 | 21126 |
|
|
51
70
|
|
|
52
71
|
|
|
53
72
|
### ASYNCHRONOUS RESULTS (WITH DATA ACCESS)
|
|
54
73
|
|
|
55
74
|
| Library | Speed (MB/s) | Avg Time (ms) | Operations/sec |
|
|
56
75
|
|--------------------------|--------------|---------------|----------------|
|
|
57
|
-
| cisv (async/stream) |
|
|
58
|
-
| papaparse (async/stream) |
|
|
59
|
-
| neat-csv (async/promise) | 9.
|
|
76
|
+
| cisv (async/stream) | 25.23 | 0.02 | 54545 |
|
|
77
|
+
| papaparse (async/stream) | 22.49 | 0.02 | 48622 |
|
|
78
|
+
| neat-csv (async/promise) | 9.91 | 0.05 | 21428 |
|
|
60
79
|
|
|
61
80
|
## INSTALLATION
|
|
62
81
|
|
package/cisv/cisv_addon.cc
CHANGED
|
@@ -174,6 +174,8 @@ public:
|
|
|
174
174
|
// Initialize configuration with defaults
|
|
175
175
|
cisv_config_init(&config_);
|
|
176
176
|
|
|
177
|
+
config_.max_row_size = 0;
|
|
178
|
+
|
|
177
179
|
// Handle constructor options if provided
|
|
178
180
|
if (info.Length() > 0 && info[0].IsObject()) {
|
|
179
181
|
Napi::Object options = info[0].As<Napi::Object>();
|
|
@@ -261,7 +263,10 @@ public:
|
|
|
261
263
|
|
|
262
264
|
// Numeric options
|
|
263
265
|
if (options.Has("maxRowSize")) {
|
|
264
|
-
|
|
266
|
+
Napi::Value val = options.Get("maxRowSize");
|
|
267
|
+
if (!val.IsNull() && !val.IsUndefined()) {
|
|
268
|
+
config_.max_row_size = val.As<Napi::Number>().Uint32Value();
|
|
269
|
+
}
|
|
265
270
|
}
|
|
266
271
|
|
|
267
272
|
if (options.Has("fromLine")) {
|
package/cisv/cisv_parser.c
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
#include <errno.h>
|
|
7
7
|
#include <time.h>
|
|
8
8
|
#include <stdbool.h>
|
|
9
|
-
#include <ctype.h>
|
|
10
9
|
// NOTE: not dealing with windows for now, too much issues
|
|
11
10
|
#include <sys/mman.h>
|
|
12
11
|
#include <fcntl.h>
|
|
@@ -14,10 +13,17 @@
|
|
|
14
13
|
#include <getopt.h>
|
|
15
14
|
#include <sys/time.h>
|
|
16
15
|
#include "cisv_parser.h"
|
|
17
|
-
#include "cisv_simd.h"
|
|
18
16
|
|
|
19
|
-
#
|
|
20
|
-
|
|
17
|
+
#ifdef __AVX512F__
|
|
18
|
+
#include <immintrin.h>
|
|
19
|
+
#endif
|
|
20
|
+
|
|
21
|
+
#ifdef __AVX2__
|
|
22
|
+
#include <immintrin.h>
|
|
23
|
+
#endif
|
|
24
|
+
|
|
25
|
+
#define RINGBUF_SIZE (256 * 1024)
|
|
26
|
+
#define DIRECT_PARSE_THRESHOLD (64 * 1024) // Parse directly if chunk > 64KB
|
|
21
27
|
#define PREFETCH_DISTANCE 256
|
|
22
28
|
|
|
23
29
|
struct cisv_parser {
|
|
@@ -95,133 +101,285 @@ void cisv_config_init(cisv_config *config) {
|
|
|
95
101
|
static void init_tables(cisv_parser *parser) {
|
|
96
102
|
if (parser->tables_initialized) return;
|
|
97
103
|
|
|
98
|
-
// Allocate tables
|
|
104
|
+
// Allocate both tables in one allocation for better cache locality
|
|
99
105
|
if (!parser->state_table) {
|
|
100
|
-
parser->state_table =
|
|
101
|
-
parser->
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
}
|
|
106
|
+
parser->state_table = aligned_alloc(64, 8 * 256); // Align to cache line
|
|
107
|
+
if (!parser->state_table) return;
|
|
108
|
+
parser->action_table = parser->state_table + (4 * 256);
|
|
109
|
+
memset(parser->state_table, 0, 8 * 256);
|
|
105
110
|
}
|
|
106
111
|
|
|
107
|
-
|
|
108
|
-
uint8_t (*
|
|
109
|
-
|
|
112
|
+
uint8_t (*st)[256] = (uint8_t (*)[256])parser->state_table;
|
|
113
|
+
uint8_t (*at)[256] = (uint8_t (*)[256])parser->action_table;
|
|
114
|
+
|
|
115
|
+
// Unroll initialization loops for better performance
|
|
116
|
+
// Pre-calculate commonly used values
|
|
117
|
+
const uint8_t q = parser->quote;
|
|
118
|
+
const uint8_t d = parser->delimiter;
|
|
119
|
+
const uint8_t e = parser->escape;
|
|
120
|
+
const uint8_t c = parser->comment;
|
|
121
|
+
|
|
122
|
+
// Initialize with SIMD where possible
|
|
123
|
+
#ifdef __AVX2__
|
|
124
|
+
__m256i unquoted_state = _mm256_set1_epi8(S_UNQUOTED);
|
|
125
|
+
__m256i quoted_state = _mm256_set1_epi8(S_QUOTED);
|
|
126
|
+
__m256i comment_state = _mm256_set1_epi8(S_COMMENT);
|
|
127
|
+
|
|
128
|
+
for (int i = 0; i < 256; i += 32) {
|
|
129
|
+
_mm256_store_si256((__m256i*)&st[S_UNQUOTED][i], unquoted_state);
|
|
130
|
+
_mm256_store_si256((__m256i*)&st[S_QUOTED][i], quoted_state);
|
|
131
|
+
_mm256_store_si256((__m256i*)&st[S_COMMENT][i], comment_state);
|
|
132
|
+
}
|
|
133
|
+
#else
|
|
134
|
+
memset(st[S_UNQUOTED], S_UNQUOTED, 256);
|
|
135
|
+
memset(st[S_QUOTED], S_QUOTED, 256);
|
|
136
|
+
memset(st[S_COMMENT], S_COMMENT, 256);
|
|
137
|
+
#endif
|
|
110
138
|
|
|
111
|
-
//
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
state_table[S_UNQUOTED][c] = S_UNQUOTED;
|
|
115
|
-
if (c == parser->quote) {
|
|
116
|
-
state_table[S_UNQUOTED][c] = S_QUOTED;
|
|
117
|
-
} else if (parser->comment && c == parser->comment) {
|
|
118
|
-
state_table[S_UNQUOTED][c] = S_COMMENT;
|
|
119
|
-
}
|
|
139
|
+
// Set special transitions
|
|
140
|
+
st[S_UNQUOTED][q] = S_QUOTED;
|
|
141
|
+
if (c) st[S_UNQUOTED][c] = S_COMMENT;
|
|
120
142
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
143
|
+
if (e) {
|
|
144
|
+
st[S_QUOTED][e] = S_QUOTE_ESC;
|
|
145
|
+
memset(st[S_QUOTE_ESC], S_QUOTED, 256);
|
|
146
|
+
} else {
|
|
147
|
+
st[S_QUOTED][q] = S_QUOTE_ESC;
|
|
148
|
+
memset(st[S_QUOTE_ESC], S_UNQUOTED, 256);
|
|
149
|
+
st[S_QUOTE_ESC][q] = S_QUOTED;
|
|
150
|
+
}
|
|
128
151
|
|
|
129
|
-
|
|
130
|
-
if (parser->escape) {
|
|
131
|
-
// With explicit escape character, always return to quoted state
|
|
132
|
-
state_table[S_QUOTE_ESC][c] = S_QUOTED;
|
|
133
|
-
} else {
|
|
134
|
-
// RFC4180-style: "" becomes a literal quote
|
|
135
|
-
if (c == parser->quote) {
|
|
136
|
-
state_table[S_QUOTE_ESC][c] = S_QUOTED;
|
|
137
|
-
} else {
|
|
138
|
-
state_table[S_QUOTE_ESC][c] = S_UNQUOTED;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
152
|
+
st[S_COMMENT]['\n'] = S_UNQUOTED;
|
|
141
153
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
154
|
+
// Initialize actions with minimal branches
|
|
155
|
+
memset(at, ACT_NONE, 4 * 256);
|
|
156
|
+
at[S_UNQUOTED][d] = ACT_FIELD;
|
|
157
|
+
at[S_UNQUOTED]['\n'] = ACT_FIELD | ACT_ROW;
|
|
158
|
+
at[S_UNQUOTED]['\r'] = ACT_FIELD;
|
|
159
|
+
|
|
160
|
+
if (!e) {
|
|
161
|
+
// Vectorize the action table initialization
|
|
162
|
+
for (int i = 0; i < 256; i++) {
|
|
163
|
+
at[S_QUOTE_ESC][i] = (i != q) ? ACT_REPROCESS : ACT_NONE;
|
|
146
164
|
}
|
|
147
165
|
}
|
|
148
166
|
|
|
149
|
-
//
|
|
150
|
-
|
|
167
|
+
// Use SIMD for comment actions
|
|
168
|
+
#ifdef __AVX2__
|
|
169
|
+
__m256i skip_act = _mm256_set1_epi8(ACT_SKIP);
|
|
170
|
+
for (int i = 0; i < 256; i += 32) {
|
|
171
|
+
_mm256_store_si256((__m256i*)&at[S_COMMENT][i], skip_act);
|
|
172
|
+
}
|
|
173
|
+
#else
|
|
174
|
+
memset(at[S_COMMENT], ACT_SKIP, 256);
|
|
175
|
+
#endif
|
|
176
|
+
at[S_COMMENT]['\n'] = ACT_ROW;
|
|
151
177
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
action_table[S_UNQUOTED]['\n'] = ACT_FIELD | ACT_ROW;
|
|
155
|
-
action_table[S_UNQUOTED]['\r'] = ACT_FIELD; // Handle CRLF
|
|
178
|
+
parser->tables_initialized = 1;
|
|
179
|
+
}
|
|
156
180
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
181
|
+
// SIMD-optimized whitespace detection lookup table
|
|
182
|
+
// Ultra-fast trimming with AVX512/AVX2
|
|
183
|
+
static inline const uint8_t* trim_start(const uint8_t *start, const uint8_t *end) {
|
|
184
|
+
size_t len = end - start;
|
|
185
|
+
|
|
186
|
+
#ifdef __AVX512F__
|
|
187
|
+
if (len >= 64) {
|
|
188
|
+
const __m512i max_ws = _mm512_set1_epi8(32);
|
|
189
|
+
|
|
190
|
+
while (len >= 64) {
|
|
191
|
+
__m512i chunk = _mm512_loadu_si512(start);
|
|
192
|
+
__mmask64 is_ws = _mm512_cmple_epu8_mask(chunk, max_ws);
|
|
193
|
+
|
|
194
|
+
if (is_ws != 0xFFFFFFFFFFFFFFFFULL) {
|
|
195
|
+
return start + __builtin_ctzll(~is_ws);
|
|
163
196
|
}
|
|
197
|
+
start += 64;
|
|
198
|
+
len -= 64;
|
|
164
199
|
}
|
|
165
200
|
}
|
|
201
|
+
#elif defined(__AVX2__)
|
|
202
|
+
if (len >= 32) {
|
|
203
|
+
const __m256i max_ws = _mm256_set1_epi8(32);
|
|
204
|
+
|
|
205
|
+
while (len >= 32) {
|
|
206
|
+
__m256i chunk = _mm256_loadu_si256((__m256i*)start);
|
|
207
|
+
__m256i cmp = _mm256_cmpgt_epi8(chunk, max_ws);
|
|
208
|
+
uint32_t mask = _mm256_movemask_epi8(cmp);
|
|
166
209
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
210
|
+
if (mask) {
|
|
211
|
+
return start + __builtin_ctz(mask);
|
|
212
|
+
}
|
|
213
|
+
start += 32;
|
|
214
|
+
len -= 32;
|
|
215
|
+
}
|
|
170
216
|
}
|
|
171
|
-
|
|
217
|
+
#endif
|
|
172
218
|
|
|
173
|
-
|
|
174
|
-
|
|
219
|
+
// Unrolled 8-byte processing
|
|
220
|
+
while (len >= 8) {
|
|
221
|
+
uint64_t v = *(uint64_t*)start;
|
|
222
|
+
uint64_t has_non_ws = ((v & 0xE0E0E0E0E0E0E0E0ULL) != 0) |
|
|
223
|
+
((v & 0x1F1F1F1F1F1F1F1FULL) > 0x0D0D0D0D0D0D0D0DULL);
|
|
224
|
+
if (has_non_ws) {
|
|
225
|
+
for (int i = 0; i < 8; i++) {
|
|
226
|
+
if ((uint8_t)(v >> (i*8)) > 32) return start + i;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
start += 8;
|
|
230
|
+
len -= 8;
|
|
231
|
+
}
|
|
175
232
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
233
|
+
// 4-byte processing
|
|
234
|
+
if (len >= 4) {
|
|
235
|
+
uint32_t v = *(uint32_t*)start;
|
|
236
|
+
for (int i = 0; i < 4; i++) {
|
|
237
|
+
uint8_t c = (v >> (i*8)) & 0xFF;
|
|
238
|
+
if (c > 32) return start + i;
|
|
239
|
+
}
|
|
240
|
+
start += 4;
|
|
241
|
+
len -= 4;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Remainder
|
|
245
|
+
switch(len) {
|
|
246
|
+
case 3: if (*start > 32) return start; start++;
|
|
247
|
+
/* fallthrough */
|
|
248
|
+
case 2: if (*start > 32) return start; start++;
|
|
249
|
+
/* fallthrough */
|
|
250
|
+
case 1: if (*start > 32) return start; start++;
|
|
251
|
+
}
|
|
180
252
|
|
|
181
|
-
static inline const uint8_t* trim_end(const uint8_t *start, const uint8_t *end) {
|
|
182
|
-
while (end > start && isspace(*(end - 1))) end--;
|
|
183
253
|
return end;
|
|
184
254
|
}
|
|
185
255
|
|
|
186
|
-
static inline
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
256
|
+
static inline const uint8_t* trim_end(const uint8_t *start, const uint8_t *end) {
|
|
257
|
+
size_t len = end - start;
|
|
258
|
+
|
|
259
|
+
#ifdef __AVX512F__
|
|
260
|
+
while (len >= 64) {
|
|
261
|
+
const uint8_t *check = end - 64;
|
|
262
|
+
__m512i chunk = _mm512_loadu_si512(check);
|
|
263
|
+
const __m512i max_ws = _mm512_set1_epi8(32);
|
|
264
|
+
__mmask64 is_non_ws = _mm512_cmpgt_epu8_mask(chunk, max_ws);
|
|
265
|
+
|
|
266
|
+
if (is_non_ws) {
|
|
267
|
+
int last_non_ws = 63 - __builtin_clzll(is_non_ws);
|
|
268
|
+
return check + last_non_ws + 1;
|
|
269
|
+
}
|
|
270
|
+
end -= 64;
|
|
271
|
+
len -= 64;
|
|
272
|
+
}
|
|
273
|
+
#elif defined(__AVX2__)
|
|
274
|
+
while (len >= 32) {
|
|
275
|
+
const uint8_t *check = end - 32;
|
|
276
|
+
__m256i chunk = _mm256_loadu_si256((__m256i*)check);
|
|
277
|
+
const __m256i max_ws = _mm256_set1_epi8(32);
|
|
278
|
+
__m256i cmp = _mm256_cmpgt_epi8(chunk, max_ws);
|
|
279
|
+
uint32_t mask = _mm256_movemask_epi8(cmp);
|
|
280
|
+
|
|
281
|
+
if (mask) {
|
|
282
|
+
int last_non_ws = 31 - __builtin_clz(mask);
|
|
283
|
+
return check + last_non_ws + 1;
|
|
284
|
+
}
|
|
285
|
+
end -= 32;
|
|
286
|
+
len -= 32;
|
|
191
287
|
}
|
|
288
|
+
#endif
|
|
289
|
+
|
|
290
|
+
// Unrolled 8-byte processing
|
|
291
|
+
while (len >= 8) {
|
|
292
|
+
const uint8_t *check = end - 8;
|
|
293
|
+
uint64_t v = *(uint64_t*)check;
|
|
192
294
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
295
|
+
for (int i = 7; i >= 0; i--) {
|
|
296
|
+
if ((uint8_t)(v >> (i*8)) > 32) return check + i + 1;
|
|
297
|
+
}
|
|
298
|
+
end -= 8;
|
|
299
|
+
len -= 8;
|
|
197
300
|
}
|
|
198
|
-
}
|
|
199
301
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
302
|
+
// 4-byte processing
|
|
303
|
+
if (len >= 4) {
|
|
304
|
+
const uint8_t *check = end - 4;
|
|
305
|
+
uint32_t v = *(uint32_t*)check;
|
|
306
|
+
for (int i = 3; i >= 0; i--) {
|
|
307
|
+
if ((uint8_t)(v >> (i*8)) > 32) return check + i + 1;
|
|
308
|
+
}
|
|
309
|
+
end -= 4;
|
|
310
|
+
len -= 4;
|
|
205
311
|
}
|
|
206
312
|
|
|
207
|
-
//
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
parser->row_start = parser->field_start;
|
|
211
|
-
return;
|
|
313
|
+
// Remainder
|
|
314
|
+
while (len-- > 0) {
|
|
315
|
+
if (*(--end) > 32) return end + 1;
|
|
212
316
|
}
|
|
213
317
|
|
|
214
|
-
|
|
215
|
-
|
|
318
|
+
return start;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// yield_field with prefetching and branchless code
|
|
322
|
+
static inline void yield_field(cisv_parser *parser, const uint8_t *start, const uint8_t *end) {
|
|
323
|
+
// Prefetch parser structure for next access
|
|
324
|
+
__builtin_prefetch(parser, 0, 3);
|
|
325
|
+
|
|
326
|
+
// Branchless trimming using conditional move
|
|
327
|
+
const uint8_t *s = start;
|
|
328
|
+
const uint8_t *e = end;
|
|
329
|
+
|
|
330
|
+
// Use conditional assignment instead of branch
|
|
331
|
+
const uint8_t *trimmed_s = trim_start(s, e);
|
|
332
|
+
const uint8_t *trimmed_e = trim_end(trimmed_s, e);
|
|
333
|
+
|
|
334
|
+
// Branchless selection: if trim is 0, use original, if 1, use trimmed
|
|
335
|
+
uintptr_t mask = -(uintptr_t)parser->trim;
|
|
336
|
+
s = (const uint8_t*)(((uintptr_t)trimmed_s & mask) | ((uintptr_t)s & ~mask));
|
|
337
|
+
e = (const uint8_t*)(((uintptr_t)trimmed_e & mask) | ((uintptr_t)e & ~mask));
|
|
338
|
+
|
|
339
|
+
// Combine all conditions into single branch
|
|
340
|
+
uintptr_t fcb_addr = (uintptr_t)parser->fcb;
|
|
341
|
+
uintptr_t valid_mask = -(fcb_addr != 0);
|
|
342
|
+
valid_mask &= -(s != 0);
|
|
343
|
+
valid_mask &= -(e != 0);
|
|
344
|
+
valid_mask &= -(e >= s);
|
|
345
|
+
|
|
346
|
+
// Single branch for callback execution
|
|
347
|
+
if (valid_mask) {
|
|
348
|
+
// Prefetch user data for callback
|
|
349
|
+
__builtin_prefetch(parser->user, 0, 1);
|
|
350
|
+
parser->fcb(parser->user, (const char *)s, (size_t)(e - s));
|
|
216
351
|
}
|
|
352
|
+
}
|
|
217
353
|
|
|
218
|
-
|
|
354
|
+
// yield_row with reduced branches
|
|
355
|
+
static inline void yield_row(cisv_parser *parser) {
|
|
356
|
+
// Prefetch frequently accessed memory
|
|
357
|
+
__builtin_prefetch(&parser->current_line, 1, 3);
|
|
358
|
+
__builtin_prefetch(&parser->row_start, 1, 3);
|
|
359
|
+
|
|
360
|
+
// Compute all conditions upfront
|
|
361
|
+
int is_empty_line = (parser->field_start == parser->row_start);
|
|
362
|
+
int skip_empty = parser->skip_empty_lines & is_empty_line;
|
|
363
|
+
int before_range = (parser->current_line < parser->from_line);
|
|
364
|
+
int after_range = (parser->to_line > 0) & (parser->current_line > parser->to_line);
|
|
365
|
+
int in_range = !before_range & !after_range;
|
|
366
|
+
|
|
367
|
+
// Branchless increment of current_line (always happens except when after range)
|
|
368
|
+
parser->current_line += !after_range;
|
|
369
|
+
|
|
370
|
+
// Branchless update of row_start (happens except when after range)
|
|
371
|
+
uintptr_t new_row_start = (uintptr_t)parser->field_start;
|
|
372
|
+
uintptr_t old_row_start = (uintptr_t)parser->row_start;
|
|
373
|
+
parser->row_start = (uint8_t*)((old_row_start & -after_range) | (new_row_start & ~(-after_range)));
|
|
374
|
+
|
|
375
|
+
// Branchless reset of row_size
|
|
376
|
+
parser->current_row_size &= after_range;
|
|
377
|
+
|
|
378
|
+
// Single branch for callback (most common case last for better prediction)
|
|
379
|
+
if ((!skip_empty) & in_range & (parser->rcb != NULL)) {
|
|
380
|
+
__builtin_prefetch(parser->user, 0, 1);
|
|
219
381
|
parser->rcb(parser->user);
|
|
220
382
|
}
|
|
221
|
-
|
|
222
|
-
parser->current_line++;
|
|
223
|
-
parser->row_start = parser->field_start;
|
|
224
|
-
parser->current_row_size = 0;
|
|
225
383
|
}
|
|
226
384
|
|
|
227
385
|
static inline void handle_error(cisv_parser *parser, const char *msg) {
|
|
@@ -334,6 +492,8 @@ static void parse_simd_chunk(cisv_parser *parser, const uint8_t *buffer, size_t
|
|
|
334
492
|
// Handle newline
|
|
335
493
|
if (is_newline) {
|
|
336
494
|
yield_row(parser);
|
|
495
|
+
parser->current_row_size = 0;
|
|
496
|
+
parser->row_start = special_pos + 1;
|
|
337
497
|
}
|
|
338
498
|
|
|
339
499
|
// Update state branchlessly
|
|
@@ -626,6 +786,7 @@ static void parse_simd_chunk(cisv_parser *parser, const uint8_t *buffer, size_t
|
|
|
626
786
|
if (action & ACT_ROW) {
|
|
627
787
|
yield_row(parser);
|
|
628
788
|
parser->current_row_size = 0;
|
|
789
|
+
parser->row_start = cur + 1;
|
|
629
790
|
}
|
|
630
791
|
|
|
631
792
|
cur += 1 - ((action & ACT_REPROCESS) >> 2);
|
|
@@ -648,6 +809,7 @@ static int parse_memory(cisv_parser *parser, const uint8_t *buffer, size_t len)
|
|
|
648
809
|
// Yield final row if there's content
|
|
649
810
|
if (parser->field_start > parser->row_start || !parser->skip_empty_lines) {
|
|
650
811
|
yield_row(parser);
|
|
812
|
+
parser->current_row_size = 0;
|
|
651
813
|
}
|
|
652
814
|
}
|
|
653
815
|
return 0;
|
|
@@ -865,24 +1027,36 @@ int cisv_parser_parse_file(cisv_parser *parser, const char *path) {
|
|
|
865
1027
|
}
|
|
866
1028
|
|
|
867
1029
|
int cisv_parser_write(cisv_parser *parser, const uint8_t *chunk, size_t len) {
|
|
868
|
-
if (!parser || !chunk
|
|
1030
|
+
if (!parser || !chunk) return -EINVAL;
|
|
869
1031
|
|
|
870
|
-
//
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
parse_memory(parser,
|
|
874
|
-
|
|
1032
|
+
// For large chunks, bypass ring buffer entirely
|
|
1033
|
+
if (len > DIRECT_PARSE_THRESHOLD) {
|
|
1034
|
+
// Parse directly - this is actually FASTER for large data
|
|
1035
|
+
return parse_memory(parser, chunk, len);
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
// Small chunks use ring buffer for efficiency
|
|
1039
|
+
if (parser->head + len > RINGBUF_SIZE) {
|
|
1040
|
+
// Flush current buffer
|
|
1041
|
+
if (parser->head > 0) {
|
|
1042
|
+
parse_memory(parser, parser->ring, parser->head);
|
|
1043
|
+
parser->head = 0;
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
// If still too large, parse directly
|
|
1047
|
+
if (len > RINGBUF_SIZE) {
|
|
1048
|
+
return parse_memory(parser, chunk, len);
|
|
1049
|
+
}
|
|
875
1050
|
}
|
|
876
1051
|
|
|
877
1052
|
memcpy(parser->ring + parser->head, chunk, len);
|
|
878
1053
|
parser->head += len;
|
|
879
1054
|
|
|
880
|
-
//
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
if (has_newline | threshold) {
|
|
884
|
-
parse_memory(parser, parser->ring, parser->head);
|
|
1055
|
+
// Process on newline or when buffer is getting full
|
|
1056
|
+
if (memchr(chunk, '\n', len) || parser->head > (RINGBUF_SIZE * 3 / 4)) {
|
|
1057
|
+
int result = parse_memory(parser, parser->ring, parser->head);
|
|
885
1058
|
parser->head = 0;
|
|
1059
|
+
return result;
|
|
886
1060
|
}
|
|
887
1061
|
return 0;
|
|
888
1062
|
}
|
package/cisv/cisv_transformer.c
CHANGED
|
@@ -5,12 +5,15 @@
|
|
|
5
5
|
#include <ctype.h>
|
|
6
6
|
#include <stdio.h>
|
|
7
7
|
|
|
8
|
+
#ifdef __AVX512F__
|
|
9
|
+
#include <immintrin.h>
|
|
10
|
+
#endif
|
|
11
|
+
|
|
8
12
|
#ifdef __AVX2__
|
|
9
13
|
#include <immintrin.h>
|
|
10
14
|
#endif
|
|
11
15
|
|
|
12
16
|
#define TRANSFORM_POOL_SIZE (1 << 20) // 1MB default pool
|
|
13
|
-
// #define TRANSFORM_POOL_SIZE (1 << 16) // 64kb (for memory safe reasons)
|
|
14
17
|
#define SIMD_ALIGNMENT 64
|
|
15
18
|
|
|
16
19
|
// Create transform pipeline
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cisv",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.41",
|
|
4
4
|
"description": "The fastest csv parser of the multiverse",
|
|
5
5
|
"author": "sanix<s4nixd@gmail.com>",
|
|
6
6
|
"main": "./build/Release/cisv.node",
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
"install": "node-gyp rebuild",
|
|
10
10
|
"build": "node-gyp rebuild",
|
|
11
11
|
"test": "mocha ./tests/*.test.js && bash ./test_transform.sh",
|
|
12
|
-
"test:build": "npm run test",
|
|
12
|
+
"test:build": "npm run build && npm run test",
|
|
13
13
|
"benchmark": "node benchmark/benchmark.js",
|
|
14
14
|
"lint": "clang-format -i cisv/*.{cc,h}",
|
|
15
15
|
"prepublishOnly": "npm run benchmark",
|