jtcsv 2.2.7 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -1
- package/bin/jtcsv.js +891 -821
- package/bin/jtcsv.ts +2534 -0
- package/csv-to-json.js +168 -145
- package/dist/jtcsv-core.cjs.js +1407 -0
- package/dist/jtcsv-core.cjs.js.map +1 -0
- package/dist/jtcsv-core.esm.js +1379 -0
- package/dist/jtcsv-core.esm.js.map +1 -0
- package/dist/jtcsv-core.umd.js +1413 -0
- package/dist/jtcsv-core.umd.js.map +1 -0
- package/dist/jtcsv-full.cjs.js +1912 -0
- package/dist/jtcsv-full.cjs.js.map +1 -0
- package/dist/jtcsv-full.esm.js +1880 -0
- package/dist/jtcsv-full.esm.js.map +1 -0
- package/dist/jtcsv-full.umd.js +1918 -0
- package/dist/jtcsv-full.umd.js.map +1 -0
- package/dist/jtcsv-workers.esm.js +759 -0
- package/dist/jtcsv-workers.esm.js.map +1 -0
- package/dist/jtcsv-workers.umd.js +773 -0
- package/dist/jtcsv-workers.umd.js.map +1 -0
- package/dist/jtcsv.cjs.js +61 -19
- package/dist/jtcsv.cjs.js.map +1 -1
- package/dist/jtcsv.esm.js +61 -19
- package/dist/jtcsv.esm.js.map +1 -1
- package/dist/jtcsv.umd.js +61 -19
- package/dist/jtcsv.umd.js.map +1 -1
- package/errors.js +188 -2
- package/examples/advanced/conditional-transformations.js +446 -0
- package/examples/advanced/conditional-transformations.ts +446 -0
- package/examples/advanced/csv-parser.worker.js +89 -0
- package/examples/advanced/csv-parser.worker.ts +89 -0
- package/examples/advanced/nested-objects-example.js +306 -0
- package/examples/advanced/nested-objects-example.ts +306 -0
- package/examples/advanced/performance-optimization.js +504 -0
- package/examples/advanced/performance-optimization.ts +504 -0
- package/examples/advanced/run-demo-server.js +116 -0
- package/examples/advanced/run-demo-server.ts +116 -0
- package/examples/advanced/web-worker-usage.html +874 -0
- package/examples/async-multithreaded-example.ts +335 -0
- package/examples/cli-advanced-usage.md +288 -0
- package/examples/cli-batch-processing.ts +38 -0
- package/examples/cli-tool.js +0 -3
- package/examples/cli-tool.ts +183 -0
- package/examples/error-handling.js +21 -7
- package/examples/error-handling.ts +356 -0
- package/examples/express-api.js +0 -3
- package/examples/express-api.ts +164 -0
- package/examples/large-dataset-example.js +0 -3
- package/examples/large-dataset-example.ts +204 -0
- package/examples/ndjson-processing.js +1 -1
- package/examples/ndjson-processing.ts +456 -0
- package/examples/plugin-excel-exporter.js +3 -4
- package/examples/plugin-excel-exporter.ts +406 -0
- package/examples/react-integration.tsx +637 -0
- package/examples/schema-validation.ts +640 -0
- package/examples/simple-usage.js +254 -254
- package/examples/simple-usage.ts +194 -0
- package/examples/streaming-example.js +4 -5
- package/examples/streaming-example.ts +419 -0
- package/examples/web-workers-advanced.ts +28 -0
- package/index.d.ts +1 -3
- package/index.js +15 -1
- package/json-save.js +9 -3
- package/json-to-csv.js +168 -21
- package/package.json +69 -10
- package/plugins/express-middleware/README.md +21 -2
- package/plugins/express-middleware/example.js +3 -4
- package/plugins/express-middleware/example.ts +135 -0
- package/plugins/express-middleware/index.d.ts +1 -1
- package/plugins/express-middleware/index.js +270 -118
- package/plugins/express-middleware/index.ts +557 -0
- package/plugins/fastify-plugin/index.js +2 -4
- package/plugins/fastify-plugin/index.ts +443 -0
- package/plugins/hono/index.ts +226 -0
- package/plugins/nestjs/index.ts +201 -0
- package/plugins/nextjs-api/examples/ConverterComponent.tsx +386 -0
- package/plugins/nextjs-api/examples/api-convert.js +0 -2
- package/plugins/nextjs-api/examples/api-convert.ts +67 -0
- package/plugins/nextjs-api/index.tsx +339 -0
- package/plugins/nextjs-api/route.js +2 -3
- package/plugins/nextjs-api/route.ts +370 -0
- package/plugins/nuxt/index.ts +94 -0
- package/plugins/nuxt/runtime/composables/useJtcsv.ts +100 -0
- package/plugins/nuxt/runtime/plugin.ts +71 -0
- package/plugins/remix/index.js +1 -1
- package/plugins/remix/index.ts +260 -0
- package/plugins/sveltekit/index.js +1 -1
- package/plugins/sveltekit/index.ts +301 -0
- package/plugins/trpc/index.ts +267 -0
- package/src/browser/browser-functions.ts +402 -0
- package/src/browser/core.js +92 -0
- package/src/browser/core.ts +152 -0
- package/src/browser/csv-to-json-browser.d.ts +3 -0
- package/src/browser/csv-to-json-browser.js +36 -14
- package/src/browser/csv-to-json-browser.ts +264 -0
- package/src/browser/errors-browser.ts +303 -0
- package/src/browser/extensions/plugins.js +92 -0
- package/src/browser/extensions/plugins.ts +93 -0
- package/src/browser/extensions/workers.js +39 -0
- package/src/browser/extensions/workers.ts +39 -0
- package/src/browser/globals.d.ts +5 -0
- package/src/browser/index.ts +192 -0
- package/src/browser/json-to-csv-browser.d.ts +3 -0
- package/src/browser/json-to-csv-browser.js +13 -3
- package/src/browser/json-to-csv-browser.ts +262 -0
- package/src/browser/streams.js +12 -2
- package/src/browser/streams.ts +336 -0
- package/src/browser/workers/csv-parser.worker.ts +377 -0
- package/src/browser/workers/worker-pool.ts +548 -0
- package/src/core/delimiter-cache.js +22 -8
- package/src/core/delimiter-cache.ts +310 -0
- package/src/core/node-optimizations.ts +449 -0
- package/src/core/plugin-system.js +29 -11
- package/src/core/plugin-system.ts +400 -0
- package/src/core/transform-hooks.ts +558 -0
- package/src/engines/fast-path-engine-new.ts +347 -0
- package/src/engines/fast-path-engine.ts +854 -0
- package/src/errors.ts +72 -0
- package/src/formats/ndjson-parser.ts +469 -0
- package/src/formats/tsv-parser.ts +334 -0
- package/src/index-with-plugins.js +16 -9
- package/src/index-with-plugins.ts +395 -0
- package/src/types/index.ts +255 -0
- package/src/utils/bom-utils.js +259 -0
- package/src/utils/bom-utils.ts +373 -0
- package/src/utils/encoding-support.js +124 -0
- package/src/utils/encoding-support.ts +155 -0
- package/src/utils/schema-validator.js +19 -19
- package/src/utils/schema-validator.ts +819 -0
- package/src/utils/transform-loader.js +1 -1
- package/src/utils/transform-loader.ts +389 -0
- package/src/utils/zod-adapter.js +170 -0
- package/src/utils/zod-adapter.ts +280 -0
- package/src/web-server/index.js +10 -10
- package/src/web-server/index.ts +683 -0
- package/src/workers/csv-multithreaded.ts +310 -0
- package/src/workers/csv-parser.worker.ts +227 -0
- package/src/workers/worker-pool.ts +409 -0
- package/stream-csv-to-json.js +26 -8
- package/stream-json-to-csv.js +1 -0
package/json-save.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
1
2
|
/**
|
|
2
3
|
* JSON Save Module - Node.js Module
|
|
3
4
|
*
|
|
@@ -21,17 +22,22 @@ const {
|
|
|
21
22
|
*/
|
|
22
23
|
function validateJsonFilePath(filePath) {
|
|
23
24
|
const path = require('path');
|
|
24
|
-
|
|
25
|
+
|
|
25
26
|
// Basic validation
|
|
26
27
|
if (typeof filePath !== 'string' || filePath.trim() === '') {
|
|
27
28
|
throw new ValidationError('File path must be a non-empty string');
|
|
28
29
|
}
|
|
29
|
-
|
|
30
|
+
|
|
30
31
|
// Ensure file has .json extension
|
|
31
32
|
if (!filePath.toLowerCase().endsWith('.json')) {
|
|
32
33
|
throw new ValidationError('File must have .json extension');
|
|
33
34
|
}
|
|
34
|
-
|
|
35
|
+
|
|
36
|
+
// Block UNC paths BEFORE path.resolve() to avoid network lookup timeouts
|
|
37
|
+
if (filePath.startsWith('\\\\') || filePath.startsWith('//')) {
|
|
38
|
+
throw new SecurityError('UNC paths are not allowed');
|
|
39
|
+
}
|
|
40
|
+
|
|
35
41
|
// Get absolute path and check for traversal
|
|
36
42
|
const absolutePath = path.resolve(filePath);
|
|
37
43
|
const normalizedPath = path.normalize(filePath);
|
package/json-to-csv.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
1
2
|
/**
|
|
2
3
|
* JSON to CSV Converter - Node.js Module
|
|
3
4
|
*
|
|
@@ -74,33 +75,50 @@ function validateInput(data, options) {
|
|
|
74
75
|
|
|
75
76
|
/**
|
|
76
77
|
* Converts JSON data to CSV format
|
|
77
|
-
*
|
|
78
|
+
*
|
|
78
79
|
* @param {Array<Object>} data - Array of objects to convert to CSV
|
|
79
80
|
* @param {Object} [options] - Configuration options
|
|
80
81
|
* @param {string} [options.delimiter=';'] - CSV delimiter character
|
|
81
82
|
* @param {boolean} [options.includeHeaders=true] - Whether to include headers row
|
|
82
83
|
* @param {Object} [options.renameMap={}] - Map for renaming column headers (oldKey: newKey)
|
|
83
|
-
* @param {Object} [options.template={}] - Template object to ensure consistent column order
|
|
84
|
+
* @param {Object} [options.template={}] - Template object to ensure consistent column order
|
|
84
85
|
* @param {number} [options.maxRecords] - Maximum number of records to process (optional, no limit by default)
|
|
85
86
|
* @param {boolean} [options.preventCsvInjection=true] - Prevent CSV injection attacks by escaping formulas
|
|
86
87
|
* @param {boolean} [options.rfc4180Compliant=true] - Ensure RFC 4180 compliance (proper quoting, line endings)
|
|
87
88
|
* @param {Object} [options.schema] - JSON schema for data validation and formatting
|
|
89
|
+
* @param {boolean} [options.flatten=false] - Whether to flatten nested objects into dot notation keys
|
|
90
|
+
* @param {string} [options.flattenSeparator='.'] - Separator for flattened keys (e.g., 'user.name' with '.')
|
|
91
|
+
* @param {number} [options.flattenMaxDepth=3] - Maximum depth for flattening nested objects
|
|
92
|
+
* @param {string} [options.arrayHandling='stringify'] - How to handle arrays ('stringify', 'join', 'expand')
|
|
88
93
|
* @returns {string} CSV formatted string
|
|
89
|
-
*
|
|
94
|
+
*
|
|
90
95
|
* @example
|
|
91
96
|
* const jsonToCsv = require('./json-to-csv');
|
|
92
|
-
*
|
|
97
|
+
*
|
|
93
98
|
* const data = [
|
|
94
99
|
* { id: 1, name: 'John', email: 'john@example.com' },
|
|
95
100
|
* { id: 2, name: 'Jane', email: 'jane@example.com' }
|
|
96
101
|
* ];
|
|
97
|
-
*
|
|
102
|
+
*
|
|
98
103
|
* const csv = jsonToCsv(data, {
|
|
99
104
|
* delimiter: ',',
|
|
100
105
|
* renameMap: { id: 'ID', name: 'Full Name' },
|
|
101
106
|
* preventCsvInjection: true,
|
|
102
107
|
* rfc4180Compliant: true
|
|
103
108
|
* });
|
|
109
|
+
*
|
|
110
|
+
* @example
|
|
111
|
+
* // With nested object flattening
|
|
112
|
+
* const nestedData = [
|
|
113
|
+
* { id: 1, user: { name: 'John', profile: { age: 30 } } }
|
|
114
|
+
* ];
|
|
115
|
+
* const csv = jsonToCsv(nestedData, {
|
|
116
|
+
* flatten: true,
|
|
117
|
+
* flattenSeparator: '_',
|
|
118
|
+
* flattenMaxDepth: 4
|
|
119
|
+
* });
|
|
120
|
+
* // Result: id,user_name,user_profile_age
|
|
121
|
+
* // 1,John,30
|
|
104
122
|
*/
|
|
105
123
|
function jsonToCsv(data, options = {}) {
|
|
106
124
|
return safeExecute(() => {
|
|
@@ -117,7 +135,11 @@ function jsonToCsv(data, options = {}) {
|
|
|
117
135
|
maxRecords,
|
|
118
136
|
preventCsvInjection = true,
|
|
119
137
|
rfc4180Compliant = true,
|
|
120
|
-
schema = null
|
|
138
|
+
schema = null,
|
|
139
|
+
flatten = false,
|
|
140
|
+
flattenSeparator = '.',
|
|
141
|
+
flattenMaxDepth = 3,
|
|
142
|
+
_arrayHandling = 'stringify'
|
|
121
143
|
} = opts;
|
|
122
144
|
|
|
123
145
|
// Initialize schema validators if schema is provided
|
|
@@ -150,11 +172,19 @@ function jsonToCsv(data, options = {}) {
|
|
|
150
172
|
);
|
|
151
173
|
}
|
|
152
174
|
|
|
175
|
+
// Preprocess data with flattening options if needed
|
|
176
|
+
const processedData = preprocessData(data, {
|
|
177
|
+
flatten,
|
|
178
|
+
flattenSeparator,
|
|
179
|
+
flattenMaxDepth,
|
|
180
|
+
arrayHandling: _arrayHandling
|
|
181
|
+
});
|
|
182
|
+
|
|
153
183
|
// Get all unique keys from all objects with minimal allocations.
|
|
154
184
|
const allKeys = new Set();
|
|
155
185
|
const originalKeys = [];
|
|
156
|
-
for (let i = 0; i <
|
|
157
|
-
const item =
|
|
186
|
+
for (let i = 0; i < processedData.length; i++) {
|
|
187
|
+
const item = processedData[i];
|
|
158
188
|
if (!item || typeof item !== 'object') {
|
|
159
189
|
continue;
|
|
160
190
|
}
|
|
@@ -231,15 +261,23 @@ function jsonToCsv(data, options = {}) {
|
|
|
231
261
|
let stringValue = value;
|
|
232
262
|
if (typeof stringValue !== 'string') {
|
|
233
263
|
stringValue = String(stringValue);
|
|
234
|
-
}
|
|
235
|
-
|
|
264
|
+
}
|
|
236
265
|
// CSV Injection protection - escape formulas if enabled
|
|
237
266
|
let escapedValue = stringValue;
|
|
238
267
|
if (preventCsvInjection) {
|
|
239
268
|
const firstCharCode = stringValue.charCodeAt(0);
|
|
240
|
-
|
|
269
|
+
// Dangerous prefixes: =, +, -, @, tab (\t), carriage return (\r)
|
|
270
|
+
if (firstCharCode === 61 || firstCharCode === 43 || firstCharCode === 45 || firstCharCode === 64 ||
|
|
271
|
+
firstCharCode === 9 || firstCharCode === 13) {
|
|
241
272
|
escapedValue = "'" + stringValue;
|
|
242
273
|
}
|
|
274
|
+
// Unicode Bidi override characters
|
|
275
|
+
const bidiChars = ['\u202A', '\u202B', '\u202C', '\u202D', '\u202E'];
|
|
276
|
+
for (const bidi of bidiChars) {
|
|
277
|
+
if (stringValue.includes(bidi)) {
|
|
278
|
+
escapedValue = escapedValue.replace(new RegExp(bidi, 'g'), '');
|
|
279
|
+
}
|
|
280
|
+
}
|
|
243
281
|
}
|
|
244
282
|
|
|
245
283
|
let needsQuoting = false;
|
|
@@ -273,8 +311,8 @@ function jsonToCsv(data, options = {}) {
|
|
|
273
311
|
|
|
274
312
|
// Add data rows.
|
|
275
313
|
const rowValues = new Array(columnCount);
|
|
276
|
-
for (let rowIndex = 0; rowIndex <
|
|
277
|
-
const item =
|
|
314
|
+
for (let rowIndex = 0; rowIndex < processedData.length; rowIndex++) {
|
|
315
|
+
const item = processedData[rowIndex];
|
|
278
316
|
if (!item || typeof item !== 'object') {
|
|
279
317
|
continue;
|
|
280
318
|
}
|
|
@@ -317,15 +355,96 @@ function jsonToCsv(data, options = {}) {
|
|
|
317
355
|
}, 'PARSE_FAILED', { function: 'jsonToCsv' });
|
|
318
356
|
}
|
|
319
357
|
|
|
358
|
+
/**
|
|
359
|
+
* Flattens nested objects into single-level objects with dot notation keys
|
|
360
|
+
*
|
|
361
|
+
* @private
|
|
362
|
+
* @param {Object} obj - Object to flatten
|
|
363
|
+
* @param {string} prefix - Current key prefix
|
|
364
|
+
* @param {string} separator - Separator for nested keys (default: '.')
|
|
365
|
+
* @param {number} maxDepth - Maximum flattening depth
|
|
366
|
+
* @param {number} currentDepth - Current recursion depth
|
|
367
|
+
* @returns {Object} Flattened object
|
|
368
|
+
*
|
|
369
|
+
* @example
|
|
370
|
+
* flattenObject({ a: 1, b: { c: 2, d: { e: 3 } } });
|
|
371
|
+
* // Returns { 'a': 1, 'b.c': 2, 'b.d.e': 3 }
|
|
372
|
+
*/
|
|
373
|
+
function flattenObject(obj, prefix = '', separator = '.', maxDepth = 3, currentDepth = 0) {
|
|
374
|
+
// Handle null/undefined
|
|
375
|
+
if (obj === null || obj === undefined) {
|
|
376
|
+
return { [prefix || 'value']: '' };
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// Handle primitive values
|
|
380
|
+
if (typeof obj !== 'object') {
|
|
381
|
+
return { [prefix || 'value']: obj };
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// Handle arrays
|
|
385
|
+
if (Array.isArray(obj)) {
|
|
386
|
+
if (obj.length === 0) {
|
|
387
|
+
return { [prefix || 'value']: '' };
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
// For simple arrays, join with comma
|
|
391
|
+
if (obj.every(item => typeof item === 'string' || typeof item === 'number' || typeof item === 'boolean')) {
|
|
392
|
+
return { [prefix || 'value']: obj.join(', ') };
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// For complex arrays, stringify
|
|
396
|
+
try {
|
|
397
|
+
return { [prefix || 'value']: JSON.stringify(obj) };
|
|
398
|
+
} catch {
|
|
399
|
+
return { [prefix || 'value']: '[Complex Array]' };
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// Check depth limit
|
|
404
|
+
if (currentDepth >= maxDepth) {
|
|
405
|
+
try {
|
|
406
|
+
return { [prefix || 'value']: JSON.stringify(obj) };
|
|
407
|
+
} catch {
|
|
408
|
+
return { [prefix || 'value']: '[Too Deep]' };
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
const result = {};
|
|
413
|
+
|
|
414
|
+
for (const key in obj) {
|
|
415
|
+
if (Object.prototype.hasOwnProperty.call(obj, key)) {
|
|
416
|
+
const value = obj[key];
|
|
417
|
+
const newKey = prefix ? `${prefix}${separator}${key}` : key;
|
|
418
|
+
|
|
419
|
+
if (value && typeof value === 'object') {
|
|
420
|
+
// Recursively flatten nested objects
|
|
421
|
+
const flattened = flattenObject(
|
|
422
|
+
value,
|
|
423
|
+
newKey,
|
|
424
|
+
separator,
|
|
425
|
+
maxDepth,
|
|
426
|
+
currentDepth + 1
|
|
427
|
+
);
|
|
428
|
+
Object.assign(result, flattened);
|
|
429
|
+
} else {
|
|
430
|
+
// Primitive values
|
|
431
|
+
result[newKey] = value === null || value === undefined ? '' : value;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
return result;
|
|
437
|
+
}
|
|
438
|
+
|
|
320
439
|
/**
|
|
321
440
|
* Deeply unwraps nested objects and arrays to extract primitive values
|
|
322
|
-
*
|
|
441
|
+
*
|
|
323
442
|
* @param {*} value - Value to unwrap
|
|
324
443
|
* @param {number} [depth=0] - Current recursion depth
|
|
325
444
|
* @param {number} [maxDepth=5] - Maximum recursion depth
|
|
326
445
|
* @param {Set} [visited=new Set()] - Set of visited objects to detect circular references
|
|
327
446
|
* @returns {string} Unwrapped string value
|
|
328
|
-
*
|
|
447
|
+
*
|
|
329
448
|
* @private
|
|
330
449
|
*/
|
|
331
450
|
function deepUnwrap(value, depth = 0, maxDepth = 5, visited = new Set()) {
|
|
@@ -386,15 +505,33 @@ function deepUnwrap(value, depth = 0, maxDepth = 5, visited = new Set()) {
|
|
|
386
505
|
|
|
387
506
|
/**
|
|
388
507
|
* Preprocesses JSON data by deeply unwrapping nested structures
|
|
389
|
-
*
|
|
508
|
+
*
|
|
390
509
|
* @param {Array<Object>} data - Array of objects to preprocess
|
|
510
|
+
* @param {Object} [options] - Preprocessing options
|
|
511
|
+
* @param {boolean} [options.flatten=false] - Whether to flatten nested objects
|
|
512
|
+
* @param {string} [options.flattenSeparator='.'] - Separator for flattened keys
|
|
513
|
+
* @param {number} [options.flattenMaxDepth=3] - Maximum flattening depth
|
|
514
|
+
* @param {string} [options.arrayHandling='stringify'] - How to handle arrays ('stringify', 'join', 'expand')
|
|
391
515
|
* @returns {Array<Object>} Preprocessed data with unwrapped values
|
|
392
|
-
*
|
|
516
|
+
*
|
|
393
517
|
* @example
|
|
518
|
+
* // Standard preprocessing
|
|
394
519
|
* const processed = preprocessData(complexJsonData);
|
|
395
520
|
* const csv = jsonToCsv(processed);
|
|
521
|
+
*
|
|
522
|
+
* @example
|
|
523
|
+
* // With flattening
|
|
524
|
+
* const flattened = preprocessData(complexJsonData, { flatten: true });
|
|
525
|
+
* const csv = jsonToCsv(flattened);
|
|
396
526
|
*/
|
|
397
|
-
function preprocessData(data) {
|
|
527
|
+
function preprocessData(data, options = {}) {
|
|
528
|
+
const {
|
|
529
|
+
flatten = false,
|
|
530
|
+
flattenSeparator = '.',
|
|
531
|
+
flattenMaxDepth = 3,
|
|
532
|
+
_arrayHandling = 'stringify'
|
|
533
|
+
} = options;
|
|
534
|
+
|
|
398
535
|
if (!Array.isArray(data)) {
|
|
399
536
|
return [];
|
|
400
537
|
}
|
|
@@ -404,6 +541,11 @@ function preprocessData(data) {
|
|
|
404
541
|
return {};
|
|
405
542
|
}
|
|
406
543
|
|
|
544
|
+
if (flatten) {
|
|
545
|
+
return flattenObject(item, '', flattenSeparator, flattenMaxDepth);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// Standard processing (backward compatibility)
|
|
407
549
|
const processed = {};
|
|
408
550
|
|
|
409
551
|
for (const key in item) {
|
|
@@ -427,17 +569,22 @@ function preprocessData(data) {
|
|
|
427
569
|
*/
|
|
428
570
|
function validateFilePath(filePath) {
|
|
429
571
|
const path = require('path');
|
|
430
|
-
|
|
572
|
+
|
|
431
573
|
// Basic validation
|
|
432
574
|
if (typeof filePath !== 'string' || filePath.trim() === '') {
|
|
433
575
|
throw new ValidationError('File path must be a non-empty string');
|
|
434
576
|
}
|
|
435
|
-
|
|
577
|
+
|
|
436
578
|
// Ensure file has .csv extension
|
|
437
579
|
if (!filePath.toLowerCase().endsWith('.csv')) {
|
|
438
580
|
throw new ValidationError('File must have .csv extension');
|
|
439
581
|
}
|
|
440
|
-
|
|
582
|
+
|
|
583
|
+
// Block UNC paths BEFORE path.resolve() to avoid network lookup timeouts
|
|
584
|
+
if (filePath.startsWith('\\\\') || filePath.startsWith('//')) {
|
|
585
|
+
throw new SecurityError('UNC paths are not allowed');
|
|
586
|
+
}
|
|
587
|
+
|
|
441
588
|
// Get absolute path and check for traversal
|
|
442
589
|
const absolutePath = path.resolve(filePath);
|
|
443
590
|
const normalizedPath = path.normalize(filePath);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "jtcsv",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.0",
|
|
4
4
|
"description": "Complete JSON<->CSV and CSV<->JSON converter for Node.js and Browser with streaming, security, Web Workers, TypeScript support, and optional ecosystem (zero-deps core)",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"browser": "dist/jtcsv.umd.js",
|
|
@@ -76,11 +76,38 @@
|
|
|
76
76
|
"require": "./plugins/trpc/index.js",
|
|
77
77
|
"import": "./plugins/trpc/index.js",
|
|
78
78
|
"default": "./plugins/trpc/index.js"
|
|
79
|
+
},
|
|
80
|
+
"./core": {
|
|
81
|
+
"require": "./dist/jtcsv-core.cjs.js",
|
|
82
|
+
"import": "./dist/jtcsv-core.esm.js",
|
|
83
|
+
"browser": "./dist/jtcsv-core.umd.js",
|
|
84
|
+
"default": "./dist/jtcsv-core.cjs.js"
|
|
85
|
+
},
|
|
86
|
+
"./full": {
|
|
87
|
+
"require": "./dist/jtcsv-full.cjs.js",
|
|
88
|
+
"import": "./dist/jtcsv-full.esm.js",
|
|
89
|
+
"browser": "./dist/jtcsv-full.umd.js",
|
|
90
|
+
"default": "./dist/jtcsv-full.cjs.js"
|
|
91
|
+
},
|
|
92
|
+
"./workers": {
|
|
93
|
+
"require": "./dist/jtcsv-workers.cjs.js",
|
|
94
|
+
"import": "./dist/jtcsv-workers.esm.js",
|
|
95
|
+
"browser": "./dist/jtcsv-workers.umd.js",
|
|
96
|
+
"default": "./dist/jtcsv-workers.esm.js"
|
|
97
|
+
},
|
|
98
|
+
"./optimized": {
|
|
99
|
+
"require": "./dist/jtcsv-core.cjs.js",
|
|
100
|
+
"import": "./dist/jtcsv-core.esm.js",
|
|
101
|
+
"browser": "./dist/jtcsv-core.umd.js",
|
|
102
|
+
"default": "./dist/jtcsv-core.cjs.js"
|
|
79
103
|
}
|
|
80
104
|
},
|
|
81
105
|
"scripts": {
|
|
82
106
|
"test": "jest",
|
|
83
|
-
"test:
|
|
107
|
+
"test:unit": "jest --testPathIgnorePatterns=\"(__tests__/benchmark-suite.test.js|__tests__/load-tests.test.js|__tests__/soak-memory.test.js|__tests__/security-fuzzing.test.js|__tests__/memory-profiling.test.js)\"",
|
|
108
|
+
"test:coverage": "node scripts/run-coverage.js --target=js --scope=full --strict=0",
|
|
109
|
+
"test:coverage:entry": "node scripts/run-coverage.js --target=js --scope=entry --strict=1",
|
|
110
|
+
"test:coverage:ts": "node scripts/run-coverage.js --target=ts --scope=full --strict=0",
|
|
84
111
|
"test:watch": "jest --watch",
|
|
85
112
|
"test:browser": "jest --testEnvironment=jsdom",
|
|
86
113
|
"test:plugins": "jest __tests__/plugin-system.test.js",
|
|
@@ -102,10 +129,11 @@
|
|
|
102
129
|
"prepublishOnly": "npm test && npm run build && eslint index.js json-to-csv.js csv-to-json.js errors.js stream-json-to-csv.js stream-csv-to-json.js json-save.js src/browser src/engines src/formats src/core",
|
|
103
130
|
"tui": "node packages/jtcsv-tui/bin/jtcsv-tui.js",
|
|
104
131
|
"cli": "node bin/jtcsv.js",
|
|
105
|
-
"build": "rollup -c rollup.config.mjs",
|
|
106
|
-
"build:watch": "rollup -c rollup.config.mjs -w",
|
|
107
|
-
"build:prod": "NODE_ENV=production rollup -c rollup.config.mjs",
|
|
108
|
-
"dev": "NODE_ENV=development rollup -c rollup.config.mjs -w",
|
|
132
|
+
"build": "rollup -c rollup.config.mjs",
|
|
133
|
+
"build:watch": "rollup -c rollup.config.mjs -w",
|
|
134
|
+
"build:prod": "NODE_ENV=production rollup -c rollup.config.mjs",
|
|
135
|
+
"dev": "NODE_ENV=development rollup -c rollup.config.mjs -w",
|
|
136
|
+
"verify:browser": "node scripts/verify-browser-build.js",
|
|
109
137
|
"demo": "node run-demo.js",
|
|
110
138
|
"demo:web": "cd demo && npm run dev",
|
|
111
139
|
"demo:serve": "cd demo && npm run serve",
|
|
@@ -121,7 +149,22 @@
|
|
|
121
149
|
"plugins:build": "npm run build && cd plugins/express-middleware && npm run build && cd ../fastify-plugin && npm run build && cd ../nextjs-api && npm run build",
|
|
122
150
|
"docs": "typedoc",
|
|
123
151
|
"docs:watch": "typedoc --watch",
|
|
124
|
-
"docs:serve": "typedoc && npx serve docs/api"
|
|
152
|
+
"docs:serve": "typedoc && npx serve docs/api",
|
|
153
|
+
"tsc": "tsc",
|
|
154
|
+
"tsc:dev": "tsc --project tsconfig.dev.json",
|
|
155
|
+
"tsc:build": "tsc --project tsconfig.build.json",
|
|
156
|
+
"tsc:watch": "tsc --project tsconfig.dev.json --watch",
|
|
157
|
+
"tsc:check": "tsc --noEmit",
|
|
158
|
+
"tsc:check-strict": "tsc --noEmit --strict",
|
|
159
|
+
"tsc:types": "tsc --project tsconfig.types.json",
|
|
160
|
+
"test:ts": "jest --config jest.config.ts.js",
|
|
161
|
+
"test:ts:coverage": "jest --config jest.config.ts.js --coverage",
|
|
162
|
+
"test:ts:watch": "jest --config jest.config.ts.js --watch",
|
|
163
|
+
"test:types": "node scripts/run-type-tests.js",
|
|
164
|
+
"build:ts": "npm run tsc:build && npm run build",
|
|
165
|
+
"build:ts:watch": "npm run tsc:watch & npm run build:watch",
|
|
166
|
+
"lint:ts": "eslint --ext .ts,.tsx .",
|
|
167
|
+
"lint:ts:fix": "eslint --ext .ts,.tsx --fix ."
|
|
125
168
|
},
|
|
126
169
|
"keywords": [
|
|
127
170
|
"json",
|
|
@@ -200,30 +243,46 @@
|
|
|
200
243
|
"examples/",
|
|
201
244
|
"plugins/"
|
|
202
245
|
],
|
|
203
|
-
"dependencies": {},
|
|
204
246
|
"optionalDependencies": {
|
|
205
247
|
"glob": "^11.0.0"
|
|
206
248
|
},
|
|
207
249
|
"devDependencies": {
|
|
208
250
|
"@babel/core": "^7.26.0",
|
|
251
|
+
"@babel/plugin-proposal-nullish-coalescing-operator": "7.18.6",
|
|
252
|
+
"@babel/plugin-proposal-optional-chaining": "7.21.0",
|
|
253
|
+
"@babel/plugin-syntax-dynamic-import": "7.8.3",
|
|
254
|
+
"@babel/plugin-syntax-jsx": "7.28.6",
|
|
209
255
|
"@babel/preset-env": "^7.26.0",
|
|
256
|
+
"@babel/preset-react": "7.28.5",
|
|
210
257
|
"@eslint/js": "^9.18.0",
|
|
211
258
|
"@rollup/plugin-babel": "^6.0.4",
|
|
212
259
|
"@rollup/plugin-commonjs": "^28.0.0",
|
|
213
260
|
"@rollup/plugin-node-resolve": "^16.0.0",
|
|
214
261
|
"@rollup/plugin-terser": "^0.4.4",
|
|
262
|
+
"@rollup/plugin-typescript": "^12.1.2",
|
|
215
263
|
"@size-limit/preset-small-lib": "^11.1.0",
|
|
264
|
+
"@types/express": "5.0.6",
|
|
265
|
+
"@types/jest": "^29.5.14",
|
|
266
|
+
"@types/node": "^22.10.6",
|
|
216
267
|
"blessed": "^0.1.81",
|
|
217
|
-
"blessed-contrib": "
|
|
268
|
+
"blessed-contrib": "4.11.0",
|
|
218
269
|
"eslint": "^9.18.0",
|
|
270
|
+
"express": "5.2.1",
|
|
219
271
|
"globals": "^15.14.0",
|
|
220
272
|
"jest": "^29.7.0",
|
|
221
273
|
"jest-environment-jsdom": "^29.7.0",
|
|
222
274
|
"rollup": "^4.30.0",
|
|
223
275
|
"size-limit": "^11.1.0",
|
|
224
|
-
"
|
|
276
|
+
"supertest": "7.2.2",
|
|
277
|
+
"ts-jest": "^29.2.5",
|
|
278
|
+
"ts-node": "^10.9.2",
|
|
279
|
+
"tsd": "0.33.0",
|
|
280
|
+
"tslib": "^2.8.1",
|
|
281
|
+
"typedoc": "^0.27.0",
|
|
282
|
+
"typescript": "^5.7.3"
|
|
225
283
|
},
|
|
226
284
|
"type": "commonjs",
|
|
285
|
+
"sideEffects": false,
|
|
227
286
|
"size-limit": [
|
|
228
287
|
{
|
|
229
288
|
"path": "dist/jtcsv.umd.js",
|
|
@@ -32,6 +32,11 @@ app.use(middleware({
|
|
|
32
32
|
enableFastPath: true,
|
|
33
33
|
preventCsvInjection: true,
|
|
34
34
|
rfc4180Compliant: true,
|
|
35
|
+
// Security & DoS protection
|
|
36
|
+
maxFileSize: '500MB', // Maximum allowed file size (default: 500MB)
|
|
37
|
+
maxFieldSize: 1024 * 1024, // Maximum field size in bytes (default: 1MB)
|
|
38
|
+
timeout: 300000, // Processing timeout in milliseconds (default: 5 minutes)
|
|
39
|
+
// Conversion options
|
|
35
40
|
conversionOptions: {
|
|
36
41
|
parseNumbers: true,
|
|
37
42
|
parseBooleans: true
|
|
@@ -39,7 +44,21 @@ app.use(middleware({
|
|
|
39
44
|
}));
|
|
40
45
|
```
|
|
41
46
|
|
|
42
|
-
|
|
47
|
+
**Security notes**:
|
|
48
|
+
- `maxFileSize` rejects requests with `Content-Length` exceeding the limit (HTTP 413).
|
|
49
|
+
- `maxFieldSize` limits individual field size during parsing (helps prevent memory exhaustion).
|
|
50
|
+
- `timeout` cancels long-running conversions and returns HTTP 503.
|
|
51
|
+
- For additional protection, combine with `express-rate-limit` and body parser limits.
|
|
52
|
+
|
|
53
|
+
**Example with rate limiting**:
|
|
54
|
+
```javascript
|
|
55
|
+
const rateLimit = require('express-rate-limit');
|
|
56
|
+
const importLimiter = rateLimit({
|
|
57
|
+
windowMs: 15 * 60 * 1000, // 15 minutes
|
|
58
|
+
max: 10 // limit each IP to 10 requests per window
|
|
59
|
+
});
|
|
60
|
+
app.post('/api/import', importLimiter, middleware());
|
|
61
|
+
```
|
|
43
62
|
|
|
44
63
|
## Helpers
|
|
45
64
|
```javascript
|
|
@@ -61,4 +80,4 @@ const {
|
|
|
61
80
|
"stats": { "inputSize": 0, "outputSize": 0, "processingTime": 0, "conversion": "csv->json" },
|
|
62
81
|
"options": {}
|
|
63
82
|
}
|
|
64
|
-
```
|
|
83
|
+
```
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
1
2
|
/**
|
|
2
3
|
* Пример использования Express middleware для JTCSV
|
|
3
4
|
*
|
|
@@ -119,7 +120,7 @@ app.use((req, res) => {
|
|
|
119
120
|
// Запуск сервера
|
|
120
121
|
app.listen(PORT, () => {
|
|
121
122
|
console.log(`🚀 JTCSV Express server запущен на порту ${PORT}`);
|
|
122
|
-
console.log(
|
|
123
|
+
console.log('📚 Примеры запросов:');
|
|
123
124
|
console.log(` curl -X POST http://localhost:${PORT}/api/convert \
|
|
124
125
|
-H "Content-Type: application/json" \
|
|
125
126
|
-d '[{"name":"John","age":30},{"name":"Jane","age":25}]'`);
|
|
@@ -131,6 +132,4 @@ app.listen(PORT, () => {
|
|
|
131
132
|
console.log(` curl -X GET http://localhost:${PORT}/api/health`);
|
|
132
133
|
});
|
|
133
134
|
|
|
134
|
-
module.exports = app;
|
|
135
|
-
|
|
136
|
-
|
|
135
|
+
module.exports = app;
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
2
|
+
/**
|
|
3
|
+
* Пример использования Express middleware для JTCSV
|
|
4
|
+
*
|
|
5
|
+
* Запуск: node example.js
|
|
6
|
+
* Затем отправьте запросы:
|
|
7
|
+
* - POST /api/convert с JSON телом → получите CSV
|
|
8
|
+
* - POST /api/convert с CSV телом → получите JSON
|
|
9
|
+
* - POST /api/csv-to-json → конвертация CSV в JSON
|
|
10
|
+
* - POST /api/json-to-csv → конвертация JSON в CSV
|
|
11
|
+
* - GET /api/health → проверка состояния
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import express from "express";
|
|
15
|
+
import bodyParser from "body-parser";
|
|
16
|
+
const {
|
|
17
|
+
middleware,
|
|
18
|
+
csvToJsonRoute,
|
|
19
|
+
jsonToCsvRoute,
|
|
20
|
+
healthCheck
|
|
21
|
+
} = await import("./index");
|
|
22
|
+
|
|
23
|
+
const app = express();
|
|
24
|
+
const PORT = process.env.PORT || 3000;
|
|
25
|
+
|
|
26
|
+
// Middleware для парсинга JSON и текста
|
|
27
|
+
app.use(bodyParser.json());
|
|
28
|
+
app.use(bodyParser.text({ type: 'text/csv' }));
|
|
29
|
+
|
|
30
|
+
// Добавляем время начала обработки запроса
|
|
31
|
+
app.use((req, res, next) => {
|
|
32
|
+
req.startTime = Date.now();
|
|
33
|
+
next();
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
// Основное middleware для автоматической конвертации
|
|
37
|
+
app.use(middleware({
|
|
38
|
+
maxSize: '50mb',
|
|
39
|
+
delimiter: ',',
|
|
40
|
+
enableFastPath: true,
|
|
41
|
+
preventCsvInjection: true
|
|
42
|
+
}));
|
|
43
|
+
|
|
44
|
+
// Роуты для конкретных операций
|
|
45
|
+
app.post('/api/csv-to-json', csvToJsonRoute({
|
|
46
|
+
delimiter: ',',
|
|
47
|
+
parseNumbers: true,
|
|
48
|
+
parseBooleans: true
|
|
49
|
+
}));
|
|
50
|
+
|
|
51
|
+
app.post('/api/json-to-csv', jsonToCsvRoute({
|
|
52
|
+
delimiter: ',',
|
|
53
|
+
includeHeaders: true,
|
|
54
|
+
preventCsvInjection: true
|
|
55
|
+
}));
|
|
56
|
+
|
|
57
|
+
// Health check
|
|
58
|
+
app.get('/api/health', healthCheck());
|
|
59
|
+
|
|
60
|
+
// Пример роута, использующего автоматическую конвертацию
|
|
61
|
+
app.post('/api/convert', (req, res) => {
|
|
62
|
+
if (!req.converted) {
|
|
63
|
+
return res.status(400).json({
|
|
64
|
+
success: false,
|
|
65
|
+
error: 'No data to convert'
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
res.json({
|
|
70
|
+
success: true,
|
|
71
|
+
conversion: req.converted.conversion,
|
|
72
|
+
data: req.converted.data,
|
|
73
|
+
stats: {
|
|
74
|
+
...req.converted.stats,
|
|
75
|
+
totalTime: Date.now() - req.startTime
|
|
76
|
+
},
|
|
77
|
+
format: req.converted.outputFormat
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
// Пример роута для скачивания CSV
|
|
82
|
+
app.post('/api/download-csv', (req, res) => {
|
|
83
|
+
if (!req.converted || req.converted.outputFormat !== 'csv') {
|
|
84
|
+
return res.status(400).json({
|
|
85
|
+
success: false,
|
|
86
|
+
error: 'CSV data not available'
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
res.setHeader('Content-Type', 'text/csv');
|
|
91
|
+
res.setHeader('Content-Disposition', 'attachment; filename="converted.csv"');
|
|
92
|
+
res.send(req.converted.data);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
// Обработка ошибок
|
|
96
|
+
app.use((err, req, res, next) => {
|
|
97
|
+
console.error('Server error:', err);
|
|
98
|
+
res.status(500).json({
|
|
99
|
+
success: false,
|
|
100
|
+
error: 'Internal server error',
|
|
101
|
+
message: process.env.NODE_ENV === 'development' ? err.message : undefined
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
// 404 handler
|
|
106
|
+
app.use((req, res) => {
|
|
107
|
+
res.status(404).json({
|
|
108
|
+
success: false,
|
|
109
|
+
error: 'Route not found',
|
|
110
|
+
availableRoutes: [
|
|
111
|
+
'POST /api/convert',
|
|
112
|
+
'POST /api/csv-to-json',
|
|
113
|
+
'POST /api/json-to-csv',
|
|
114
|
+
'POST /api/download-csv',
|
|
115
|
+
'GET /api/health'
|
|
116
|
+
]
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
// Запуск сервера
|
|
121
|
+
app.listen(PORT, () => {
|
|
122
|
+
console.log(`🚀 JTCSV Express server запущен на порту ${PORT}`);
|
|
123
|
+
console.log('📚 Примеры запросов:');
|
|
124
|
+
console.log(` curl -X POST http://localhost:${PORT}/api/convert \
|
|
125
|
+
-H "Content-Type: application/json" \
|
|
126
|
+
-d '[{"name":"John","age":30},{"name":"Jane","age":25}]'`);
|
|
127
|
+
console.log();
|
|
128
|
+
console.log(` curl -X POST http://localhost:${PORT}/api/convert \
|
|
129
|
+
-H "Content-Type: text/csv" \
|
|
130
|
+
-d 'name,age\nJohn,30\nJane,25'`);
|
|
131
|
+
console.log();
|
|
132
|
+
console.log(` curl -X GET http://localhost:${PORT}/api/health`);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
export default app;
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
declare module '@jtcsv/express-middleware' {
|
|
9
9
|
import { RequestHandler } from 'express';
|
|
10
|
-
import { CsvToJsonOptions, JsonToCsvOptions } from 'jtcsv
|
|
10
|
+
import { CsvToJsonOptions, JsonToCsvOptions } from 'jtcsv';
|
|
11
11
|
|
|
12
12
|
export interface JtcsvMiddlewareOptions {
|
|
13
13
|
/** Максимальный размер тела запроса */
|