jtcsv 2.1.3 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +60 -341
- package/bin/jtcsv.js +2462 -1372
- package/csv-to-json.js +35 -26
- package/dist/jtcsv.cjs.js +807 -133
- package/dist/jtcsv.cjs.js.map +1 -1
- package/dist/jtcsv.esm.js +800 -134
- package/dist/jtcsv.esm.js.map +1 -1
- package/dist/jtcsv.umd.js +807 -133
- package/dist/jtcsv.umd.js.map +1 -1
- package/errors.js +20 -0
- package/examples/browser-vanilla.html +37 -0
- package/examples/cli-batch-processing.js +38 -0
- package/examples/error-handling.js +324 -0
- package/examples/ndjson-processing.js +434 -0
- package/examples/react-integration.jsx +637 -0
- package/examples/schema-validation.js +640 -0
- package/examples/simple-usage.js +10 -7
- package/examples/typescript-example.ts +486 -0
- package/examples/web-workers-advanced.js +28 -0
- package/index.d.ts +2 -0
- package/json-save.js +2 -1
- package/json-to-csv.js +171 -131
- package/package.json +20 -4
- package/plugins/README.md +41 -467
- package/plugins/express-middleware/README.md +32 -274
- package/plugins/hono/README.md +16 -13
- package/plugins/nestjs/README.md +13 -11
- package/plugins/nextjs-api/README.md +28 -423
- package/plugins/nextjs-api/index.js +1 -2
- package/plugins/nextjs-api/route.js +1 -2
- package/plugins/nuxt/README.md +6 -7
- package/plugins/remix/README.md +9 -9
- package/plugins/sveltekit/README.md +8 -8
- package/plugins/trpc/README.md +8 -5
- package/src/browser/browser-functions.js +33 -3
- package/src/browser/csv-to-json-browser.js +269 -11
- package/src/browser/errors-browser.js +19 -1
- package/src/browser/index.js +39 -5
- package/src/browser/streams.js +393 -0
- package/src/browser/workers/csv-parser.worker.js +20 -2
- package/src/browser/workers/worker-pool.js +507 -447
- package/src/core/plugin-system.js +4 -0
- package/src/engines/fast-path-engine.js +31 -23
- package/src/errors.js +26 -0
- package/src/formats/ndjson-parser.js +54 -5
- package/src/formats/tsv-parser.js +4 -1
- package/src/utils/schema-validator.js +594 -0
- package/src/utils/transform-loader.js +205 -0
- package/src/web-server/index.js +683 -0
- package/stream-csv-to-json.js +16 -87
- package/stream-json-to-csv.js +18 -86
package/dist/jtcsv.cjs.js
CHANGED
|
@@ -98,6 +98,22 @@ class ConfigurationError extends JTCSVError {
|
|
|
98
98
|
this.name = 'ConfigurationError';
|
|
99
99
|
}
|
|
100
100
|
}
|
|
101
|
+
const ERROR_CODES = {
|
|
102
|
+
JTCSV_ERROR: 'JTCSV_ERROR',
|
|
103
|
+
VALIDATION_ERROR: 'VALIDATION_ERROR',
|
|
104
|
+
SECURITY_ERROR: 'SECURITY_ERROR',
|
|
105
|
+
FILE_SYSTEM_ERROR: 'FILE_SYSTEM_ERROR',
|
|
106
|
+
PARSING_ERROR: 'PARSING_ERROR',
|
|
107
|
+
LIMIT_ERROR: 'LIMIT_ERROR',
|
|
108
|
+
CONFIGURATION_ERROR: 'CONFIGURATION_ERROR',
|
|
109
|
+
INVALID_INPUT: 'INVALID_INPUT',
|
|
110
|
+
SECURITY_VIOLATION: 'SECURITY_VIOLATION',
|
|
111
|
+
FILE_NOT_FOUND: 'FILE_NOT_FOUND',
|
|
112
|
+
PARSE_FAILED: 'PARSE_FAILED',
|
|
113
|
+
SIZE_LIMIT: 'SIZE_LIMIT',
|
|
114
|
+
INVALID_CONFIG: 'INVALID_CONFIG',
|
|
115
|
+
UNKNOWN_ERROR: 'UNKNOWN_ERROR'
|
|
116
|
+
};
|
|
101
117
|
|
|
102
118
|
/**
|
|
103
119
|
* Безопасное выполнение функции с обработкой ошибок
|
|
@@ -233,6 +249,7 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
233
249
|
ParsingError,
|
|
234
250
|
LimitError,
|
|
235
251
|
ConfigurationError,
|
|
252
|
+
ERROR_CODES,
|
|
236
253
|
safeExecute,
|
|
237
254
|
safeExecuteAsync
|
|
238
255
|
};
|
|
@@ -516,15 +533,10 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
516
533
|
|
|
517
534
|
|
|
518
535
|
/**
|
|
519
|
-
* Валидация
|
|
536
|
+
* Валидация опций парсинга
|
|
520
537
|
* @private
|
|
521
538
|
*/
|
|
522
|
-
function
|
|
523
|
-
// Validate CSV input
|
|
524
|
-
if (typeof csv !== 'string') {
|
|
525
|
-
throw new ValidationError('Input must be a CSV string');
|
|
526
|
-
}
|
|
527
|
-
|
|
539
|
+
function validateCsvOptions(options) {
|
|
528
540
|
// Validate options
|
|
529
541
|
if (options && typeof options !== 'object') {
|
|
530
542
|
throw new ConfigurationError('Options must be an object');
|
|
@@ -552,9 +564,24 @@ function validateCsvInput(csv, options) {
|
|
|
552
564
|
if ((options === null || options === void 0 ? void 0 : options.maxRows) !== undefined && (typeof options.maxRows !== 'number' || options.maxRows <= 0)) {
|
|
553
565
|
throw new ConfigurationError('maxRows must be a positive number');
|
|
554
566
|
}
|
|
567
|
+
if ((options === null || options === void 0 ? void 0 : options.warnExtraFields) !== undefined && typeof options.warnExtraFields !== 'boolean') {
|
|
568
|
+
throw new ConfigurationError('warnExtraFields must be a boolean');
|
|
569
|
+
}
|
|
555
570
|
return true;
|
|
556
571
|
}
|
|
557
572
|
|
|
573
|
+
/**
|
|
574
|
+
* Валидация CSV ввода и опций
|
|
575
|
+
* @private
|
|
576
|
+
*/
|
|
577
|
+
function validateCsvInput(csv, options) {
|
|
578
|
+
// Validate CSV input
|
|
579
|
+
if (typeof csv !== 'string') {
|
|
580
|
+
throw new ValidationError('Input must be a CSV string');
|
|
581
|
+
}
|
|
582
|
+
return validateCsvOptions(options);
|
|
583
|
+
}
|
|
584
|
+
|
|
558
585
|
/**
|
|
559
586
|
* Парсинг одной строки CSV с правильным экранированием
|
|
560
587
|
* @private
|
|
@@ -700,6 +727,78 @@ function parseCsvValue(value, options) {
|
|
|
700
727
|
}
|
|
701
728
|
return result;
|
|
702
729
|
}
|
|
730
|
+
function isSimpleCsv(csv) {
|
|
731
|
+
return csv.indexOf('"') === -1 && csv.indexOf('\\') === -1;
|
|
732
|
+
}
|
|
733
|
+
function parseSimpleCsv(csv, delimiter, options) {
|
|
734
|
+
const {
|
|
735
|
+
hasHeaders = true,
|
|
736
|
+
renameMap = {},
|
|
737
|
+
trim = true,
|
|
738
|
+
parseNumbers = false,
|
|
739
|
+
parseBooleans = false,
|
|
740
|
+
maxRows
|
|
741
|
+
} = options;
|
|
742
|
+
const result = [];
|
|
743
|
+
let headers = null;
|
|
744
|
+
let fieldStart = 0;
|
|
745
|
+
let currentRow = [];
|
|
746
|
+
let rowHasData = false;
|
|
747
|
+
let rowCount = 0;
|
|
748
|
+
const finalizeRow = fields => {
|
|
749
|
+
if (fields.length === 1 && fields[0].trim() === '') {
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
if (!headers) {
|
|
753
|
+
if (hasHeaders) {
|
|
754
|
+
headers = fields.map(header => {
|
|
755
|
+
const trimmed = trim ? header.trim() : header;
|
|
756
|
+
return renameMap[trimmed] || trimmed;
|
|
757
|
+
});
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
headers = fields.map((_, index) => `column${index + 1}`);
|
|
761
|
+
}
|
|
762
|
+
rowCount++;
|
|
763
|
+
if (maxRows && rowCount > maxRows) {
|
|
764
|
+
throw new LimitError(`CSV size exceeds maximum limit of ${maxRows} rows`, maxRows, rowCount);
|
|
765
|
+
}
|
|
766
|
+
const row = {};
|
|
767
|
+
const fieldCount = Math.min(fields.length, headers.length);
|
|
768
|
+
for (let i = 0; i < fieldCount; i++) {
|
|
769
|
+
row[headers[i]] = parseCsvValue(fields[i], {
|
|
770
|
+
trim,
|
|
771
|
+
parseNumbers,
|
|
772
|
+
parseBooleans
|
|
773
|
+
});
|
|
774
|
+
}
|
|
775
|
+
result.push(row);
|
|
776
|
+
};
|
|
777
|
+
let i = 0;
|
|
778
|
+
while (i <= csv.length) {
|
|
779
|
+
const char = i < csv.length ? csv[i] : '\n';
|
|
780
|
+
if (char !== '\r' && char !== '\n' && char !== ' ' && char !== '\t') {
|
|
781
|
+
rowHasData = true;
|
|
782
|
+
}
|
|
783
|
+
if (char === delimiter || char === '\n' || char === '\r' || i === csv.length) {
|
|
784
|
+
const field = csv.slice(fieldStart, i);
|
|
785
|
+
currentRow.push(field);
|
|
786
|
+
if (char === '\n' || char === '\r' || i === csv.length) {
|
|
787
|
+
if (rowHasData || currentRow.length > 1) {
|
|
788
|
+
finalizeRow(currentRow);
|
|
789
|
+
}
|
|
790
|
+
currentRow = [];
|
|
791
|
+
rowHasData = false;
|
|
792
|
+
}
|
|
793
|
+
if (char === '\r' && csv[i + 1] === '\n') {
|
|
794
|
+
i++;
|
|
795
|
+
}
|
|
796
|
+
fieldStart = i + 1;
|
|
797
|
+
}
|
|
798
|
+
i++;
|
|
799
|
+
}
|
|
800
|
+
return result;
|
|
801
|
+
}
|
|
703
802
|
|
|
704
803
|
/**
|
|
705
804
|
* Автоматическое определение разделителя CSV
|
|
@@ -775,7 +874,8 @@ function csvToJson(csv, options = {}) {
|
|
|
775
874
|
trim = true,
|
|
776
875
|
parseNumbers = false,
|
|
777
876
|
parseBooleans = false,
|
|
778
|
-
maxRows
|
|
877
|
+
maxRows,
|
|
878
|
+
warnExtraFields = true
|
|
779
879
|
} = opts;
|
|
780
880
|
|
|
781
881
|
// Определение разделителя
|
|
@@ -789,6 +889,16 @@ function csvToJson(csv, options = {}) {
|
|
|
789
889
|
if (csv.trim() === '') {
|
|
790
890
|
return [];
|
|
791
891
|
}
|
|
892
|
+
if (isSimpleCsv(csv)) {
|
|
893
|
+
return parseSimpleCsv(csv, finalDelimiter, {
|
|
894
|
+
hasHeaders,
|
|
895
|
+
renameMap,
|
|
896
|
+
trim,
|
|
897
|
+
parseNumbers,
|
|
898
|
+
parseBooleans,
|
|
899
|
+
maxRows
|
|
900
|
+
});
|
|
901
|
+
}
|
|
792
902
|
|
|
793
903
|
// Парсинг CSV с обработкой кавычек и переносов строк
|
|
794
904
|
const lines = [];
|
|
@@ -892,7 +1002,8 @@ function csvToJson(csv, options = {}) {
|
|
|
892
1002
|
}
|
|
893
1003
|
|
|
894
1004
|
// Предупреждение о лишних полях
|
|
895
|
-
|
|
1005
|
+
const isDev = typeof process !== 'undefined' && process.env && process.env.NODE_ENV === 'development';
|
|
1006
|
+
if (fields.length > headers.length && warnExtraFields && isDev) {
|
|
896
1007
|
console.warn(`[jtcsv] Line ${i + 1}: ${fields.length - headers.length} extra fields ignored`);
|
|
897
1008
|
}
|
|
898
1009
|
result.push(row);
|
|
@@ -908,12 +1019,464 @@ function csvToJson(csv, options = {}) {
|
|
|
908
1019
|
function: 'csvToJson'
|
|
909
1020
|
});
|
|
910
1021
|
}
|
|
1022
|
+
async function* csvToJsonIterator(input, options = {}) {
|
|
1023
|
+
const opts = options && typeof options === 'object' ? options : {};
|
|
1024
|
+
validateCsvOptions(opts);
|
|
1025
|
+
if (typeof input === 'string') {
|
|
1026
|
+
const rows = csvToJson(input, options);
|
|
1027
|
+
for (const row of rows) {
|
|
1028
|
+
yield row;
|
|
1029
|
+
}
|
|
1030
|
+
return;
|
|
1031
|
+
}
|
|
1032
|
+
const {
|
|
1033
|
+
delimiter,
|
|
1034
|
+
autoDetect = true,
|
|
1035
|
+
candidates = [';', ',', '\t', '|'],
|
|
1036
|
+
hasHeaders = true,
|
|
1037
|
+
renameMap = {},
|
|
1038
|
+
trim = true,
|
|
1039
|
+
parseNumbers = false,
|
|
1040
|
+
parseBooleans = false,
|
|
1041
|
+
maxRows
|
|
1042
|
+
} = opts;
|
|
1043
|
+
const stream = input instanceof Blob && input.stream ? input.stream() : input;
|
|
1044
|
+
if (!stream || typeof stream.getReader !== 'function') {
|
|
1045
|
+
throw new ValidationError('Input must be a CSV string, Blob/File, or ReadableStream');
|
|
1046
|
+
}
|
|
1047
|
+
const reader = stream.getReader();
|
|
1048
|
+
const decoder = new TextDecoder('utf-8');
|
|
1049
|
+
let buffer = '';
|
|
1050
|
+
let insideQuotes = false;
|
|
1051
|
+
let headers = null;
|
|
1052
|
+
let rowCount = 0;
|
|
1053
|
+
let lineNumber = 0;
|
|
1054
|
+
let finalDelimiter = delimiter;
|
|
1055
|
+
let delimiterResolved = Boolean(finalDelimiter);
|
|
1056
|
+
const processFields = fields => {
|
|
1057
|
+
if (fields.length === 1 && fields[0].trim() === '') {
|
|
1058
|
+
return null;
|
|
1059
|
+
}
|
|
1060
|
+
rowCount++;
|
|
1061
|
+
if (maxRows && rowCount > maxRows) {
|
|
1062
|
+
throw new LimitError(`CSV size exceeds maximum limit of ${maxRows} rows`, maxRows, rowCount);
|
|
1063
|
+
}
|
|
1064
|
+
const row = {};
|
|
1065
|
+
const fieldCount = Math.min(fields.length, headers.length);
|
|
1066
|
+
for (let j = 0; j < fieldCount; j++) {
|
|
1067
|
+
row[headers[j]] = parseCsvValue(fields[j], {
|
|
1068
|
+
trim,
|
|
1069
|
+
parseNumbers,
|
|
1070
|
+
parseBooleans
|
|
1071
|
+
});
|
|
1072
|
+
}
|
|
1073
|
+
return row;
|
|
1074
|
+
};
|
|
1075
|
+
const processLine = line => {
|
|
1076
|
+
lineNumber++;
|
|
1077
|
+
let cleanLine = line;
|
|
1078
|
+
if (cleanLine.endsWith('\r')) {
|
|
1079
|
+
cleanLine = cleanLine.slice(0, -1);
|
|
1080
|
+
}
|
|
1081
|
+
if (!delimiterResolved) {
|
|
1082
|
+
if (!finalDelimiter && autoDetect) {
|
|
1083
|
+
finalDelimiter = autoDetectDelimiter(cleanLine, candidates);
|
|
1084
|
+
}
|
|
1085
|
+
finalDelimiter = finalDelimiter || ';';
|
|
1086
|
+
delimiterResolved = true;
|
|
1087
|
+
}
|
|
1088
|
+
if (cleanLine.trim() === '') {
|
|
1089
|
+
return null;
|
|
1090
|
+
}
|
|
1091
|
+
if (!headers) {
|
|
1092
|
+
if (hasHeaders) {
|
|
1093
|
+
headers = parseCsvLine(cleanLine, lineNumber, finalDelimiter).map(header => {
|
|
1094
|
+
const trimmed = trim ? header.trim() : header;
|
|
1095
|
+
return renameMap[trimmed] || trimmed;
|
|
1096
|
+
});
|
|
1097
|
+
return null;
|
|
1098
|
+
}
|
|
1099
|
+
const fields = parseCsvLine(cleanLine, lineNumber, finalDelimiter);
|
|
1100
|
+
headers = fields.map((_, index) => `column${index + 1}`);
|
|
1101
|
+
return processFields(fields);
|
|
1102
|
+
}
|
|
1103
|
+
const fields = parseCsvLine(cleanLine, lineNumber, finalDelimiter);
|
|
1104
|
+
return processFields(fields);
|
|
1105
|
+
};
|
|
1106
|
+
while (true) {
|
|
1107
|
+
const {
|
|
1108
|
+
value,
|
|
1109
|
+
done
|
|
1110
|
+
} = await reader.read();
|
|
1111
|
+
if (done) {
|
|
1112
|
+
break;
|
|
1113
|
+
}
|
|
1114
|
+
buffer += decoder.decode(value, {
|
|
1115
|
+
stream: true
|
|
1116
|
+
});
|
|
1117
|
+
let start = 0;
|
|
1118
|
+
for (let i = 0; i < buffer.length; i++) {
|
|
1119
|
+
const char = buffer[i];
|
|
1120
|
+
if (char === '"') {
|
|
1121
|
+
if (insideQuotes && buffer[i + 1] === '"') {
|
|
1122
|
+
i++;
|
|
1123
|
+
continue;
|
|
1124
|
+
}
|
|
1125
|
+
insideQuotes = !insideQuotes;
|
|
1126
|
+
continue;
|
|
1127
|
+
}
|
|
1128
|
+
if (char === '\n' && !insideQuotes) {
|
|
1129
|
+
const line = buffer.slice(start, i);
|
|
1130
|
+
start = i + 1;
|
|
1131
|
+
const row = processLine(line);
|
|
1132
|
+
if (row) {
|
|
1133
|
+
yield row;
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
buffer = buffer.slice(start);
|
|
1138
|
+
}
|
|
1139
|
+
if (buffer.length > 0) {
|
|
1140
|
+
const row = processLine(buffer);
|
|
1141
|
+
if (row) {
|
|
1142
|
+
yield row;
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
if (insideQuotes) {
|
|
1146
|
+
throw new ParsingError('Unclosed quotes in CSV', lineNumber);
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
911
1149
|
|
|
912
1150
|
// Экспорт для Node.js совместимости
|
|
913
1151
|
if (typeof module !== 'undefined' && module.exports) {
|
|
914
1152
|
module.exports = {
|
|
915
1153
|
csvToJson,
|
|
916
|
-
autoDetectDelimiter
|
|
1154
|
+
autoDetectDelimiter,
|
|
1155
|
+
csvToJsonIterator
|
|
1156
|
+
};
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
const DEFAULT_MAX_CHUNK_SIZE = 64 * 1024;
|
|
1160
|
+
function isReadableStream(value) {
|
|
1161
|
+
return value && typeof value.getReader === 'function';
|
|
1162
|
+
}
|
|
1163
|
+
function isAsyncIterable(value) {
|
|
1164
|
+
return value && typeof value[Symbol.asyncIterator] === 'function';
|
|
1165
|
+
}
|
|
1166
|
+
function isIterable(value) {
|
|
1167
|
+
return value && typeof value[Symbol.iterator] === 'function';
|
|
1168
|
+
}
|
|
1169
|
+
function createReadableStreamFromIterator(iterator) {
|
|
1170
|
+
return new ReadableStream({
|
|
1171
|
+
async pull(controller) {
|
|
1172
|
+
try {
|
|
1173
|
+
const {
|
|
1174
|
+
value,
|
|
1175
|
+
done
|
|
1176
|
+
} = await iterator.next();
|
|
1177
|
+
if (done) {
|
|
1178
|
+
controller.close();
|
|
1179
|
+
return;
|
|
1180
|
+
}
|
|
1181
|
+
controller.enqueue(value);
|
|
1182
|
+
} catch (error) {
|
|
1183
|
+
controller.error(error);
|
|
1184
|
+
}
|
|
1185
|
+
},
|
|
1186
|
+
cancel() {
|
|
1187
|
+
if (iterator.return) {
|
|
1188
|
+
iterator.return();
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
});
|
|
1192
|
+
}
|
|
1193
|
+
function detectInputFormat(input, options) {
|
|
1194
|
+
if (options && options.inputFormat) {
|
|
1195
|
+
return options.inputFormat;
|
|
1196
|
+
}
|
|
1197
|
+
if (typeof input === 'string') {
|
|
1198
|
+
const trimmed = input.trim();
|
|
1199
|
+
if (trimmed.startsWith('[')) {
|
|
1200
|
+
return 'json-array';
|
|
1201
|
+
}
|
|
1202
|
+
if (trimmed.includes('\n')) {
|
|
1203
|
+
return 'ndjson';
|
|
1204
|
+
}
|
|
1205
|
+
return 'json-array';
|
|
1206
|
+
}
|
|
1207
|
+
if (input instanceof Blob || isReadableStream(input)) {
|
|
1208
|
+
return 'ndjson';
|
|
1209
|
+
}
|
|
1210
|
+
return 'json-array';
|
|
1211
|
+
}
|
|
1212
|
+
async function* parseNdjsonText(text) {
|
|
1213
|
+
const lines = text.split(/\r?\n/);
|
|
1214
|
+
for (const line of lines) {
|
|
1215
|
+
const trimmed = line.trim();
|
|
1216
|
+
if (!trimmed) {
|
|
1217
|
+
continue;
|
|
1218
|
+
}
|
|
1219
|
+
yield JSON.parse(trimmed);
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
async function* parseNdjsonStream(stream) {
|
|
1223
|
+
const reader = stream.getReader();
|
|
1224
|
+
const decoder = new TextDecoder('utf-8');
|
|
1225
|
+
let buffer = '';
|
|
1226
|
+
while (true) {
|
|
1227
|
+
const {
|
|
1228
|
+
value,
|
|
1229
|
+
done
|
|
1230
|
+
} = await reader.read();
|
|
1231
|
+
if (done) {
|
|
1232
|
+
break;
|
|
1233
|
+
}
|
|
1234
|
+
buffer += decoder.decode(value, {
|
|
1235
|
+
stream: true
|
|
1236
|
+
});
|
|
1237
|
+
const lines = buffer.split(/\r?\n/);
|
|
1238
|
+
buffer = lines.pop() || '';
|
|
1239
|
+
for (const line of lines) {
|
|
1240
|
+
const trimmed = line.trim();
|
|
1241
|
+
if (!trimmed) {
|
|
1242
|
+
continue;
|
|
1243
|
+
}
|
|
1244
|
+
yield JSON.parse(trimmed);
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
if (buffer.trim()) {
|
|
1248
|
+
yield JSON.parse(buffer.trim());
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
async function* normalizeJsonInput(input, options = {}) {
|
|
1252
|
+
const format = detectInputFormat(input, options);
|
|
1253
|
+
if (Array.isArray(input)) {
|
|
1254
|
+
for (const item of input) {
|
|
1255
|
+
yield item;
|
|
1256
|
+
}
|
|
1257
|
+
return;
|
|
1258
|
+
}
|
|
1259
|
+
if (isAsyncIterable(input)) {
|
|
1260
|
+
for await (const item of input) {
|
|
1261
|
+
yield item;
|
|
1262
|
+
}
|
|
1263
|
+
return;
|
|
1264
|
+
}
|
|
1265
|
+
if (isIterable(input)) {
|
|
1266
|
+
for (const item of input) {
|
|
1267
|
+
yield item;
|
|
1268
|
+
}
|
|
1269
|
+
return;
|
|
1270
|
+
}
|
|
1271
|
+
if (typeof input === 'string') {
|
|
1272
|
+
if (format === 'ndjson') {
|
|
1273
|
+
yield* parseNdjsonText(input);
|
|
1274
|
+
return;
|
|
1275
|
+
}
|
|
1276
|
+
const parsed = JSON.parse(input);
|
|
1277
|
+
if (Array.isArray(parsed)) {
|
|
1278
|
+
for (const item of parsed) {
|
|
1279
|
+
yield item;
|
|
1280
|
+
}
|
|
1281
|
+
return;
|
|
1282
|
+
}
|
|
1283
|
+
yield parsed;
|
|
1284
|
+
return;
|
|
1285
|
+
}
|
|
1286
|
+
if (input instanceof Blob) {
|
|
1287
|
+
if (format === 'ndjson') {
|
|
1288
|
+
yield* parseNdjsonStream(input.stream());
|
|
1289
|
+
return;
|
|
1290
|
+
}
|
|
1291
|
+
const text = await input.text();
|
|
1292
|
+
const parsed = JSON.parse(text);
|
|
1293
|
+
if (Array.isArray(parsed)) {
|
|
1294
|
+
for (const item of parsed) {
|
|
1295
|
+
yield item;
|
|
1296
|
+
}
|
|
1297
|
+
return;
|
|
1298
|
+
}
|
|
1299
|
+
yield parsed;
|
|
1300
|
+
return;
|
|
1301
|
+
}
|
|
1302
|
+
if (isReadableStream(input)) {
|
|
1303
|
+
if (format !== 'ndjson') {
|
|
1304
|
+
throw new ValidationError('ReadableStream input requires inputFormat="ndjson"');
|
|
1305
|
+
}
|
|
1306
|
+
yield* parseNdjsonStream(input);
|
|
1307
|
+
return;
|
|
1308
|
+
}
|
|
1309
|
+
throw new ValidationError('Input must be an array, iterable, string, Blob, or ReadableStream');
|
|
1310
|
+
}
|
|
1311
|
+
function validateStreamOptions(options) {
|
|
1312
|
+
if (options && typeof options !== 'object') {
|
|
1313
|
+
throw new ConfigurationError('Options must be an object');
|
|
1314
|
+
}
|
|
1315
|
+
if (options !== null && options !== void 0 && options.delimiter && typeof options.delimiter !== 'string') {
|
|
1316
|
+
throw new ConfigurationError('Delimiter must be a string');
|
|
1317
|
+
}
|
|
1318
|
+
if (options !== null && options !== void 0 && options.delimiter && options.delimiter.length !== 1) {
|
|
1319
|
+
throw new ConfigurationError('Delimiter must be a single character');
|
|
1320
|
+
}
|
|
1321
|
+
if (options !== null && options !== void 0 && options.renameMap && typeof options.renameMap !== 'object') {
|
|
1322
|
+
throw new ConfigurationError('renameMap must be an object');
|
|
1323
|
+
}
|
|
1324
|
+
if ((options === null || options === void 0 ? void 0 : options.maxRecords) !== undefined) {
|
|
1325
|
+
if (typeof options.maxRecords !== 'number' || options.maxRecords <= 0) {
|
|
1326
|
+
throw new ConfigurationError('maxRecords must be a positive number');
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
function escapeCsvValue(value, options) {
|
|
1331
|
+
const {
|
|
1332
|
+
delimiter,
|
|
1333
|
+
preventCsvInjection = true,
|
|
1334
|
+
rfc4180Compliant = true
|
|
1335
|
+
} = options;
|
|
1336
|
+
if (value === null || value === undefined || value === '') {
|
|
1337
|
+
return '';
|
|
1338
|
+
}
|
|
1339
|
+
const stringValue = String(value);
|
|
1340
|
+
let escapedValue = stringValue;
|
|
1341
|
+
if (preventCsvInjection && /^[=+\-@]/.test(stringValue)) {
|
|
1342
|
+
escapedValue = "'" + stringValue;
|
|
1343
|
+
}
|
|
1344
|
+
const needsQuoting = rfc4180Compliant ? escapedValue.includes(delimiter) || escapedValue.includes('"') || escapedValue.includes('\n') || escapedValue.includes('\r') : escapedValue.includes(delimiter) || escapedValue.includes('"') || escapedValue.includes('\n') || escapedValue.includes('\r');
|
|
1345
|
+
if (needsQuoting) {
|
|
1346
|
+
return `"${escapedValue.replace(/"/g, '""')}"`;
|
|
1347
|
+
}
|
|
1348
|
+
return escapedValue;
|
|
1349
|
+
}
|
|
1350
|
+
function buildHeaderState(keys, options) {
|
|
1351
|
+
const renameMap = options.renameMap || {};
|
|
1352
|
+
const template = options.template || {};
|
|
1353
|
+
const originalKeys = Array.isArray(options.headers) ? options.headers : keys;
|
|
1354
|
+
const headers = originalKeys.map(key => renameMap[key] || key);
|
|
1355
|
+
const reverseRenameMap = {};
|
|
1356
|
+
originalKeys.forEach((key, index) => {
|
|
1357
|
+
reverseRenameMap[headers[index]] = key;
|
|
1358
|
+
});
|
|
1359
|
+
let finalHeaders = headers;
|
|
1360
|
+
if (Object.keys(template).length > 0) {
|
|
1361
|
+
const templateHeaders = Object.keys(template).map(key => renameMap[key] || key);
|
|
1362
|
+
const extraHeaders = headers.filter(h => !templateHeaders.includes(h));
|
|
1363
|
+
finalHeaders = [...templateHeaders, ...extraHeaders];
|
|
1364
|
+
}
|
|
1365
|
+
return {
|
|
1366
|
+
headers: finalHeaders,
|
|
1367
|
+
reverseRenameMap
|
|
1368
|
+
};
|
|
1369
|
+
}
|
|
1370
|
+
async function* jsonToCsvChunkIterator(input, options = {}) {
|
|
1371
|
+
validateStreamOptions(options);
|
|
1372
|
+
const opts = options && typeof options === 'object' ? options : {};
|
|
1373
|
+
const {
|
|
1374
|
+
delimiter = ';',
|
|
1375
|
+
includeHeaders = true,
|
|
1376
|
+
maxRecords,
|
|
1377
|
+
maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
|
|
1378
|
+
headerMode
|
|
1379
|
+
} = opts;
|
|
1380
|
+
let headerState = null;
|
|
1381
|
+
let buffer = '';
|
|
1382
|
+
let recordCount = 0;
|
|
1383
|
+
const lineEnding = opts.rfc4180Compliant === false ? '\n' : '\r\n';
|
|
1384
|
+
if (Array.isArray(input) && !opts.headers && (!headerMode || headerMode === 'all')) {
|
|
1385
|
+
const allKeys = new Set();
|
|
1386
|
+
for (const item of input) {
|
|
1387
|
+
if (!item || typeof item !== 'object') {
|
|
1388
|
+
continue;
|
|
1389
|
+
}
|
|
1390
|
+
Object.keys(item).forEach(key => allKeys.add(key));
|
|
1391
|
+
}
|
|
1392
|
+
headerState = buildHeaderState(Array.from(allKeys), opts);
|
|
1393
|
+
if (includeHeaders && headerState.headers.length > 0) {
|
|
1394
|
+
buffer += headerState.headers.join(delimiter) + lineEnding;
|
|
1395
|
+
}
|
|
1396
|
+
} else if (Array.isArray(opts.headers)) {
|
|
1397
|
+
headerState = buildHeaderState(opts.headers, opts);
|
|
1398
|
+
if (includeHeaders && headerState.headers.length > 0) {
|
|
1399
|
+
buffer += headerState.headers.join(delimiter) + lineEnding;
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
for await (const item of normalizeJsonInput(input, opts)) {
|
|
1403
|
+
if (!item || typeof item !== 'object') {
|
|
1404
|
+
continue;
|
|
1405
|
+
}
|
|
1406
|
+
if (!headerState) {
|
|
1407
|
+
headerState = buildHeaderState(Object.keys(item), opts);
|
|
1408
|
+
if (includeHeaders && headerState.headers.length > 0) {
|
|
1409
|
+
buffer += headerState.headers.join(delimiter) + lineEnding;
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
recordCount += 1;
|
|
1413
|
+
if (maxRecords && recordCount > maxRecords) {
|
|
1414
|
+
throw new LimitError(`Data size exceeds maximum limit of ${maxRecords} records`, maxRecords, recordCount);
|
|
1415
|
+
}
|
|
1416
|
+
const row = headerState.headers.map(header => {
|
|
1417
|
+
const originalKey = headerState.reverseRenameMap[header] || header;
|
|
1418
|
+
return escapeCsvValue(item[originalKey], {
|
|
1419
|
+
delimiter,
|
|
1420
|
+
preventCsvInjection: opts.preventCsvInjection !== false,
|
|
1421
|
+
rfc4180Compliant: opts.rfc4180Compliant !== false
|
|
1422
|
+
});
|
|
1423
|
+
}).join(delimiter);
|
|
1424
|
+
buffer += row + lineEnding;
|
|
1425
|
+
if (buffer.length >= maxChunkSize) {
|
|
1426
|
+
yield buffer;
|
|
1427
|
+
buffer = '';
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
if (buffer.length > 0) {
|
|
1431
|
+
yield buffer;
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
async function* jsonToNdjsonChunkIterator(input, options = {}) {
|
|
1435
|
+
validateStreamOptions(options);
|
|
1436
|
+
for await (const item of normalizeJsonInput(input, options)) {
|
|
1437
|
+
if (item === undefined) {
|
|
1438
|
+
continue;
|
|
1439
|
+
}
|
|
1440
|
+
yield JSON.stringify(item) + '\n';
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
async function* csvToJsonChunkIterator(input, options = {}) {
|
|
1444
|
+
const outputFormat = options.outputFormat || 'ndjson';
|
|
1445
|
+
const asArray = outputFormat === 'json-array' || outputFormat === 'array' || outputFormat === 'json';
|
|
1446
|
+
let first = true;
|
|
1447
|
+
if (asArray) {
|
|
1448
|
+
yield '[';
|
|
1449
|
+
}
|
|
1450
|
+
for await (const row of csvToJsonIterator(input, options)) {
|
|
1451
|
+
const payload = JSON.stringify(row);
|
|
1452
|
+
if (asArray) {
|
|
1453
|
+
yield (first ? '' : ',') + payload;
|
|
1454
|
+
} else {
|
|
1455
|
+
yield payload + '\n';
|
|
1456
|
+
}
|
|
1457
|
+
first = false;
|
|
1458
|
+
}
|
|
1459
|
+
if (asArray) {
|
|
1460
|
+
yield ']';
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
function jsonToCsvStream(input, options = {}) {
|
|
1464
|
+
const iterator = jsonToCsvChunkIterator(input, options);
|
|
1465
|
+
return createReadableStreamFromIterator(iterator);
|
|
1466
|
+
}
|
|
1467
|
+
function jsonToNdjsonStream(input, options = {}) {
|
|
1468
|
+
const iterator = jsonToNdjsonChunkIterator(input, options);
|
|
1469
|
+
return createReadableStreamFromIterator(iterator);
|
|
1470
|
+
}
|
|
1471
|
+
function csvToJsonStream(input, options = {}) {
|
|
1472
|
+
const iterator = csvToJsonChunkIterator(input, options);
|
|
1473
|
+
return createReadableStreamFromIterator(iterator);
|
|
1474
|
+
}
|
|
1475
|
+
if (typeof module !== 'undefined' && module.exports) {
|
|
1476
|
+
module.exports = {
|
|
1477
|
+
jsonToCsvStream,
|
|
1478
|
+
jsonToNdjsonStream,
|
|
1479
|
+
csvToJsonStream
|
|
917
1480
|
};
|
|
918
1481
|
}
|
|
919
1482
|
|
|
@@ -1038,6 +1601,26 @@ async function parseCsvFile(file, options = {}) {
|
|
|
1038
1601
|
});
|
|
1039
1602
|
}
|
|
1040
1603
|
|
|
1604
|
+
/**
|
|
1605
|
+
* Stream CSV file as async iterator without full buffering.
|
|
1606
|
+
*
|
|
1607
|
+
* @param {File} file - File selected from input
|
|
1608
|
+
* @param {Object} [options] - csvToJson options
|
|
1609
|
+
* @returns {AsyncGenerator<Object>} Async iterator of rows
|
|
1610
|
+
*/
|
|
1611
|
+
function parseCsvFileStream(file, options = {}) {
|
|
1612
|
+
if (typeof window === 'undefined') {
|
|
1613
|
+
throw new ValidationError('parseCsvFileStream() is browser-only. Use readCsvAsJson() in Node.js');
|
|
1614
|
+
}
|
|
1615
|
+
if (!(file instanceof File)) {
|
|
1616
|
+
throw new ValidationError('Input must be a File object');
|
|
1617
|
+
}
|
|
1618
|
+
if (!file.name.toLowerCase().endsWith('.csv')) {
|
|
1619
|
+
throw new ValidationError('File must have .csv extension');
|
|
1620
|
+
}
|
|
1621
|
+
return csvToJsonIterator(file, options);
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1041
1624
|
/**
|
|
1042
1625
|
* Создает CSV файл из JSON данных (альтернатива downloadAsCsv)
|
|
1043
1626
|
* Возвращает Blob вместо автоматического скачивания
|
|
@@ -1087,8 +1670,12 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
1087
1670
|
module.exports = {
|
|
1088
1671
|
downloadAsCsv,
|
|
1089
1672
|
parseCsvFile,
|
|
1673
|
+
parseCsvFileStream,
|
|
1090
1674
|
createCsvBlob,
|
|
1091
|
-
parseCsvBlob
|
|
1675
|
+
parseCsvBlob,
|
|
1676
|
+
jsonToCsvStream,
|
|
1677
|
+
jsonToNdjsonStream,
|
|
1678
|
+
csvToJsonStream
|
|
1092
1679
|
};
|
|
1093
1680
|
}
|
|
1094
1681
|
|
|
@@ -1098,44 +1685,74 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
1098
1685
|
|
|
1099
1686
|
// Проверка поддержки Web Workers
|
|
1100
1687
|
const WORKERS_SUPPORTED = typeof Worker !== 'undefined';
|
|
1688
|
+
function isTransferableBuffer(value) {
|
|
1689
|
+
if (!(value instanceof ArrayBuffer)) {
|
|
1690
|
+
return false;
|
|
1691
|
+
}
|
|
1692
|
+
if (typeof SharedArrayBuffer !== 'undefined' && value instanceof SharedArrayBuffer) {
|
|
1693
|
+
return false;
|
|
1694
|
+
}
|
|
1695
|
+
return true;
|
|
1696
|
+
}
|
|
1697
|
+
function collectTransferables(args) {
|
|
1698
|
+
const transferables = [];
|
|
1699
|
+
const collectFromValue = value => {
|
|
1700
|
+
if (!value) {
|
|
1701
|
+
return;
|
|
1702
|
+
}
|
|
1703
|
+
if (isTransferableBuffer(value)) {
|
|
1704
|
+
transferables.push(value);
|
|
1705
|
+
return;
|
|
1706
|
+
}
|
|
1707
|
+
if (ArrayBuffer.isView(value) && isTransferableBuffer(value.buffer)) {
|
|
1708
|
+
transferables.push(value.buffer);
|
|
1709
|
+
return;
|
|
1710
|
+
}
|
|
1711
|
+
if (Array.isArray(value)) {
|
|
1712
|
+
value.forEach(collectFromValue);
|
|
1713
|
+
}
|
|
1714
|
+
};
|
|
1715
|
+
args.forEach(collectFromValue);
|
|
1716
|
+
return transferables.length ? transferables : null;
|
|
1717
|
+
}
|
|
1101
1718
|
|
|
1102
|
-
/**
|
|
1103
|
-
* Опции для Worker Pool
|
|
1104
|
-
* @typedef {Object} WorkerPoolOptions
|
|
1105
|
-
* @property {number} [workerCount=4] - Количество workers в pool
|
|
1106
|
-
* @property {number} [maxQueueSize=100] - Максимальный размер очереди задач
|
|
1107
|
-
* @property {boolean} [autoScale=true] - Автоматическое масштабирование pool
|
|
1108
|
-
* @property {number} [idleTimeout=60000] - Таймаут простоя worker (мс)
|
|
1719
|
+
/**
|
|
1720
|
+
* Опции для Worker Pool
|
|
1721
|
+
* @typedef {Object} WorkerPoolOptions
|
|
1722
|
+
* @property {number} [workerCount=4] - Количество workers в pool
|
|
1723
|
+
* @property {number} [maxQueueSize=100] - Максимальный размер очереди задач
|
|
1724
|
+
* @property {boolean} [autoScale=true] - Автоматическое масштабирование pool
|
|
1725
|
+
* @property {number} [idleTimeout=60000] - Таймаут простоя worker (мс)
|
|
1109
1726
|
*/
|
|
1110
1727
|
|
|
1111
|
-
/**
|
|
1112
|
-
* Статистика Worker Pool
|
|
1113
|
-
* @typedef {Object} WorkerPoolStats
|
|
1114
|
-
* @property {number} totalWorkers - Всего workers
|
|
1115
|
-
* @property {number} activeWorkers - Активные workers
|
|
1116
|
-
* @property {number} idleWorkers - Простаивающие workers
|
|
1117
|
-
* @property {number} queueSize - Размер очереди
|
|
1118
|
-
* @property {number} tasksCompleted - Завершенные задачи
|
|
1119
|
-
* @property {number} tasksFailed - Неудачные задачи
|
|
1728
|
+
/**
|
|
1729
|
+
* Статистика Worker Pool
|
|
1730
|
+
* @typedef {Object} WorkerPoolStats
|
|
1731
|
+
* @property {number} totalWorkers - Всего workers
|
|
1732
|
+
* @property {number} activeWorkers - Активные workers
|
|
1733
|
+
* @property {number} idleWorkers - Простаивающие workers
|
|
1734
|
+
* @property {number} queueSize - Размер очереди
|
|
1735
|
+
* @property {number} tasksCompleted - Завершенные задачи
|
|
1736
|
+
* @property {number} tasksFailed - Неудачные задачи
|
|
1120
1737
|
*/
|
|
1121
1738
|
|
|
1122
|
-
/**
|
|
1123
|
-
* Прогресс обработки задачи
|
|
1124
|
-
* @typedef {Object} TaskProgress
|
|
1125
|
-
* @property {number} processed - Обработано элементов
|
|
1126
|
-
* @property {number} total - Всего элементов
|
|
1127
|
-
* @property {number} percentage - Процент выполнения
|
|
1128
|
-
* @property {number} speed - Скорость обработки (элементов/сек)
|
|
1739
|
+
/**
|
|
1740
|
+
* Прогресс обработки задачи
|
|
1741
|
+
* @typedef {Object} TaskProgress
|
|
1742
|
+
* @property {number} processed - Обработано элементов
|
|
1743
|
+
* @property {number} total - Всего элементов
|
|
1744
|
+
* @property {number} percentage - Процент выполнения
|
|
1745
|
+
* @property {number} speed - Скорость обработки (элементов/сек)
|
|
1129
1746
|
*/
|
|
1130
1747
|
|
|
1131
|
-
/**
|
|
1132
|
-
* Worker Pool для параллельной обработки CSV
|
|
1748
|
+
/**
|
|
1749
|
+
* Worker Pool для параллельной обработки CSV
|
|
1133
1750
|
*/
|
|
1134
1751
|
class WorkerPool {
|
|
1135
|
-
/**
|
|
1136
|
-
* Создает новый Worker Pool
|
|
1137
|
-
* @param {string} workerScript - URL скрипта worker
|
|
1138
|
-
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
1752
|
+
/**
|
|
1753
|
+
* Создает новый Worker Pool
|
|
1754
|
+
* @param {string} workerScript - URL скрипта worker
|
|
1755
|
+
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
1139
1756
|
*/
|
|
1140
1757
|
constructor(workerScript, options = {}) {
|
|
1141
1758
|
if (!WORKERS_SUPPORTED) {
|
|
@@ -1163,9 +1780,9 @@ class WorkerPool {
|
|
|
1163
1780
|
this.initializeWorkers();
|
|
1164
1781
|
}
|
|
1165
1782
|
|
|
1166
|
-
/**
|
|
1167
|
-
* Инициализация workers
|
|
1168
|
-
* @private
|
|
1783
|
+
/**
|
|
1784
|
+
* Инициализация workers
|
|
1785
|
+
* @private
|
|
1169
1786
|
*/
|
|
1170
1787
|
initializeWorkers() {
|
|
1171
1788
|
const {
|
|
@@ -1177,9 +1794,9 @@ class WorkerPool {
|
|
|
1177
1794
|
this.updateStats();
|
|
1178
1795
|
}
|
|
1179
1796
|
|
|
1180
|
-
/**
|
|
1181
|
-
* Создает нового worker
|
|
1182
|
-
* @private
|
|
1797
|
+
/**
|
|
1798
|
+
* Создает нового worker
|
|
1799
|
+
* @private
|
|
1183
1800
|
*/
|
|
1184
1801
|
createWorker() {
|
|
1185
1802
|
try {
|
|
@@ -1204,9 +1821,9 @@ class WorkerPool {
|
|
|
1204
1821
|
}
|
|
1205
1822
|
}
|
|
1206
1823
|
|
|
1207
|
-
/**
|
|
1208
|
-
* Обработка сообщений от worker
|
|
1209
|
-
* @private
|
|
1824
|
+
/**
|
|
1825
|
+
* Обработка сообщений от worker
|
|
1826
|
+
* @private
|
|
1210
1827
|
*/
|
|
1211
1828
|
handleWorkerMessage(worker, event) {
|
|
1212
1829
|
const {
|
|
@@ -1221,9 +1838,9 @@ class WorkerPool {
|
|
|
1221
1838
|
}
|
|
1222
1839
|
}
|
|
1223
1840
|
|
|
1224
|
-
/**
|
|
1225
|
-
* Обработка прогресса задачи
|
|
1226
|
-
* @private
|
|
1841
|
+
/**
|
|
1842
|
+
* Обработка прогресса задачи
|
|
1843
|
+
* @private
|
|
1227
1844
|
*/
|
|
1228
1845
|
handleProgress(worker, progressData) {
|
|
1229
1846
|
const taskId = worker.taskId;
|
|
@@ -1240,9 +1857,9 @@ class WorkerPool {
|
|
|
1240
1857
|
}
|
|
1241
1858
|
}
|
|
1242
1859
|
|
|
1243
|
-
/**
|
|
1244
|
-
* Обработка результата задачи
|
|
1245
|
-
* @private
|
|
1860
|
+
/**
|
|
1861
|
+
* Обработка результата задачи
|
|
1862
|
+
* @private
|
|
1246
1863
|
*/
|
|
1247
1864
|
handleResult(worker, resultData) {
|
|
1248
1865
|
const taskId = worker.taskId;
|
|
@@ -1267,9 +1884,9 @@ class WorkerPool {
|
|
|
1267
1884
|
}
|
|
1268
1885
|
}
|
|
1269
1886
|
|
|
1270
|
-
/**
|
|
1271
|
-
* Обработка ошибки задачи
|
|
1272
|
-
* @private
|
|
1887
|
+
/**
|
|
1888
|
+
* Обработка ошибки задачи
|
|
1889
|
+
* @private
|
|
1273
1890
|
*/
|
|
1274
1891
|
handleWorkerTaskError(worker, errorData) {
|
|
1275
1892
|
const taskId = worker.taskId;
|
|
@@ -1284,7 +1901,14 @@ class WorkerPool {
|
|
|
1284
1901
|
this.stats.idleWorkers++;
|
|
1285
1902
|
|
|
1286
1903
|
// Завершение с ошибкой
|
|
1287
|
-
|
|
1904
|
+
const workerError = new Error(errorData.message || 'Ошибка в worker');
|
|
1905
|
+
if (errorData.code) {
|
|
1906
|
+
workerError.code = errorData.code;
|
|
1907
|
+
}
|
|
1908
|
+
if (errorData.details) {
|
|
1909
|
+
workerError.details = errorData.details;
|
|
1910
|
+
}
|
|
1911
|
+
task.reject(workerError);
|
|
1288
1912
|
this.activeTasks.delete(taskId);
|
|
1289
1913
|
this.stats.tasksFailed++;
|
|
1290
1914
|
|
|
@@ -1294,9 +1918,9 @@ class WorkerPool {
|
|
|
1294
1918
|
}
|
|
1295
1919
|
}
|
|
1296
1920
|
|
|
1297
|
-
/**
|
|
1298
|
-
* Обработка ошибок worker
|
|
1299
|
-
* @private
|
|
1921
|
+
/**
|
|
1922
|
+
* Обработка ошибок worker
|
|
1923
|
+
* @private
|
|
1300
1924
|
*/
|
|
1301
1925
|
handleWorkerError(worker, error) {
|
|
1302
1926
|
console.error(`Worker ${worker.id} error:`, error);
|
|
@@ -1305,17 +1929,17 @@ class WorkerPool {
|
|
|
1305
1929
|
this.restartWorker(worker);
|
|
1306
1930
|
}
|
|
1307
1931
|
|
|
1308
|
-
/**
|
|
1309
|
-
* Обработка ошибок сообщений
|
|
1310
|
-
* @private
|
|
1932
|
+
/**
|
|
1933
|
+
* Обработка ошибок сообщений
|
|
1934
|
+
* @private
|
|
1311
1935
|
*/
|
|
1312
1936
|
handleWorkerMessageError(worker, error) {
|
|
1313
1937
|
console.error(`Worker ${worker.id} message error:`, error);
|
|
1314
1938
|
}
|
|
1315
1939
|
|
|
1316
|
-
/**
|
|
1317
|
-
* Перезапуск worker
|
|
1318
|
-
* @private
|
|
1940
|
+
/**
|
|
1941
|
+
* Перезапуск worker
|
|
1942
|
+
* @private
|
|
1319
1943
|
*/
|
|
1320
1944
|
restartWorker(worker) {
|
|
1321
1945
|
const index = this.workers.indexOf(worker);
|
|
@@ -1343,9 +1967,9 @@ class WorkerPool {
|
|
|
1343
1967
|
}
|
|
1344
1968
|
}
|
|
1345
1969
|
|
|
1346
|
-
/**
|
|
1347
|
-
* Выполнение задачи на worker
|
|
1348
|
-
* @private
|
|
1970
|
+
/**
|
|
1971
|
+
* Выполнение задачи на worker
|
|
1972
|
+
* @private
|
|
1349
1973
|
*/
|
|
1350
1974
|
executeTask(worker, task) {
|
|
1351
1975
|
worker.status = 'active';
|
|
@@ -1355,59 +1979,59 @@ class WorkerPool {
|
|
|
1355
1979
|
this.stats.activeWorkers++;
|
|
1356
1980
|
|
|
1357
1981
|
// Отправка задачи в worker
|
|
1358
|
-
|
|
1982
|
+
const payload = {
|
|
1359
1983
|
type: 'EXECUTE',
|
|
1360
1984
|
taskId: task.id,
|
|
1361
1985
|
method: task.method,
|
|
1362
1986
|
args: task.args,
|
|
1363
1987
|
options: task.options
|
|
1364
|
-
}
|
|
1988
|
+
};
|
|
1989
|
+
if (task.transferList && task.transferList.length) {
|
|
1990
|
+
worker.postMessage(payload, task.transferList);
|
|
1991
|
+
} else {
|
|
1992
|
+
worker.postMessage(payload);
|
|
1993
|
+
}
|
|
1365
1994
|
}
|
|
1366
1995
|
|
|
1367
|
-
/**
|
|
1368
|
-
* Обработка очереди задач
|
|
1369
|
-
* @private
|
|
1996
|
+
/**
|
|
1997
|
+
* Обработка очереди задач
|
|
1998
|
+
* @private
|
|
1370
1999
|
*/
|
|
1371
2000
|
processQueue() {
|
|
1372
2001
|
if (this.taskQueue.length === 0) {
|
|
1373
2002
|
return;
|
|
1374
2003
|
}
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
2004
|
+
while (this.taskQueue.length > 0) {
|
|
2005
|
+
const idleWorker = this.workers.find(w => w.status === 'idle');
|
|
2006
|
+
if (!idleWorker) {
|
|
2007
|
+
if (this.options.autoScale && this.workers.length < this.options.maxQueueSize) {
|
|
2008
|
+
this.createWorker();
|
|
2009
|
+
continue;
|
|
2010
|
+
}
|
|
2011
|
+
break;
|
|
1383
2012
|
}
|
|
1384
|
-
|
|
2013
|
+
const task = this.taskQueue.shift();
|
|
2014
|
+
this.stats.queueSize--;
|
|
2015
|
+
this.executeTask(idleWorker, task);
|
|
1385
2016
|
}
|
|
1386
|
-
|
|
1387
|
-
// Получение задачи из очереди
|
|
1388
|
-
const task = this.taskQueue.shift();
|
|
1389
|
-
this.stats.queueSize--;
|
|
1390
|
-
|
|
1391
|
-
// Выполнение задачи
|
|
1392
|
-
this.executeTask(idleWorker, task);
|
|
1393
2017
|
this.updateStats();
|
|
1394
2018
|
}
|
|
1395
2019
|
|
|
1396
|
-
/**
|
|
1397
|
-
* Обновление статистики
|
|
1398
|
-
* @private
|
|
2020
|
+
/**
|
|
2021
|
+
* Обновление статистики
|
|
2022
|
+
* @private
|
|
1399
2023
|
*/
|
|
1400
2024
|
updateStats() {
|
|
1401
2025
|
this.stats.queueSize = this.taskQueue.length;
|
|
1402
2026
|
}
|
|
1403
2027
|
|
|
1404
|
-
/**
|
|
1405
|
-
* Выполнение задачи через pool
|
|
1406
|
-
* @param {string} method - Метод для вызова в worker
|
|
1407
|
-
* @param {Array} args - Аргументы метода
|
|
1408
|
-
* @param {Object} [options] - Опции задачи
|
|
1409
|
-
* @param {Function} [onProgress] - Callback прогресса
|
|
1410
|
-
* @returns {Promise<any>} Результат выполнения
|
|
2028
|
+
/**
|
|
2029
|
+
* Выполнение задачи через pool
|
|
2030
|
+
* @param {string} method - Метод для вызова в worker
|
|
2031
|
+
* @param {Array} args - Аргументы метода
|
|
2032
|
+
* @param {Object} [options] - Опции задачи
|
|
2033
|
+
* @param {Function} [onProgress] - Callback прогресса
|
|
2034
|
+
* @returns {Promise<any>} Результат выполнения
|
|
1411
2035
|
*/
|
|
1412
2036
|
async exec(method, args = [], options = {}, onProgress = null) {
|
|
1413
2037
|
return new Promise((resolve, reject) => {
|
|
@@ -1419,11 +2043,17 @@ class WorkerPool {
|
|
|
1419
2043
|
|
|
1420
2044
|
// Создание задачи
|
|
1421
2045
|
const taskId = `task-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
2046
|
+
const {
|
|
2047
|
+
transfer,
|
|
2048
|
+
...taskOptions
|
|
2049
|
+
} = options || {};
|
|
2050
|
+
const transferList = transfer || collectTransferables(args);
|
|
1422
2051
|
const task = {
|
|
1423
2052
|
id: taskId,
|
|
1424
2053
|
method,
|
|
1425
2054
|
args,
|
|
1426
|
-
options,
|
|
2055
|
+
options: taskOptions,
|
|
2056
|
+
transferList,
|
|
1427
2057
|
onProgress,
|
|
1428
2058
|
resolve,
|
|
1429
2059
|
reject,
|
|
@@ -1440,9 +2070,9 @@ class WorkerPool {
|
|
|
1440
2070
|
});
|
|
1441
2071
|
}
|
|
1442
2072
|
|
|
1443
|
-
/**
|
|
1444
|
-
* Получение статистики pool
|
|
1445
|
-
* @returns {WorkerPoolStats} Статистика
|
|
2073
|
+
/**
|
|
2074
|
+
* Получение статистики pool
|
|
2075
|
+
* @returns {WorkerPoolStats} Статистика
|
|
1446
2076
|
*/
|
|
1447
2077
|
getStats() {
|
|
1448
2078
|
return {
|
|
@@ -1450,8 +2080,8 @@ class WorkerPool {
|
|
|
1450
2080
|
};
|
|
1451
2081
|
}
|
|
1452
2082
|
|
|
1453
|
-
/**
|
|
1454
|
-
* Очистка простаивающих workers
|
|
2083
|
+
/**
|
|
2084
|
+
* Очистка простаивающих workers
|
|
1455
2085
|
*/
|
|
1456
2086
|
cleanupIdleWorkers() {
|
|
1457
2087
|
const now = Date.now();
|
|
@@ -1472,8 +2102,8 @@ class WorkerPool {
|
|
|
1472
2102
|
}
|
|
1473
2103
|
}
|
|
1474
2104
|
|
|
1475
|
-
/**
|
|
1476
|
-
* Завершение всех workers
|
|
2105
|
+
/**
|
|
2106
|
+
* Завершение всех workers
|
|
1477
2107
|
*/
|
|
1478
2108
|
terminate() {
|
|
1479
2109
|
this.workers.forEach(worker => {
|
|
@@ -1495,10 +2125,10 @@ class WorkerPool {
|
|
|
1495
2125
|
}
|
|
1496
2126
|
}
|
|
1497
2127
|
|
|
1498
|
-
/**
|
|
1499
|
-
* Создает Worker Pool для обработки CSV
|
|
1500
|
-
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
1501
|
-
* @returns {WorkerPool} Worker Pool
|
|
2128
|
+
/**
|
|
2129
|
+
* Создает Worker Pool для обработки CSV
|
|
2130
|
+
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
2131
|
+
* @returns {WorkerPool} Worker Pool
|
|
1502
2132
|
*/
|
|
1503
2133
|
function createWorkerPool(options = {}) {
|
|
1504
2134
|
// Используем встроенный worker скрипт
|
|
@@ -1506,12 +2136,12 @@ function createWorkerPool(options = {}) {
|
|
|
1506
2136
|
return new WorkerPool(workerScript, options);
|
|
1507
2137
|
}
|
|
1508
2138
|
|
|
1509
|
-
/**
|
|
1510
|
-
* Парсит CSV с использованием Web Workers
|
|
1511
|
-
* @param {string|File} csvInput - CSV строка или File объект
|
|
1512
|
-
* @param {Object} [options] - Опции парсинга
|
|
1513
|
-
* @param {Function} [onProgress] - Callback прогресса
|
|
1514
|
-
* @returns {Promise<Array<Object>>} JSON данные
|
|
2139
|
+
/**
|
|
2140
|
+
* Парсит CSV с использованием Web Workers
|
|
2141
|
+
* @param {string|File} csvInput - CSV строка или File объект
|
|
2142
|
+
* @param {Object} [options] - Опции парсинга
|
|
2143
|
+
* @param {Function} [onProgress] - Callback прогресса
|
|
2144
|
+
* @returns {Promise<Array<Object>>} JSON данные
|
|
1515
2145
|
*/
|
|
1516
2146
|
async function parseCSVWithWorker(csvInput, options = {}, onProgress = null) {
|
|
1517
2147
|
// Создание pool если нужно
|
|
@@ -1521,29 +2151,42 @@ async function parseCSVWithWorker(csvInput, options = {}, onProgress = null) {
|
|
|
1521
2151
|
const pool = parseCSVWithWorker.pool;
|
|
1522
2152
|
|
|
1523
2153
|
// Подготовка CSV строки
|
|
1524
|
-
|
|
2154
|
+
// ?????????? CSV ??????
|
|
2155
|
+
let csvPayload = csvInput;
|
|
2156
|
+
let transfer = null;
|
|
1525
2157
|
if (csvInput instanceof File) {
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
} else {
|
|
1530
|
-
|
|
2158
|
+
const buffer = await readFileAsArrayBuffer(csvInput);
|
|
2159
|
+
csvPayload = new Uint8Array(buffer);
|
|
2160
|
+
transfer = [buffer];
|
|
2161
|
+
} else if (csvInput instanceof ArrayBuffer) {
|
|
2162
|
+
csvPayload = csvInput;
|
|
2163
|
+
transfer = [csvInput];
|
|
2164
|
+
} else if (ArrayBuffer.isView(csvInput)) {
|
|
2165
|
+
csvPayload = csvInput;
|
|
2166
|
+
if (csvInput.buffer instanceof ArrayBuffer) {
|
|
2167
|
+
transfer = [csvInput.buffer];
|
|
2168
|
+
}
|
|
2169
|
+
} else if (typeof csvInput !== 'string') {
|
|
2170
|
+
throw new ValidationError('Input must be a CSV string, File, or ArrayBuffer');
|
|
1531
2171
|
}
|
|
1532
2172
|
|
|
1533
|
-
//
|
|
1534
|
-
|
|
2173
|
+
// ????????? ?????? ????? pool
|
|
2174
|
+
const execOptions = transfer ? {
|
|
2175
|
+
transfer
|
|
2176
|
+
} : {};
|
|
2177
|
+
return pool.exec('parseCSV', [csvPayload, options], execOptions, onProgress);
|
|
1535
2178
|
}
|
|
1536
2179
|
|
|
1537
|
-
/**
|
|
1538
|
-
* Чтение файла как текст
|
|
1539
|
-
* @private
|
|
2180
|
+
/**
|
|
2181
|
+
* Чтение файла как текст
|
|
2182
|
+
* @private
|
|
1540
2183
|
*/
|
|
1541
|
-
async function
|
|
2184
|
+
async function readFileAsArrayBuffer(file) {
|
|
1542
2185
|
return new Promise((resolve, reject) => {
|
|
1543
2186
|
const reader = new FileReader();
|
|
1544
2187
|
reader.onload = event => resolve(event.target.result);
|
|
1545
2188
|
reader.onerror = error => reject(error);
|
|
1546
|
-
reader.
|
|
2189
|
+
reader.readAsArrayBuffer(file);
|
|
1547
2190
|
});
|
|
1548
2191
|
}
|
|
1549
2192
|
|
|
@@ -1556,9 +2199,24 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
1556
2199
|
};
|
|
1557
2200
|
}
|
|
1558
2201
|
|
|
2202
|
+
var workerPool = /*#__PURE__*/Object.freeze({
|
|
2203
|
+
__proto__: null,
|
|
2204
|
+
WorkerPool: WorkerPool,
|
|
2205
|
+
createWorkerPool: createWorkerPool,
|
|
2206
|
+
parseCSVWithWorker: parseCSVWithWorker
|
|
2207
|
+
});
|
|
2208
|
+
|
|
1559
2209
|
// Браузерный entry point для jtcsv
|
|
1560
2210
|
// Экспортирует все функции с поддержкой браузера
|
|
1561
2211
|
|
|
2212
|
+
async function createWorkerPoolLazy(options = {}) {
|
|
2213
|
+
const mod = await Promise.resolve().then(function () { return workerPool; });
|
|
2214
|
+
return mod.createWorkerPool(options);
|
|
2215
|
+
}
|
|
2216
|
+
async function parseCSVWithWorkerLazy(csvInput, options = {}, onProgress = null) {
|
|
2217
|
+
const mod = await Promise.resolve().then(function () { return workerPool; });
|
|
2218
|
+
return mod.parseCSVWithWorker(csvInput, options, onProgress);
|
|
2219
|
+
}
|
|
1562
2220
|
|
|
1563
2221
|
// Основной экспорт
|
|
1564
2222
|
const jtcsv = {
|
|
@@ -1569,11 +2227,18 @@ const jtcsv = {
|
|
|
1569
2227
|
deepUnwrap,
|
|
1570
2228
|
// CSV to JSON функции
|
|
1571
2229
|
csvToJson,
|
|
2230
|
+
csvToJsonIterator,
|
|
1572
2231
|
parseCsvFile,
|
|
2232
|
+
parseCsvFileStream,
|
|
2233
|
+
jsonToCsvStream,
|
|
2234
|
+
jsonToNdjsonStream,
|
|
2235
|
+
csvToJsonStream,
|
|
1573
2236
|
autoDetectDelimiter,
|
|
1574
2237
|
// Web Workers функции
|
|
1575
2238
|
createWorkerPool,
|
|
1576
2239
|
parseCSVWithWorker,
|
|
2240
|
+
createWorkerPoolLazy,
|
|
2241
|
+
parseCSVWithWorkerLazy,
|
|
1577
2242
|
// Error classes
|
|
1578
2243
|
ValidationError,
|
|
1579
2244
|
SecurityError,
|
|
@@ -1581,6 +2246,7 @@ const jtcsv = {
|
|
|
1581
2246
|
ParsingError,
|
|
1582
2247
|
LimitError,
|
|
1583
2248
|
ConfigurationError,
|
|
2249
|
+
ERROR_CODES,
|
|
1584
2250
|
// Удобные алиасы
|
|
1585
2251
|
parse: csvToJson,
|
|
1586
2252
|
unparse: jsonToCsv,
|
|
@@ -1601,6 +2267,7 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
1601
2267
|
}
|
|
1602
2268
|
|
|
1603
2269
|
exports.ConfigurationError = ConfigurationError;
|
|
2270
|
+
exports.ERROR_CODES = ERROR_CODES;
|
|
1604
2271
|
exports.FileSystemError = FileSystemError;
|
|
1605
2272
|
exports.LimitError = LimitError;
|
|
1606
2273
|
exports.ParsingError = ParsingError;
|
|
@@ -1608,12 +2275,19 @@ exports.SecurityError = SecurityError;
|
|
|
1608
2275
|
exports.ValidationError = ValidationError;
|
|
1609
2276
|
exports.autoDetectDelimiter = autoDetectDelimiter;
|
|
1610
2277
|
exports.createWorkerPool = createWorkerPool;
|
|
2278
|
+
exports.createWorkerPoolLazy = createWorkerPoolLazy;
|
|
1611
2279
|
exports.csvToJson = csvToJson;
|
|
2280
|
+
exports.csvToJsonIterator = csvToJsonIterator;
|
|
2281
|
+
exports.csvToJsonStream = csvToJsonStream;
|
|
1612
2282
|
exports.deepUnwrap = deepUnwrap;
|
|
1613
2283
|
exports.default = jtcsv;
|
|
1614
2284
|
exports.downloadAsCsv = downloadAsCsv;
|
|
1615
2285
|
exports.jsonToCsv = jsonToCsv;
|
|
2286
|
+
exports.jsonToCsvStream = jsonToCsvStream;
|
|
2287
|
+
exports.jsonToNdjsonStream = jsonToNdjsonStream;
|
|
1616
2288
|
exports.parseCSVWithWorker = parseCSVWithWorker;
|
|
2289
|
+
exports.parseCSVWithWorkerLazy = parseCSVWithWorkerLazy;
|
|
1617
2290
|
exports.parseCsvFile = parseCsvFile;
|
|
2291
|
+
exports.parseCsvFileStream = parseCsvFileStream;
|
|
1618
2292
|
exports.preprocessData = preprocessData;
|
|
1619
2293
|
//# sourceMappingURL=jtcsv.cjs.js.map
|