jtcsv 2.1.3 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +60 -341
- package/bin/jtcsv.js +2462 -1372
- package/csv-to-json.js +35 -26
- package/dist/jtcsv.cjs.js +807 -133
- package/dist/jtcsv.cjs.js.map +1 -1
- package/dist/jtcsv.esm.js +800 -134
- package/dist/jtcsv.esm.js.map +1 -1
- package/dist/jtcsv.umd.js +807 -133
- package/dist/jtcsv.umd.js.map +1 -1
- package/errors.js +20 -0
- package/examples/browser-vanilla.html +37 -0
- package/examples/cli-batch-processing.js +38 -0
- package/examples/error-handling.js +324 -0
- package/examples/ndjson-processing.js +434 -0
- package/examples/react-integration.jsx +637 -0
- package/examples/schema-validation.js +640 -0
- package/examples/simple-usage.js +10 -7
- package/examples/typescript-example.ts +486 -0
- package/examples/web-workers-advanced.js +28 -0
- package/index.d.ts +2 -0
- package/json-save.js +2 -1
- package/json-to-csv.js +171 -131
- package/package.json +20 -4
- package/plugins/README.md +41 -467
- package/plugins/express-middleware/README.md +32 -274
- package/plugins/hono/README.md +16 -13
- package/plugins/nestjs/README.md +13 -11
- package/plugins/nextjs-api/README.md +28 -423
- package/plugins/nextjs-api/index.js +1 -2
- package/plugins/nextjs-api/route.js +1 -2
- package/plugins/nuxt/README.md +6 -7
- package/plugins/remix/README.md +9 -9
- package/plugins/sveltekit/README.md +8 -8
- package/plugins/trpc/README.md +8 -5
- package/src/browser/browser-functions.js +33 -3
- package/src/browser/csv-to-json-browser.js +269 -11
- package/src/browser/errors-browser.js +19 -1
- package/src/browser/index.js +39 -5
- package/src/browser/streams.js +393 -0
- package/src/browser/workers/csv-parser.worker.js +20 -2
- package/src/browser/workers/worker-pool.js +507 -447
- package/src/core/plugin-system.js +4 -0
- package/src/engines/fast-path-engine.js +31 -23
- package/src/errors.js +26 -0
- package/src/formats/ndjson-parser.js +54 -5
- package/src/formats/tsv-parser.js +4 -1
- package/src/utils/schema-validator.js +594 -0
- package/src/utils/transform-loader.js +205 -0
- package/src/web-server/index.js +683 -0
- package/stream-csv-to-json.js +16 -87
- package/stream-json-to-csv.js +18 -86
package/dist/jtcsv.esm.js
CHANGED
|
@@ -93,6 +93,22 @@ class ConfigurationError extends JTCSVError {
|
|
|
93
93
|
this.name = 'ConfigurationError';
|
|
94
94
|
}
|
|
95
95
|
}
|
|
96
|
+
const ERROR_CODES = {
|
|
97
|
+
JTCSV_ERROR: 'JTCSV_ERROR',
|
|
98
|
+
VALIDATION_ERROR: 'VALIDATION_ERROR',
|
|
99
|
+
SECURITY_ERROR: 'SECURITY_ERROR',
|
|
100
|
+
FILE_SYSTEM_ERROR: 'FILE_SYSTEM_ERROR',
|
|
101
|
+
PARSING_ERROR: 'PARSING_ERROR',
|
|
102
|
+
LIMIT_ERROR: 'LIMIT_ERROR',
|
|
103
|
+
CONFIGURATION_ERROR: 'CONFIGURATION_ERROR',
|
|
104
|
+
INVALID_INPUT: 'INVALID_INPUT',
|
|
105
|
+
SECURITY_VIOLATION: 'SECURITY_VIOLATION',
|
|
106
|
+
FILE_NOT_FOUND: 'FILE_NOT_FOUND',
|
|
107
|
+
PARSE_FAILED: 'PARSE_FAILED',
|
|
108
|
+
SIZE_LIMIT: 'SIZE_LIMIT',
|
|
109
|
+
INVALID_CONFIG: 'INVALID_CONFIG',
|
|
110
|
+
UNKNOWN_ERROR: 'UNKNOWN_ERROR'
|
|
111
|
+
};
|
|
96
112
|
|
|
97
113
|
/**
|
|
98
114
|
* Безопасное выполнение функции с обработкой ошибок
|
|
@@ -228,6 +244,7 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
228
244
|
ParsingError,
|
|
229
245
|
LimitError,
|
|
230
246
|
ConfigurationError,
|
|
247
|
+
ERROR_CODES,
|
|
231
248
|
safeExecute,
|
|
232
249
|
safeExecuteAsync
|
|
233
250
|
};
|
|
@@ -511,15 +528,10 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
511
528
|
|
|
512
529
|
|
|
513
530
|
/**
|
|
514
|
-
* Валидация
|
|
531
|
+
* Валидация опций парсинга
|
|
515
532
|
* @private
|
|
516
533
|
*/
|
|
517
|
-
function
|
|
518
|
-
// Validate CSV input
|
|
519
|
-
if (typeof csv !== 'string') {
|
|
520
|
-
throw new ValidationError('Input must be a CSV string');
|
|
521
|
-
}
|
|
522
|
-
|
|
534
|
+
function validateCsvOptions(options) {
|
|
523
535
|
// Validate options
|
|
524
536
|
if (options && typeof options !== 'object') {
|
|
525
537
|
throw new ConfigurationError('Options must be an object');
|
|
@@ -547,9 +559,24 @@ function validateCsvInput(csv, options) {
|
|
|
547
559
|
if (options?.maxRows !== undefined && (typeof options.maxRows !== 'number' || options.maxRows <= 0)) {
|
|
548
560
|
throw new ConfigurationError('maxRows must be a positive number');
|
|
549
561
|
}
|
|
562
|
+
if (options?.warnExtraFields !== undefined && typeof options.warnExtraFields !== 'boolean') {
|
|
563
|
+
throw new ConfigurationError('warnExtraFields must be a boolean');
|
|
564
|
+
}
|
|
550
565
|
return true;
|
|
551
566
|
}
|
|
552
567
|
|
|
568
|
+
/**
|
|
569
|
+
* Валидация CSV ввода и опций
|
|
570
|
+
* @private
|
|
571
|
+
*/
|
|
572
|
+
function validateCsvInput(csv, options) {
|
|
573
|
+
// Validate CSV input
|
|
574
|
+
if (typeof csv !== 'string') {
|
|
575
|
+
throw new ValidationError('Input must be a CSV string');
|
|
576
|
+
}
|
|
577
|
+
return validateCsvOptions(options);
|
|
578
|
+
}
|
|
579
|
+
|
|
553
580
|
/**
|
|
554
581
|
* Парсинг одной строки CSV с правильным экранированием
|
|
555
582
|
* @private
|
|
@@ -695,6 +722,78 @@ function parseCsvValue(value, options) {
|
|
|
695
722
|
}
|
|
696
723
|
return result;
|
|
697
724
|
}
|
|
725
|
+
function isSimpleCsv(csv) {
|
|
726
|
+
return csv.indexOf('"') === -1 && csv.indexOf('\\') === -1;
|
|
727
|
+
}
|
|
728
|
+
function parseSimpleCsv(csv, delimiter, options) {
|
|
729
|
+
const {
|
|
730
|
+
hasHeaders = true,
|
|
731
|
+
renameMap = {},
|
|
732
|
+
trim = true,
|
|
733
|
+
parseNumbers = false,
|
|
734
|
+
parseBooleans = false,
|
|
735
|
+
maxRows
|
|
736
|
+
} = options;
|
|
737
|
+
const result = [];
|
|
738
|
+
let headers = null;
|
|
739
|
+
let fieldStart = 0;
|
|
740
|
+
let currentRow = [];
|
|
741
|
+
let rowHasData = false;
|
|
742
|
+
let rowCount = 0;
|
|
743
|
+
const finalizeRow = fields => {
|
|
744
|
+
if (fields.length === 1 && fields[0].trim() === '') {
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
if (!headers) {
|
|
748
|
+
if (hasHeaders) {
|
|
749
|
+
headers = fields.map(header => {
|
|
750
|
+
const trimmed = trim ? header.trim() : header;
|
|
751
|
+
return renameMap[trimmed] || trimmed;
|
|
752
|
+
});
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
headers = fields.map((_, index) => `column${index + 1}`);
|
|
756
|
+
}
|
|
757
|
+
rowCount++;
|
|
758
|
+
if (maxRows && rowCount > maxRows) {
|
|
759
|
+
throw new LimitError(`CSV size exceeds maximum limit of ${maxRows} rows`, maxRows, rowCount);
|
|
760
|
+
}
|
|
761
|
+
const row = {};
|
|
762
|
+
const fieldCount = Math.min(fields.length, headers.length);
|
|
763
|
+
for (let i = 0; i < fieldCount; i++) {
|
|
764
|
+
row[headers[i]] = parseCsvValue(fields[i], {
|
|
765
|
+
trim,
|
|
766
|
+
parseNumbers,
|
|
767
|
+
parseBooleans
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
result.push(row);
|
|
771
|
+
};
|
|
772
|
+
let i = 0;
|
|
773
|
+
while (i <= csv.length) {
|
|
774
|
+
const char = i < csv.length ? csv[i] : '\n';
|
|
775
|
+
if (char !== '\r' && char !== '\n' && char !== ' ' && char !== '\t') {
|
|
776
|
+
rowHasData = true;
|
|
777
|
+
}
|
|
778
|
+
if (char === delimiter || char === '\n' || char === '\r' || i === csv.length) {
|
|
779
|
+
const field = csv.slice(fieldStart, i);
|
|
780
|
+
currentRow.push(field);
|
|
781
|
+
if (char === '\n' || char === '\r' || i === csv.length) {
|
|
782
|
+
if (rowHasData || currentRow.length > 1) {
|
|
783
|
+
finalizeRow(currentRow);
|
|
784
|
+
}
|
|
785
|
+
currentRow = [];
|
|
786
|
+
rowHasData = false;
|
|
787
|
+
}
|
|
788
|
+
if (char === '\r' && csv[i + 1] === '\n') {
|
|
789
|
+
i++;
|
|
790
|
+
}
|
|
791
|
+
fieldStart = i + 1;
|
|
792
|
+
}
|
|
793
|
+
i++;
|
|
794
|
+
}
|
|
795
|
+
return result;
|
|
796
|
+
}
|
|
698
797
|
|
|
699
798
|
/**
|
|
700
799
|
* Автоматическое определение разделителя CSV
|
|
@@ -770,7 +869,8 @@ function csvToJson(csv, options = {}) {
|
|
|
770
869
|
trim = true,
|
|
771
870
|
parseNumbers = false,
|
|
772
871
|
parseBooleans = false,
|
|
773
|
-
maxRows
|
|
872
|
+
maxRows,
|
|
873
|
+
warnExtraFields = true
|
|
774
874
|
} = opts;
|
|
775
875
|
|
|
776
876
|
// Определение разделителя
|
|
@@ -784,6 +884,16 @@ function csvToJson(csv, options = {}) {
|
|
|
784
884
|
if (csv.trim() === '') {
|
|
785
885
|
return [];
|
|
786
886
|
}
|
|
887
|
+
if (isSimpleCsv(csv)) {
|
|
888
|
+
return parseSimpleCsv(csv, finalDelimiter, {
|
|
889
|
+
hasHeaders,
|
|
890
|
+
renameMap,
|
|
891
|
+
trim,
|
|
892
|
+
parseNumbers,
|
|
893
|
+
parseBooleans,
|
|
894
|
+
maxRows
|
|
895
|
+
});
|
|
896
|
+
}
|
|
787
897
|
|
|
788
898
|
// Парсинг CSV с обработкой кавычек и переносов строк
|
|
789
899
|
const lines = [];
|
|
@@ -887,7 +997,8 @@ function csvToJson(csv, options = {}) {
|
|
|
887
997
|
}
|
|
888
998
|
|
|
889
999
|
// Предупреждение о лишних полях
|
|
890
|
-
|
|
1000
|
+
const isDev = typeof process !== 'undefined' && process.env && process.env.NODE_ENV === 'development';
|
|
1001
|
+
if (fields.length > headers.length && warnExtraFields && isDev) {
|
|
891
1002
|
console.warn(`[jtcsv] Line ${i + 1}: ${fields.length - headers.length} extra fields ignored`);
|
|
892
1003
|
}
|
|
893
1004
|
result.push(row);
|
|
@@ -903,12 +1014,464 @@ function csvToJson(csv, options = {}) {
|
|
|
903
1014
|
function: 'csvToJson'
|
|
904
1015
|
});
|
|
905
1016
|
}
|
|
1017
|
+
async function* csvToJsonIterator(input, options = {}) {
|
|
1018
|
+
const opts = options && typeof options === 'object' ? options : {};
|
|
1019
|
+
validateCsvOptions(opts);
|
|
1020
|
+
if (typeof input === 'string') {
|
|
1021
|
+
const rows = csvToJson(input, options);
|
|
1022
|
+
for (const row of rows) {
|
|
1023
|
+
yield row;
|
|
1024
|
+
}
|
|
1025
|
+
return;
|
|
1026
|
+
}
|
|
1027
|
+
const {
|
|
1028
|
+
delimiter,
|
|
1029
|
+
autoDetect = true,
|
|
1030
|
+
candidates = [';', ',', '\t', '|'],
|
|
1031
|
+
hasHeaders = true,
|
|
1032
|
+
renameMap = {},
|
|
1033
|
+
trim = true,
|
|
1034
|
+
parseNumbers = false,
|
|
1035
|
+
parseBooleans = false,
|
|
1036
|
+
maxRows
|
|
1037
|
+
} = opts;
|
|
1038
|
+
const stream = input instanceof Blob && input.stream ? input.stream() : input;
|
|
1039
|
+
if (!stream || typeof stream.getReader !== 'function') {
|
|
1040
|
+
throw new ValidationError('Input must be a CSV string, Blob/File, or ReadableStream');
|
|
1041
|
+
}
|
|
1042
|
+
const reader = stream.getReader();
|
|
1043
|
+
const decoder = new TextDecoder('utf-8');
|
|
1044
|
+
let buffer = '';
|
|
1045
|
+
let insideQuotes = false;
|
|
1046
|
+
let headers = null;
|
|
1047
|
+
let rowCount = 0;
|
|
1048
|
+
let lineNumber = 0;
|
|
1049
|
+
let finalDelimiter = delimiter;
|
|
1050
|
+
let delimiterResolved = Boolean(finalDelimiter);
|
|
1051
|
+
const processFields = fields => {
|
|
1052
|
+
if (fields.length === 1 && fields[0].trim() === '') {
|
|
1053
|
+
return null;
|
|
1054
|
+
}
|
|
1055
|
+
rowCount++;
|
|
1056
|
+
if (maxRows && rowCount > maxRows) {
|
|
1057
|
+
throw new LimitError(`CSV size exceeds maximum limit of ${maxRows} rows`, maxRows, rowCount);
|
|
1058
|
+
}
|
|
1059
|
+
const row = {};
|
|
1060
|
+
const fieldCount = Math.min(fields.length, headers.length);
|
|
1061
|
+
for (let j = 0; j < fieldCount; j++) {
|
|
1062
|
+
row[headers[j]] = parseCsvValue(fields[j], {
|
|
1063
|
+
trim,
|
|
1064
|
+
parseNumbers,
|
|
1065
|
+
parseBooleans
|
|
1066
|
+
});
|
|
1067
|
+
}
|
|
1068
|
+
return row;
|
|
1069
|
+
};
|
|
1070
|
+
const processLine = line => {
|
|
1071
|
+
lineNumber++;
|
|
1072
|
+
let cleanLine = line;
|
|
1073
|
+
if (cleanLine.endsWith('\r')) {
|
|
1074
|
+
cleanLine = cleanLine.slice(0, -1);
|
|
1075
|
+
}
|
|
1076
|
+
if (!delimiterResolved) {
|
|
1077
|
+
if (!finalDelimiter && autoDetect) {
|
|
1078
|
+
finalDelimiter = autoDetectDelimiter(cleanLine, candidates);
|
|
1079
|
+
}
|
|
1080
|
+
finalDelimiter = finalDelimiter || ';';
|
|
1081
|
+
delimiterResolved = true;
|
|
1082
|
+
}
|
|
1083
|
+
if (cleanLine.trim() === '') {
|
|
1084
|
+
return null;
|
|
1085
|
+
}
|
|
1086
|
+
if (!headers) {
|
|
1087
|
+
if (hasHeaders) {
|
|
1088
|
+
headers = parseCsvLine(cleanLine, lineNumber, finalDelimiter).map(header => {
|
|
1089
|
+
const trimmed = trim ? header.trim() : header;
|
|
1090
|
+
return renameMap[trimmed] || trimmed;
|
|
1091
|
+
});
|
|
1092
|
+
return null;
|
|
1093
|
+
}
|
|
1094
|
+
const fields = parseCsvLine(cleanLine, lineNumber, finalDelimiter);
|
|
1095
|
+
headers = fields.map((_, index) => `column${index + 1}`);
|
|
1096
|
+
return processFields(fields);
|
|
1097
|
+
}
|
|
1098
|
+
const fields = parseCsvLine(cleanLine, lineNumber, finalDelimiter);
|
|
1099
|
+
return processFields(fields);
|
|
1100
|
+
};
|
|
1101
|
+
while (true) {
|
|
1102
|
+
const {
|
|
1103
|
+
value,
|
|
1104
|
+
done
|
|
1105
|
+
} = await reader.read();
|
|
1106
|
+
if (done) {
|
|
1107
|
+
break;
|
|
1108
|
+
}
|
|
1109
|
+
buffer += decoder.decode(value, {
|
|
1110
|
+
stream: true
|
|
1111
|
+
});
|
|
1112
|
+
let start = 0;
|
|
1113
|
+
for (let i = 0; i < buffer.length; i++) {
|
|
1114
|
+
const char = buffer[i];
|
|
1115
|
+
if (char === '"') {
|
|
1116
|
+
if (insideQuotes && buffer[i + 1] === '"') {
|
|
1117
|
+
i++;
|
|
1118
|
+
continue;
|
|
1119
|
+
}
|
|
1120
|
+
insideQuotes = !insideQuotes;
|
|
1121
|
+
continue;
|
|
1122
|
+
}
|
|
1123
|
+
if (char === '\n' && !insideQuotes) {
|
|
1124
|
+
const line = buffer.slice(start, i);
|
|
1125
|
+
start = i + 1;
|
|
1126
|
+
const row = processLine(line);
|
|
1127
|
+
if (row) {
|
|
1128
|
+
yield row;
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
buffer = buffer.slice(start);
|
|
1133
|
+
}
|
|
1134
|
+
if (buffer.length > 0) {
|
|
1135
|
+
const row = processLine(buffer);
|
|
1136
|
+
if (row) {
|
|
1137
|
+
yield row;
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
if (insideQuotes) {
|
|
1141
|
+
throw new ParsingError('Unclosed quotes in CSV', lineNumber);
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
906
1144
|
|
|
907
1145
|
// Экспорт для Node.js совместимости
|
|
908
1146
|
if (typeof module !== 'undefined' && module.exports) {
|
|
909
1147
|
module.exports = {
|
|
910
1148
|
csvToJson,
|
|
911
|
-
autoDetectDelimiter
|
|
1149
|
+
autoDetectDelimiter,
|
|
1150
|
+
csvToJsonIterator
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
const DEFAULT_MAX_CHUNK_SIZE = 64 * 1024;
|
|
1155
|
+
function isReadableStream(value) {
|
|
1156
|
+
return value && typeof value.getReader === 'function';
|
|
1157
|
+
}
|
|
1158
|
+
function isAsyncIterable(value) {
|
|
1159
|
+
return value && typeof value[Symbol.asyncIterator] === 'function';
|
|
1160
|
+
}
|
|
1161
|
+
function isIterable(value) {
|
|
1162
|
+
return value && typeof value[Symbol.iterator] === 'function';
|
|
1163
|
+
}
|
|
1164
|
+
function createReadableStreamFromIterator(iterator) {
|
|
1165
|
+
return new ReadableStream({
|
|
1166
|
+
async pull(controller) {
|
|
1167
|
+
try {
|
|
1168
|
+
const {
|
|
1169
|
+
value,
|
|
1170
|
+
done
|
|
1171
|
+
} = await iterator.next();
|
|
1172
|
+
if (done) {
|
|
1173
|
+
controller.close();
|
|
1174
|
+
return;
|
|
1175
|
+
}
|
|
1176
|
+
controller.enqueue(value);
|
|
1177
|
+
} catch (error) {
|
|
1178
|
+
controller.error(error);
|
|
1179
|
+
}
|
|
1180
|
+
},
|
|
1181
|
+
cancel() {
|
|
1182
|
+
if (iterator.return) {
|
|
1183
|
+
iterator.return();
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
});
|
|
1187
|
+
}
|
|
1188
|
+
function detectInputFormat(input, options) {
|
|
1189
|
+
if (options && options.inputFormat) {
|
|
1190
|
+
return options.inputFormat;
|
|
1191
|
+
}
|
|
1192
|
+
if (typeof input === 'string') {
|
|
1193
|
+
const trimmed = input.trim();
|
|
1194
|
+
if (trimmed.startsWith('[')) {
|
|
1195
|
+
return 'json-array';
|
|
1196
|
+
}
|
|
1197
|
+
if (trimmed.includes('\n')) {
|
|
1198
|
+
return 'ndjson';
|
|
1199
|
+
}
|
|
1200
|
+
return 'json-array';
|
|
1201
|
+
}
|
|
1202
|
+
if (input instanceof Blob || isReadableStream(input)) {
|
|
1203
|
+
return 'ndjson';
|
|
1204
|
+
}
|
|
1205
|
+
return 'json-array';
|
|
1206
|
+
}
|
|
1207
|
+
async function* parseNdjsonText(text) {
|
|
1208
|
+
const lines = text.split(/\r?\n/);
|
|
1209
|
+
for (const line of lines) {
|
|
1210
|
+
const trimmed = line.trim();
|
|
1211
|
+
if (!trimmed) {
|
|
1212
|
+
continue;
|
|
1213
|
+
}
|
|
1214
|
+
yield JSON.parse(trimmed);
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
async function* parseNdjsonStream(stream) {
|
|
1218
|
+
const reader = stream.getReader();
|
|
1219
|
+
const decoder = new TextDecoder('utf-8');
|
|
1220
|
+
let buffer = '';
|
|
1221
|
+
while (true) {
|
|
1222
|
+
const {
|
|
1223
|
+
value,
|
|
1224
|
+
done
|
|
1225
|
+
} = await reader.read();
|
|
1226
|
+
if (done) {
|
|
1227
|
+
break;
|
|
1228
|
+
}
|
|
1229
|
+
buffer += decoder.decode(value, {
|
|
1230
|
+
stream: true
|
|
1231
|
+
});
|
|
1232
|
+
const lines = buffer.split(/\r?\n/);
|
|
1233
|
+
buffer = lines.pop() || '';
|
|
1234
|
+
for (const line of lines) {
|
|
1235
|
+
const trimmed = line.trim();
|
|
1236
|
+
if (!trimmed) {
|
|
1237
|
+
continue;
|
|
1238
|
+
}
|
|
1239
|
+
yield JSON.parse(trimmed);
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
if (buffer.trim()) {
|
|
1243
|
+
yield JSON.parse(buffer.trim());
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
async function* normalizeJsonInput(input, options = {}) {
|
|
1247
|
+
const format = detectInputFormat(input, options);
|
|
1248
|
+
if (Array.isArray(input)) {
|
|
1249
|
+
for (const item of input) {
|
|
1250
|
+
yield item;
|
|
1251
|
+
}
|
|
1252
|
+
return;
|
|
1253
|
+
}
|
|
1254
|
+
if (isAsyncIterable(input)) {
|
|
1255
|
+
for await (const item of input) {
|
|
1256
|
+
yield item;
|
|
1257
|
+
}
|
|
1258
|
+
return;
|
|
1259
|
+
}
|
|
1260
|
+
if (isIterable(input)) {
|
|
1261
|
+
for (const item of input) {
|
|
1262
|
+
yield item;
|
|
1263
|
+
}
|
|
1264
|
+
return;
|
|
1265
|
+
}
|
|
1266
|
+
if (typeof input === 'string') {
|
|
1267
|
+
if (format === 'ndjson') {
|
|
1268
|
+
yield* parseNdjsonText(input);
|
|
1269
|
+
return;
|
|
1270
|
+
}
|
|
1271
|
+
const parsed = JSON.parse(input);
|
|
1272
|
+
if (Array.isArray(parsed)) {
|
|
1273
|
+
for (const item of parsed) {
|
|
1274
|
+
yield item;
|
|
1275
|
+
}
|
|
1276
|
+
return;
|
|
1277
|
+
}
|
|
1278
|
+
yield parsed;
|
|
1279
|
+
return;
|
|
1280
|
+
}
|
|
1281
|
+
if (input instanceof Blob) {
|
|
1282
|
+
if (format === 'ndjson') {
|
|
1283
|
+
yield* parseNdjsonStream(input.stream());
|
|
1284
|
+
return;
|
|
1285
|
+
}
|
|
1286
|
+
const text = await input.text();
|
|
1287
|
+
const parsed = JSON.parse(text);
|
|
1288
|
+
if (Array.isArray(parsed)) {
|
|
1289
|
+
for (const item of parsed) {
|
|
1290
|
+
yield item;
|
|
1291
|
+
}
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
yield parsed;
|
|
1295
|
+
return;
|
|
1296
|
+
}
|
|
1297
|
+
if (isReadableStream(input)) {
|
|
1298
|
+
if (format !== 'ndjson') {
|
|
1299
|
+
throw new ValidationError('ReadableStream input requires inputFormat="ndjson"');
|
|
1300
|
+
}
|
|
1301
|
+
yield* parseNdjsonStream(input);
|
|
1302
|
+
return;
|
|
1303
|
+
}
|
|
1304
|
+
throw new ValidationError('Input must be an array, iterable, string, Blob, or ReadableStream');
|
|
1305
|
+
}
|
|
1306
|
+
function validateStreamOptions(options) {
|
|
1307
|
+
if (options && typeof options !== 'object') {
|
|
1308
|
+
throw new ConfigurationError('Options must be an object');
|
|
1309
|
+
}
|
|
1310
|
+
if (options?.delimiter && typeof options.delimiter !== 'string') {
|
|
1311
|
+
throw new ConfigurationError('Delimiter must be a string');
|
|
1312
|
+
}
|
|
1313
|
+
if (options?.delimiter && options.delimiter.length !== 1) {
|
|
1314
|
+
throw new ConfigurationError('Delimiter must be a single character');
|
|
1315
|
+
}
|
|
1316
|
+
if (options?.renameMap && typeof options.renameMap !== 'object') {
|
|
1317
|
+
throw new ConfigurationError('renameMap must be an object');
|
|
1318
|
+
}
|
|
1319
|
+
if (options?.maxRecords !== undefined) {
|
|
1320
|
+
if (typeof options.maxRecords !== 'number' || options.maxRecords <= 0) {
|
|
1321
|
+
throw new ConfigurationError('maxRecords must be a positive number');
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
function escapeCsvValue(value, options) {
|
|
1326
|
+
const {
|
|
1327
|
+
delimiter,
|
|
1328
|
+
preventCsvInjection = true,
|
|
1329
|
+
rfc4180Compliant = true
|
|
1330
|
+
} = options;
|
|
1331
|
+
if (value === null || value === undefined || value === '') {
|
|
1332
|
+
return '';
|
|
1333
|
+
}
|
|
1334
|
+
const stringValue = String(value);
|
|
1335
|
+
let escapedValue = stringValue;
|
|
1336
|
+
if (preventCsvInjection && /^[=+\-@]/.test(stringValue)) {
|
|
1337
|
+
escapedValue = "'" + stringValue;
|
|
1338
|
+
}
|
|
1339
|
+
const needsQuoting = rfc4180Compliant ? escapedValue.includes(delimiter) || escapedValue.includes('"') || escapedValue.includes('\n') || escapedValue.includes('\r') : escapedValue.includes(delimiter) || escapedValue.includes('"') || escapedValue.includes('\n') || escapedValue.includes('\r');
|
|
1340
|
+
if (needsQuoting) {
|
|
1341
|
+
return `"${escapedValue.replace(/"/g, '""')}"`;
|
|
1342
|
+
}
|
|
1343
|
+
return escapedValue;
|
|
1344
|
+
}
|
|
1345
|
+
function buildHeaderState(keys, options) {
|
|
1346
|
+
const renameMap = options.renameMap || {};
|
|
1347
|
+
const template = options.template || {};
|
|
1348
|
+
const originalKeys = Array.isArray(options.headers) ? options.headers : keys;
|
|
1349
|
+
const headers = originalKeys.map(key => renameMap[key] || key);
|
|
1350
|
+
const reverseRenameMap = {};
|
|
1351
|
+
originalKeys.forEach((key, index) => {
|
|
1352
|
+
reverseRenameMap[headers[index]] = key;
|
|
1353
|
+
});
|
|
1354
|
+
let finalHeaders = headers;
|
|
1355
|
+
if (Object.keys(template).length > 0) {
|
|
1356
|
+
const templateHeaders = Object.keys(template).map(key => renameMap[key] || key);
|
|
1357
|
+
const extraHeaders = headers.filter(h => !templateHeaders.includes(h));
|
|
1358
|
+
finalHeaders = [...templateHeaders, ...extraHeaders];
|
|
1359
|
+
}
|
|
1360
|
+
return {
|
|
1361
|
+
headers: finalHeaders,
|
|
1362
|
+
reverseRenameMap
|
|
1363
|
+
};
|
|
1364
|
+
}
|
|
1365
|
+
async function* jsonToCsvChunkIterator(input, options = {}) {
|
|
1366
|
+
validateStreamOptions(options);
|
|
1367
|
+
const opts = options && typeof options === 'object' ? options : {};
|
|
1368
|
+
const {
|
|
1369
|
+
delimiter = ';',
|
|
1370
|
+
includeHeaders = true,
|
|
1371
|
+
maxRecords,
|
|
1372
|
+
maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
|
|
1373
|
+
headerMode
|
|
1374
|
+
} = opts;
|
|
1375
|
+
let headerState = null;
|
|
1376
|
+
let buffer = '';
|
|
1377
|
+
let recordCount = 0;
|
|
1378
|
+
const lineEnding = opts.rfc4180Compliant === false ? '\n' : '\r\n';
|
|
1379
|
+
if (Array.isArray(input) && !opts.headers && (!headerMode || headerMode === 'all')) {
|
|
1380
|
+
const allKeys = new Set();
|
|
1381
|
+
for (const item of input) {
|
|
1382
|
+
if (!item || typeof item !== 'object') {
|
|
1383
|
+
continue;
|
|
1384
|
+
}
|
|
1385
|
+
Object.keys(item).forEach(key => allKeys.add(key));
|
|
1386
|
+
}
|
|
1387
|
+
headerState = buildHeaderState(Array.from(allKeys), opts);
|
|
1388
|
+
if (includeHeaders && headerState.headers.length > 0) {
|
|
1389
|
+
buffer += headerState.headers.join(delimiter) + lineEnding;
|
|
1390
|
+
}
|
|
1391
|
+
} else if (Array.isArray(opts.headers)) {
|
|
1392
|
+
headerState = buildHeaderState(opts.headers, opts);
|
|
1393
|
+
if (includeHeaders && headerState.headers.length > 0) {
|
|
1394
|
+
buffer += headerState.headers.join(delimiter) + lineEnding;
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
for await (const item of normalizeJsonInput(input, opts)) {
|
|
1398
|
+
if (!item || typeof item !== 'object') {
|
|
1399
|
+
continue;
|
|
1400
|
+
}
|
|
1401
|
+
if (!headerState) {
|
|
1402
|
+
headerState = buildHeaderState(Object.keys(item), opts);
|
|
1403
|
+
if (includeHeaders && headerState.headers.length > 0) {
|
|
1404
|
+
buffer += headerState.headers.join(delimiter) + lineEnding;
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
recordCount += 1;
|
|
1408
|
+
if (maxRecords && recordCount > maxRecords) {
|
|
1409
|
+
throw new LimitError(`Data size exceeds maximum limit of ${maxRecords} records`, maxRecords, recordCount);
|
|
1410
|
+
}
|
|
1411
|
+
const row = headerState.headers.map(header => {
|
|
1412
|
+
const originalKey = headerState.reverseRenameMap[header] || header;
|
|
1413
|
+
return escapeCsvValue(item[originalKey], {
|
|
1414
|
+
delimiter,
|
|
1415
|
+
preventCsvInjection: opts.preventCsvInjection !== false,
|
|
1416
|
+
rfc4180Compliant: opts.rfc4180Compliant !== false
|
|
1417
|
+
});
|
|
1418
|
+
}).join(delimiter);
|
|
1419
|
+
buffer += row + lineEnding;
|
|
1420
|
+
if (buffer.length >= maxChunkSize) {
|
|
1421
|
+
yield buffer;
|
|
1422
|
+
buffer = '';
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
if (buffer.length > 0) {
|
|
1426
|
+
yield buffer;
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
async function* jsonToNdjsonChunkIterator(input, options = {}) {
|
|
1430
|
+
validateStreamOptions(options);
|
|
1431
|
+
for await (const item of normalizeJsonInput(input, options)) {
|
|
1432
|
+
if (item === undefined) {
|
|
1433
|
+
continue;
|
|
1434
|
+
}
|
|
1435
|
+
yield JSON.stringify(item) + '\n';
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
async function* csvToJsonChunkIterator(input, options = {}) {
|
|
1439
|
+
const outputFormat = options.outputFormat || 'ndjson';
|
|
1440
|
+
const asArray = outputFormat === 'json-array' || outputFormat === 'array' || outputFormat === 'json';
|
|
1441
|
+
let first = true;
|
|
1442
|
+
if (asArray) {
|
|
1443
|
+
yield '[';
|
|
1444
|
+
}
|
|
1445
|
+
for await (const row of csvToJsonIterator(input, options)) {
|
|
1446
|
+
const payload = JSON.stringify(row);
|
|
1447
|
+
if (asArray) {
|
|
1448
|
+
yield (first ? '' : ',') + payload;
|
|
1449
|
+
} else {
|
|
1450
|
+
yield payload + '\n';
|
|
1451
|
+
}
|
|
1452
|
+
first = false;
|
|
1453
|
+
}
|
|
1454
|
+
if (asArray) {
|
|
1455
|
+
yield ']';
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
function jsonToCsvStream(input, options = {}) {
|
|
1459
|
+
const iterator = jsonToCsvChunkIterator(input, options);
|
|
1460
|
+
return createReadableStreamFromIterator(iterator);
|
|
1461
|
+
}
|
|
1462
|
+
function jsonToNdjsonStream(input, options = {}) {
|
|
1463
|
+
const iterator = jsonToNdjsonChunkIterator(input, options);
|
|
1464
|
+
return createReadableStreamFromIterator(iterator);
|
|
1465
|
+
}
|
|
1466
|
+
function csvToJsonStream(input, options = {}) {
|
|
1467
|
+
const iterator = csvToJsonChunkIterator(input, options);
|
|
1468
|
+
return createReadableStreamFromIterator(iterator);
|
|
1469
|
+
}
|
|
1470
|
+
if (typeof module !== 'undefined' && module.exports) {
|
|
1471
|
+
module.exports = {
|
|
1472
|
+
jsonToCsvStream,
|
|
1473
|
+
jsonToNdjsonStream,
|
|
1474
|
+
csvToJsonStream
|
|
912
1475
|
};
|
|
913
1476
|
}
|
|
914
1477
|
|
|
@@ -1033,6 +1596,26 @@ async function parseCsvFile(file, options = {}) {
|
|
|
1033
1596
|
});
|
|
1034
1597
|
}
|
|
1035
1598
|
|
|
1599
|
+
/**
|
|
1600
|
+
* Stream CSV file as async iterator without full buffering.
|
|
1601
|
+
*
|
|
1602
|
+
* @param {File} file - File selected from input
|
|
1603
|
+
* @param {Object} [options] - csvToJson options
|
|
1604
|
+
* @returns {AsyncGenerator<Object>} Async iterator of rows
|
|
1605
|
+
*/
|
|
1606
|
+
function parseCsvFileStream(file, options = {}) {
|
|
1607
|
+
if (typeof window === 'undefined') {
|
|
1608
|
+
throw new ValidationError('parseCsvFileStream() is browser-only. Use readCsvAsJson() in Node.js');
|
|
1609
|
+
}
|
|
1610
|
+
if (!(file instanceof File)) {
|
|
1611
|
+
throw new ValidationError('Input must be a File object');
|
|
1612
|
+
}
|
|
1613
|
+
if (!file.name.toLowerCase().endsWith('.csv')) {
|
|
1614
|
+
throw new ValidationError('File must have .csv extension');
|
|
1615
|
+
}
|
|
1616
|
+
return csvToJsonIterator(file, options);
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1036
1619
|
/**
|
|
1037
1620
|
* Создает CSV файл из JSON данных (альтернатива downloadAsCsv)
|
|
1038
1621
|
* Возвращает Blob вместо автоматического скачивания
|
|
@@ -1082,8 +1665,12 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
1082
1665
|
module.exports = {
|
|
1083
1666
|
downloadAsCsv,
|
|
1084
1667
|
parseCsvFile,
|
|
1668
|
+
parseCsvFileStream,
|
|
1085
1669
|
createCsvBlob,
|
|
1086
|
-
parseCsvBlob
|
|
1670
|
+
parseCsvBlob,
|
|
1671
|
+
jsonToCsvStream,
|
|
1672
|
+
jsonToNdjsonStream,
|
|
1673
|
+
csvToJsonStream
|
|
1087
1674
|
};
|
|
1088
1675
|
}
|
|
1089
1676
|
|
|
@@ -1093,44 +1680,74 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
1093
1680
|
|
|
1094
1681
|
// Проверка поддержки Web Workers
|
|
1095
1682
|
const WORKERS_SUPPORTED = typeof Worker !== 'undefined';
|
|
1683
|
+
function isTransferableBuffer(value) {
|
|
1684
|
+
if (!(value instanceof ArrayBuffer)) {
|
|
1685
|
+
return false;
|
|
1686
|
+
}
|
|
1687
|
+
if (typeof SharedArrayBuffer !== 'undefined' && value instanceof SharedArrayBuffer) {
|
|
1688
|
+
return false;
|
|
1689
|
+
}
|
|
1690
|
+
return true;
|
|
1691
|
+
}
|
|
1692
|
+
function collectTransferables(args) {
|
|
1693
|
+
const transferables = [];
|
|
1694
|
+
const collectFromValue = value => {
|
|
1695
|
+
if (!value) {
|
|
1696
|
+
return;
|
|
1697
|
+
}
|
|
1698
|
+
if (isTransferableBuffer(value)) {
|
|
1699
|
+
transferables.push(value);
|
|
1700
|
+
return;
|
|
1701
|
+
}
|
|
1702
|
+
if (ArrayBuffer.isView(value) && isTransferableBuffer(value.buffer)) {
|
|
1703
|
+
transferables.push(value.buffer);
|
|
1704
|
+
return;
|
|
1705
|
+
}
|
|
1706
|
+
if (Array.isArray(value)) {
|
|
1707
|
+
value.forEach(collectFromValue);
|
|
1708
|
+
}
|
|
1709
|
+
};
|
|
1710
|
+
args.forEach(collectFromValue);
|
|
1711
|
+
return transferables.length ? transferables : null;
|
|
1712
|
+
}
|
|
1096
1713
|
|
|
1097
|
-
/**
|
|
1098
|
-
* Опции для Worker Pool
|
|
1099
|
-
* @typedef {Object} WorkerPoolOptions
|
|
1100
|
-
* @property {number} [workerCount=4] - Количество workers в pool
|
|
1101
|
-
* @property {number} [maxQueueSize=100] - Максимальный размер очереди задач
|
|
1102
|
-
* @property {boolean} [autoScale=true] - Автоматическое масштабирование pool
|
|
1103
|
-
* @property {number} [idleTimeout=60000] - Таймаут простоя worker (мс)
|
|
1714
|
+
/**
|
|
1715
|
+
* Опции для Worker Pool
|
|
1716
|
+
* @typedef {Object} WorkerPoolOptions
|
|
1717
|
+
* @property {number} [workerCount=4] - Количество workers в pool
|
|
1718
|
+
* @property {number} [maxQueueSize=100] - Максимальный размер очереди задач
|
|
1719
|
+
* @property {boolean} [autoScale=true] - Автоматическое масштабирование pool
|
|
1720
|
+
* @property {number} [idleTimeout=60000] - Таймаут простоя worker (мс)
|
|
1104
1721
|
*/
|
|
1105
1722
|
|
|
1106
|
-
/**
|
|
1107
|
-
* Статистика Worker Pool
|
|
1108
|
-
* @typedef {Object} WorkerPoolStats
|
|
1109
|
-
* @property {number} totalWorkers - Всего workers
|
|
1110
|
-
* @property {number} activeWorkers - Активные workers
|
|
1111
|
-
* @property {number} idleWorkers - Простаивающие workers
|
|
1112
|
-
* @property {number} queueSize - Размер очереди
|
|
1113
|
-
* @property {number} tasksCompleted - Завершенные задачи
|
|
1114
|
-
* @property {number} tasksFailed - Неудачные задачи
|
|
1723
|
+
/**
|
|
1724
|
+
* Статистика Worker Pool
|
|
1725
|
+
* @typedef {Object} WorkerPoolStats
|
|
1726
|
+
* @property {number} totalWorkers - Всего workers
|
|
1727
|
+
* @property {number} activeWorkers - Активные workers
|
|
1728
|
+
* @property {number} idleWorkers - Простаивающие workers
|
|
1729
|
+
* @property {number} queueSize - Размер очереди
|
|
1730
|
+
* @property {number} tasksCompleted - Завершенные задачи
|
|
1731
|
+
* @property {number} tasksFailed - Неудачные задачи
|
|
1115
1732
|
*/
|
|
1116
1733
|
|
|
1117
|
-
/**
|
|
1118
|
-
* Прогресс обработки задачи
|
|
1119
|
-
* @typedef {Object} TaskProgress
|
|
1120
|
-
* @property {number} processed - Обработано элементов
|
|
1121
|
-
* @property {number} total - Всего элементов
|
|
1122
|
-
* @property {number} percentage - Процент выполнения
|
|
1123
|
-
* @property {number} speed - Скорость обработки (элементов/сек)
|
|
1734
|
+
/**
|
|
1735
|
+
* Прогресс обработки задачи
|
|
1736
|
+
* @typedef {Object} TaskProgress
|
|
1737
|
+
* @property {number} processed - Обработано элементов
|
|
1738
|
+
* @property {number} total - Всего элементов
|
|
1739
|
+
* @property {number} percentage - Процент выполнения
|
|
1740
|
+
* @property {number} speed - Скорость обработки (элементов/сек)
|
|
1124
1741
|
*/
|
|
1125
1742
|
|
|
1126
|
-
/**
|
|
1127
|
-
* Worker Pool для параллельной обработки CSV
|
|
1743
|
+
/**
|
|
1744
|
+
* Worker Pool для параллельной обработки CSV
|
|
1128
1745
|
*/
|
|
1129
1746
|
class WorkerPool {
|
|
1130
|
-
/**
|
|
1131
|
-
* Создает новый Worker Pool
|
|
1132
|
-
* @param {string} workerScript - URL скрипта worker
|
|
1133
|
-
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
1747
|
+
/**
|
|
1748
|
+
* Создает новый Worker Pool
|
|
1749
|
+
* @param {string} workerScript - URL скрипта worker
|
|
1750
|
+
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
1134
1751
|
*/
|
|
1135
1752
|
constructor(workerScript, options = {}) {
|
|
1136
1753
|
if (!WORKERS_SUPPORTED) {
|
|
@@ -1158,9 +1775,9 @@ class WorkerPool {
|
|
|
1158
1775
|
this.initializeWorkers();
|
|
1159
1776
|
}
|
|
1160
1777
|
|
|
1161
|
-
/**
|
|
1162
|
-
* Инициализация workers
|
|
1163
|
-
* @private
|
|
1778
|
+
/**
|
|
1779
|
+
* Инициализация workers
|
|
1780
|
+
* @private
|
|
1164
1781
|
*/
|
|
1165
1782
|
initializeWorkers() {
|
|
1166
1783
|
const {
|
|
@@ -1172,9 +1789,9 @@ class WorkerPool {
|
|
|
1172
1789
|
this.updateStats();
|
|
1173
1790
|
}
|
|
1174
1791
|
|
|
1175
|
-
/**
|
|
1176
|
-
* Создает нового worker
|
|
1177
|
-
* @private
|
|
1792
|
+
/**
|
|
1793
|
+
* Создает нового worker
|
|
1794
|
+
* @private
|
|
1178
1795
|
*/
|
|
1179
1796
|
createWorker() {
|
|
1180
1797
|
try {
|
|
@@ -1199,9 +1816,9 @@ class WorkerPool {
|
|
|
1199
1816
|
}
|
|
1200
1817
|
}
|
|
1201
1818
|
|
|
1202
|
-
/**
|
|
1203
|
-
* Обработка сообщений от worker
|
|
1204
|
-
* @private
|
|
1819
|
+
/**
|
|
1820
|
+
* Обработка сообщений от worker
|
|
1821
|
+
* @private
|
|
1205
1822
|
*/
|
|
1206
1823
|
handleWorkerMessage(worker, event) {
|
|
1207
1824
|
const {
|
|
@@ -1216,9 +1833,9 @@ class WorkerPool {
|
|
|
1216
1833
|
}
|
|
1217
1834
|
}
|
|
1218
1835
|
|
|
1219
|
-
/**
|
|
1220
|
-
* Обработка прогресса задачи
|
|
1221
|
-
* @private
|
|
1836
|
+
/**
|
|
1837
|
+
* Обработка прогресса задачи
|
|
1838
|
+
* @private
|
|
1222
1839
|
*/
|
|
1223
1840
|
handleProgress(worker, progressData) {
|
|
1224
1841
|
const taskId = worker.taskId;
|
|
@@ -1235,9 +1852,9 @@ class WorkerPool {
|
|
|
1235
1852
|
}
|
|
1236
1853
|
}
|
|
1237
1854
|
|
|
1238
|
-
/**
|
|
1239
|
-
* Обработка результата задачи
|
|
1240
|
-
* @private
|
|
1855
|
+
/**
|
|
1856
|
+
* Обработка результата задачи
|
|
1857
|
+
* @private
|
|
1241
1858
|
*/
|
|
1242
1859
|
handleResult(worker, resultData) {
|
|
1243
1860
|
const taskId = worker.taskId;
|
|
@@ -1262,9 +1879,9 @@ class WorkerPool {
|
|
|
1262
1879
|
}
|
|
1263
1880
|
}
|
|
1264
1881
|
|
|
1265
|
-
/**
|
|
1266
|
-
* Обработка ошибки задачи
|
|
1267
|
-
* @private
|
|
1882
|
+
/**
|
|
1883
|
+
* Обработка ошибки задачи
|
|
1884
|
+
* @private
|
|
1268
1885
|
*/
|
|
1269
1886
|
handleWorkerTaskError(worker, errorData) {
|
|
1270
1887
|
const taskId = worker.taskId;
|
|
@@ -1279,7 +1896,14 @@ class WorkerPool {
|
|
|
1279
1896
|
this.stats.idleWorkers++;
|
|
1280
1897
|
|
|
1281
1898
|
// Завершение с ошибкой
|
|
1282
|
-
|
|
1899
|
+
const workerError = new Error(errorData.message || 'Ошибка в worker');
|
|
1900
|
+
if (errorData.code) {
|
|
1901
|
+
workerError.code = errorData.code;
|
|
1902
|
+
}
|
|
1903
|
+
if (errorData.details) {
|
|
1904
|
+
workerError.details = errorData.details;
|
|
1905
|
+
}
|
|
1906
|
+
task.reject(workerError);
|
|
1283
1907
|
this.activeTasks.delete(taskId);
|
|
1284
1908
|
this.stats.tasksFailed++;
|
|
1285
1909
|
|
|
@@ -1289,9 +1913,9 @@ class WorkerPool {
|
|
|
1289
1913
|
}
|
|
1290
1914
|
}
|
|
1291
1915
|
|
|
1292
|
-
/**
|
|
1293
|
-
* Обработка ошибок worker
|
|
1294
|
-
* @private
|
|
1916
|
+
/**
|
|
1917
|
+
* Обработка ошибок worker
|
|
1918
|
+
* @private
|
|
1295
1919
|
*/
|
|
1296
1920
|
handleWorkerError(worker, error) {
|
|
1297
1921
|
console.error(`Worker ${worker.id} error:`, error);
|
|
@@ -1300,17 +1924,17 @@ class WorkerPool {
|
|
|
1300
1924
|
this.restartWorker(worker);
|
|
1301
1925
|
}
|
|
1302
1926
|
|
|
1303
|
-
/**
|
|
1304
|
-
* Обработка ошибок сообщений
|
|
1305
|
-
* @private
|
|
1927
|
+
/**
|
|
1928
|
+
* Обработка ошибок сообщений
|
|
1929
|
+
* @private
|
|
1306
1930
|
*/
|
|
1307
1931
|
handleWorkerMessageError(worker, error) {
|
|
1308
1932
|
console.error(`Worker ${worker.id} message error:`, error);
|
|
1309
1933
|
}
|
|
1310
1934
|
|
|
1311
|
-
/**
|
|
1312
|
-
* Перезапуск worker
|
|
1313
|
-
* @private
|
|
1935
|
+
/**
|
|
1936
|
+
* Перезапуск worker
|
|
1937
|
+
* @private
|
|
1314
1938
|
*/
|
|
1315
1939
|
restartWorker(worker) {
|
|
1316
1940
|
const index = this.workers.indexOf(worker);
|
|
@@ -1338,9 +1962,9 @@ class WorkerPool {
|
|
|
1338
1962
|
}
|
|
1339
1963
|
}
|
|
1340
1964
|
|
|
1341
|
-
/**
|
|
1342
|
-
* Выполнение задачи на worker
|
|
1343
|
-
* @private
|
|
1965
|
+
/**
|
|
1966
|
+
* Выполнение задачи на worker
|
|
1967
|
+
* @private
|
|
1344
1968
|
*/
|
|
1345
1969
|
executeTask(worker, task) {
|
|
1346
1970
|
worker.status = 'active';
|
|
@@ -1350,59 +1974,59 @@ class WorkerPool {
|
|
|
1350
1974
|
this.stats.activeWorkers++;
|
|
1351
1975
|
|
|
1352
1976
|
// Отправка задачи в worker
|
|
1353
|
-
|
|
1977
|
+
const payload = {
|
|
1354
1978
|
type: 'EXECUTE',
|
|
1355
1979
|
taskId: task.id,
|
|
1356
1980
|
method: task.method,
|
|
1357
1981
|
args: task.args,
|
|
1358
1982
|
options: task.options
|
|
1359
|
-
}
|
|
1983
|
+
};
|
|
1984
|
+
if (task.transferList && task.transferList.length) {
|
|
1985
|
+
worker.postMessage(payload, task.transferList);
|
|
1986
|
+
} else {
|
|
1987
|
+
worker.postMessage(payload);
|
|
1988
|
+
}
|
|
1360
1989
|
}
|
|
1361
1990
|
|
|
1362
|
-
/**
|
|
1363
|
-
* Обработка очереди задач
|
|
1364
|
-
* @private
|
|
1991
|
+
/**
|
|
1992
|
+
* Обработка очереди задач
|
|
1993
|
+
* @private
|
|
1365
1994
|
*/
|
|
1366
1995
|
processQueue() {
|
|
1367
1996
|
if (this.taskQueue.length === 0) {
|
|
1368
1997
|
return;
|
|
1369
1998
|
}
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1999
|
+
while (this.taskQueue.length > 0) {
|
|
2000
|
+
const idleWorker = this.workers.find(w => w.status === 'idle');
|
|
2001
|
+
if (!idleWorker) {
|
|
2002
|
+
if (this.options.autoScale && this.workers.length < this.options.maxQueueSize) {
|
|
2003
|
+
this.createWorker();
|
|
2004
|
+
continue;
|
|
2005
|
+
}
|
|
2006
|
+
break;
|
|
1378
2007
|
}
|
|
1379
|
-
|
|
2008
|
+
const task = this.taskQueue.shift();
|
|
2009
|
+
this.stats.queueSize--;
|
|
2010
|
+
this.executeTask(idleWorker, task);
|
|
1380
2011
|
}
|
|
1381
|
-
|
|
1382
|
-
// Получение задачи из очереди
|
|
1383
|
-
const task = this.taskQueue.shift();
|
|
1384
|
-
this.stats.queueSize--;
|
|
1385
|
-
|
|
1386
|
-
// Выполнение задачи
|
|
1387
|
-
this.executeTask(idleWorker, task);
|
|
1388
2012
|
this.updateStats();
|
|
1389
2013
|
}
|
|
1390
2014
|
|
|
1391
|
-
/**
|
|
1392
|
-
* Обновление статистики
|
|
1393
|
-
* @private
|
|
2015
|
+
/**
|
|
2016
|
+
* Обновление статистики
|
|
2017
|
+
* @private
|
|
1394
2018
|
*/
|
|
1395
2019
|
updateStats() {
|
|
1396
2020
|
this.stats.queueSize = this.taskQueue.length;
|
|
1397
2021
|
}
|
|
1398
2022
|
|
|
1399
|
-
/**
|
|
1400
|
-
* Выполнение задачи через pool
|
|
1401
|
-
* @param {string} method - Метод для вызова в worker
|
|
1402
|
-
* @param {Array} args - Аргументы метода
|
|
1403
|
-
* @param {Object} [options] - Опции задачи
|
|
1404
|
-
* @param {Function} [onProgress] - Callback прогресса
|
|
1405
|
-
* @returns {Promise<any>} Результат выполнения
|
|
2023
|
+
/**
|
|
2024
|
+
* Выполнение задачи через pool
|
|
2025
|
+
* @param {string} method - Метод для вызова в worker
|
|
2026
|
+
* @param {Array} args - Аргументы метода
|
|
2027
|
+
* @param {Object} [options] - Опции задачи
|
|
2028
|
+
* @param {Function} [onProgress] - Callback прогресса
|
|
2029
|
+
* @returns {Promise<any>} Результат выполнения
|
|
1406
2030
|
*/
|
|
1407
2031
|
async exec(method, args = [], options = {}, onProgress = null) {
|
|
1408
2032
|
return new Promise((resolve, reject) => {
|
|
@@ -1414,11 +2038,17 @@ class WorkerPool {
|
|
|
1414
2038
|
|
|
1415
2039
|
// Создание задачи
|
|
1416
2040
|
const taskId = `task-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
2041
|
+
const {
|
|
2042
|
+
transfer,
|
|
2043
|
+
...taskOptions
|
|
2044
|
+
} = options || {};
|
|
2045
|
+
const transferList = transfer || collectTransferables(args);
|
|
1417
2046
|
const task = {
|
|
1418
2047
|
id: taskId,
|
|
1419
2048
|
method,
|
|
1420
2049
|
args,
|
|
1421
|
-
options,
|
|
2050
|
+
options: taskOptions,
|
|
2051
|
+
transferList,
|
|
1422
2052
|
onProgress,
|
|
1423
2053
|
resolve,
|
|
1424
2054
|
reject,
|
|
@@ -1435,9 +2065,9 @@ class WorkerPool {
|
|
|
1435
2065
|
});
|
|
1436
2066
|
}
|
|
1437
2067
|
|
|
1438
|
-
/**
|
|
1439
|
-
* Получение статистики pool
|
|
1440
|
-
* @returns {WorkerPoolStats} Статистика
|
|
2068
|
+
/**
|
|
2069
|
+
* Получение статистики pool
|
|
2070
|
+
* @returns {WorkerPoolStats} Статистика
|
|
1441
2071
|
*/
|
|
1442
2072
|
getStats() {
|
|
1443
2073
|
return {
|
|
@@ -1445,8 +2075,8 @@ class WorkerPool {
|
|
|
1445
2075
|
};
|
|
1446
2076
|
}
|
|
1447
2077
|
|
|
1448
|
-
/**
|
|
1449
|
-
* Очистка простаивающих workers
|
|
2078
|
+
/**
|
|
2079
|
+
* Очистка простаивающих workers
|
|
1450
2080
|
*/
|
|
1451
2081
|
cleanupIdleWorkers() {
|
|
1452
2082
|
const now = Date.now();
|
|
@@ -1467,8 +2097,8 @@ class WorkerPool {
|
|
|
1467
2097
|
}
|
|
1468
2098
|
}
|
|
1469
2099
|
|
|
1470
|
-
/**
|
|
1471
|
-
* Завершение всех workers
|
|
2100
|
+
/**
|
|
2101
|
+
* Завершение всех workers
|
|
1472
2102
|
*/
|
|
1473
2103
|
terminate() {
|
|
1474
2104
|
this.workers.forEach(worker => {
|
|
@@ -1490,10 +2120,10 @@ class WorkerPool {
|
|
|
1490
2120
|
}
|
|
1491
2121
|
}
|
|
1492
2122
|
|
|
1493
|
-
/**
|
|
1494
|
-
* Создает Worker Pool для обработки CSV
|
|
1495
|
-
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
1496
|
-
* @returns {WorkerPool} Worker Pool
|
|
2123
|
+
/**
|
|
2124
|
+
* Создает Worker Pool для обработки CSV
|
|
2125
|
+
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
2126
|
+
* @returns {WorkerPool} Worker Pool
|
|
1497
2127
|
*/
|
|
1498
2128
|
function createWorkerPool(options = {}) {
|
|
1499
2129
|
// Используем встроенный worker скрипт
|
|
@@ -1501,12 +2131,12 @@ function createWorkerPool(options = {}) {
|
|
|
1501
2131
|
return new WorkerPool(workerScript, options);
|
|
1502
2132
|
}
|
|
1503
2133
|
|
|
1504
|
-
/**
|
|
1505
|
-
* Парсит CSV с использованием Web Workers
|
|
1506
|
-
* @param {string|File} csvInput - CSV строка или File объект
|
|
1507
|
-
* @param {Object} [options] - Опции парсинга
|
|
1508
|
-
* @param {Function} [onProgress] - Callback прогресса
|
|
1509
|
-
* @returns {Promise<Array<Object>>} JSON данные
|
|
2134
|
+
/**
|
|
2135
|
+
* Парсит CSV с использованием Web Workers
|
|
2136
|
+
* @param {string|File} csvInput - CSV строка или File объект
|
|
2137
|
+
* @param {Object} [options] - Опции парсинга
|
|
2138
|
+
* @param {Function} [onProgress] - Callback прогресса
|
|
2139
|
+
* @returns {Promise<Array<Object>>} JSON данные
|
|
1510
2140
|
*/
|
|
1511
2141
|
async function parseCSVWithWorker(csvInput, options = {}, onProgress = null) {
|
|
1512
2142
|
// Создание pool если нужно
|
|
@@ -1516,29 +2146,42 @@ async function parseCSVWithWorker(csvInput, options = {}, onProgress = null) {
|
|
|
1516
2146
|
const pool = parseCSVWithWorker.pool;
|
|
1517
2147
|
|
|
1518
2148
|
// Подготовка CSV строки
|
|
1519
|
-
|
|
2149
|
+
// ?????????? CSV ??????
|
|
2150
|
+
let csvPayload = csvInput;
|
|
2151
|
+
let transfer = null;
|
|
1520
2152
|
if (csvInput instanceof File) {
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
} else {
|
|
1525
|
-
|
|
2153
|
+
const buffer = await readFileAsArrayBuffer(csvInput);
|
|
2154
|
+
csvPayload = new Uint8Array(buffer);
|
|
2155
|
+
transfer = [buffer];
|
|
2156
|
+
} else if (csvInput instanceof ArrayBuffer) {
|
|
2157
|
+
csvPayload = csvInput;
|
|
2158
|
+
transfer = [csvInput];
|
|
2159
|
+
} else if (ArrayBuffer.isView(csvInput)) {
|
|
2160
|
+
csvPayload = csvInput;
|
|
2161
|
+
if (csvInput.buffer instanceof ArrayBuffer) {
|
|
2162
|
+
transfer = [csvInput.buffer];
|
|
2163
|
+
}
|
|
2164
|
+
} else if (typeof csvInput !== 'string') {
|
|
2165
|
+
throw new ValidationError('Input must be a CSV string, File, or ArrayBuffer');
|
|
1526
2166
|
}
|
|
1527
2167
|
|
|
1528
|
-
//
|
|
1529
|
-
|
|
2168
|
+
// ????????? ?????? ????? pool
|
|
2169
|
+
const execOptions = transfer ? {
|
|
2170
|
+
transfer
|
|
2171
|
+
} : {};
|
|
2172
|
+
return pool.exec('parseCSV', [csvPayload, options], execOptions, onProgress);
|
|
1530
2173
|
}
|
|
1531
2174
|
|
|
1532
|
-
/**
|
|
1533
|
-
* Чтение файла как текст
|
|
1534
|
-
* @private
|
|
2175
|
+
/**
|
|
2176
|
+
* Чтение файла как текст
|
|
2177
|
+
* @private
|
|
1535
2178
|
*/
|
|
1536
|
-
async function
|
|
2179
|
+
async function readFileAsArrayBuffer(file) {
|
|
1537
2180
|
return new Promise((resolve, reject) => {
|
|
1538
2181
|
const reader = new FileReader();
|
|
1539
2182
|
reader.onload = event => resolve(event.target.result);
|
|
1540
2183
|
reader.onerror = error => reject(error);
|
|
1541
|
-
reader.
|
|
2184
|
+
reader.readAsArrayBuffer(file);
|
|
1542
2185
|
});
|
|
1543
2186
|
}
|
|
1544
2187
|
|
|
@@ -1551,9 +2194,24 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
1551
2194
|
};
|
|
1552
2195
|
}
|
|
1553
2196
|
|
|
2197
|
+
var workerPool = /*#__PURE__*/Object.freeze({
|
|
2198
|
+
__proto__: null,
|
|
2199
|
+
WorkerPool: WorkerPool,
|
|
2200
|
+
createWorkerPool: createWorkerPool,
|
|
2201
|
+
parseCSVWithWorker: parseCSVWithWorker
|
|
2202
|
+
});
|
|
2203
|
+
|
|
1554
2204
|
// Браузерный entry point для jtcsv
|
|
1555
2205
|
// Экспортирует все функции с поддержкой браузера
|
|
1556
2206
|
|
|
2207
|
+
async function createWorkerPoolLazy(options = {}) {
|
|
2208
|
+
const mod = await Promise.resolve().then(function () { return workerPool; });
|
|
2209
|
+
return mod.createWorkerPool(options);
|
|
2210
|
+
}
|
|
2211
|
+
async function parseCSVWithWorkerLazy(csvInput, options = {}, onProgress = null) {
|
|
2212
|
+
const mod = await Promise.resolve().then(function () { return workerPool; });
|
|
2213
|
+
return mod.parseCSVWithWorker(csvInput, options, onProgress);
|
|
2214
|
+
}
|
|
1557
2215
|
|
|
1558
2216
|
// Основной экспорт
|
|
1559
2217
|
const jtcsv = {
|
|
@@ -1564,11 +2222,18 @@ const jtcsv = {
|
|
|
1564
2222
|
deepUnwrap,
|
|
1565
2223
|
// CSV to JSON функции
|
|
1566
2224
|
csvToJson,
|
|
2225
|
+
csvToJsonIterator,
|
|
1567
2226
|
parseCsvFile,
|
|
2227
|
+
parseCsvFileStream,
|
|
2228
|
+
jsonToCsvStream,
|
|
2229
|
+
jsonToNdjsonStream,
|
|
2230
|
+
csvToJsonStream,
|
|
1568
2231
|
autoDetectDelimiter,
|
|
1569
2232
|
// Web Workers функции
|
|
1570
2233
|
createWorkerPool,
|
|
1571
2234
|
parseCSVWithWorker,
|
|
2235
|
+
createWorkerPoolLazy,
|
|
2236
|
+
parseCSVWithWorkerLazy,
|
|
1572
2237
|
// Error classes
|
|
1573
2238
|
ValidationError,
|
|
1574
2239
|
SecurityError,
|
|
@@ -1576,6 +2241,7 @@ const jtcsv = {
|
|
|
1576
2241
|
ParsingError,
|
|
1577
2242
|
LimitError,
|
|
1578
2243
|
ConfigurationError,
|
|
2244
|
+
ERROR_CODES,
|
|
1579
2245
|
// Удобные алиасы
|
|
1580
2246
|
parse: csvToJson,
|
|
1581
2247
|
unparse: jsonToCsv,
|
|
@@ -1595,5 +2261,5 @@ if (typeof module !== 'undefined' && module.exports) {
|
|
|
1595
2261
|
window.jtcsv = jtcsv;
|
|
1596
2262
|
}
|
|
1597
2263
|
|
|
1598
|
-
export { ConfigurationError, FileSystemError, LimitError, ParsingError, SecurityError, ValidationError, autoDetectDelimiter, createWorkerPool, csvToJson, deepUnwrap, jtcsv as default, downloadAsCsv, jsonToCsv, parseCSVWithWorker, parseCsvFile, preprocessData };
|
|
2264
|
+
export { ConfigurationError, ERROR_CODES, FileSystemError, LimitError, ParsingError, SecurityError, ValidationError, autoDetectDelimiter, createWorkerPool, createWorkerPoolLazy, csvToJson, csvToJsonIterator, csvToJsonStream, deepUnwrap, jtcsv as default, downloadAsCsv, jsonToCsv, jsonToCsvStream, jsonToNdjsonStream, parseCSVWithWorker, parseCSVWithWorkerLazy, parseCsvFile, parseCsvFileStream, preprocessData };
|
|
1599
2265
|
//# sourceMappingURL=jtcsv.esm.js.map
|