jtcsv 2.1.3 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +60 -341
- package/bin/jtcsv.js +2462 -1372
- package/csv-to-json.js +35 -26
- package/dist/jtcsv.cjs.js +807 -133
- package/dist/jtcsv.cjs.js.map +1 -1
- package/dist/jtcsv.esm.js +800 -134
- package/dist/jtcsv.esm.js.map +1 -1
- package/dist/jtcsv.umd.js +807 -133
- package/dist/jtcsv.umd.js.map +1 -1
- package/errors.js +20 -0
- package/examples/browser-vanilla.html +37 -0
- package/examples/cli-batch-processing.js +38 -0
- package/examples/error-handling.js +324 -0
- package/examples/ndjson-processing.js +434 -0
- package/examples/react-integration.jsx +637 -0
- package/examples/schema-validation.js +640 -0
- package/examples/simple-usage.js +10 -7
- package/examples/typescript-example.ts +486 -0
- package/examples/web-workers-advanced.js +28 -0
- package/index.d.ts +2 -0
- package/json-save.js +2 -1
- package/json-to-csv.js +171 -131
- package/package.json +20 -4
- package/plugins/README.md +41 -467
- package/plugins/express-middleware/README.md +32 -274
- package/plugins/hono/README.md +16 -13
- package/plugins/nestjs/README.md +13 -11
- package/plugins/nextjs-api/README.md +28 -423
- package/plugins/nextjs-api/index.js +1 -2
- package/plugins/nextjs-api/route.js +1 -2
- package/plugins/nuxt/README.md +6 -7
- package/plugins/remix/README.md +9 -9
- package/plugins/sveltekit/README.md +8 -8
- package/plugins/trpc/README.md +8 -5
- package/src/browser/browser-functions.js +33 -3
- package/src/browser/csv-to-json-browser.js +269 -11
- package/src/browser/errors-browser.js +19 -1
- package/src/browser/index.js +39 -5
- package/src/browser/streams.js +393 -0
- package/src/browser/workers/csv-parser.worker.js +20 -2
- package/src/browser/workers/worker-pool.js +507 -447
- package/src/core/plugin-system.js +4 -0
- package/src/engines/fast-path-engine.js +31 -23
- package/src/errors.js +26 -0
- package/src/formats/ndjson-parser.js +54 -5
- package/src/formats/tsv-parser.js +4 -1
- package/src/utils/schema-validator.js +594 -0
- package/src/utils/transform-loader.js +205 -0
- package/src/web-server/index.js +683 -0
- package/stream-csv-to-json.js +16 -87
- package/stream-json-to-csv.js +18 -86
package/dist/jtcsv.umd.js
CHANGED
|
@@ -100,6 +100,22 @@
|
|
|
100
100
|
this.name = 'ConfigurationError';
|
|
101
101
|
}
|
|
102
102
|
}
|
|
103
|
+
const ERROR_CODES = {
|
|
104
|
+
JTCSV_ERROR: 'JTCSV_ERROR',
|
|
105
|
+
VALIDATION_ERROR: 'VALIDATION_ERROR',
|
|
106
|
+
SECURITY_ERROR: 'SECURITY_ERROR',
|
|
107
|
+
FILE_SYSTEM_ERROR: 'FILE_SYSTEM_ERROR',
|
|
108
|
+
PARSING_ERROR: 'PARSING_ERROR',
|
|
109
|
+
LIMIT_ERROR: 'LIMIT_ERROR',
|
|
110
|
+
CONFIGURATION_ERROR: 'CONFIGURATION_ERROR',
|
|
111
|
+
INVALID_INPUT: 'INVALID_INPUT',
|
|
112
|
+
SECURITY_VIOLATION: 'SECURITY_VIOLATION',
|
|
113
|
+
FILE_NOT_FOUND: 'FILE_NOT_FOUND',
|
|
114
|
+
PARSE_FAILED: 'PARSE_FAILED',
|
|
115
|
+
SIZE_LIMIT: 'SIZE_LIMIT',
|
|
116
|
+
INVALID_CONFIG: 'INVALID_CONFIG',
|
|
117
|
+
UNKNOWN_ERROR: 'UNKNOWN_ERROR'
|
|
118
|
+
};
|
|
103
119
|
|
|
104
120
|
/**
|
|
105
121
|
* Безопасное выполнение функции с обработкой ошибок
|
|
@@ -235,6 +251,7 @@
|
|
|
235
251
|
ParsingError,
|
|
236
252
|
LimitError,
|
|
237
253
|
ConfigurationError,
|
|
254
|
+
ERROR_CODES,
|
|
238
255
|
safeExecute,
|
|
239
256
|
safeExecuteAsync
|
|
240
257
|
};
|
|
@@ -518,15 +535,10 @@
|
|
|
518
535
|
|
|
519
536
|
|
|
520
537
|
/**
|
|
521
|
-
* Валидация
|
|
538
|
+
* Валидация опций парсинга
|
|
522
539
|
* @private
|
|
523
540
|
*/
|
|
524
|
-
function
|
|
525
|
-
// Validate CSV input
|
|
526
|
-
if (typeof csv !== 'string') {
|
|
527
|
-
throw new ValidationError('Input must be a CSV string');
|
|
528
|
-
}
|
|
529
|
-
|
|
541
|
+
function validateCsvOptions(options) {
|
|
530
542
|
// Validate options
|
|
531
543
|
if (options && typeof options !== 'object') {
|
|
532
544
|
throw new ConfigurationError('Options must be an object');
|
|
@@ -554,9 +566,24 @@
|
|
|
554
566
|
if (options?.maxRows !== undefined && (typeof options.maxRows !== 'number' || options.maxRows <= 0)) {
|
|
555
567
|
throw new ConfigurationError('maxRows must be a positive number');
|
|
556
568
|
}
|
|
569
|
+
if (options?.warnExtraFields !== undefined && typeof options.warnExtraFields !== 'boolean') {
|
|
570
|
+
throw new ConfigurationError('warnExtraFields must be a boolean');
|
|
571
|
+
}
|
|
557
572
|
return true;
|
|
558
573
|
}
|
|
559
574
|
|
|
575
|
+
/**
|
|
576
|
+
* Валидация CSV ввода и опций
|
|
577
|
+
* @private
|
|
578
|
+
*/
|
|
579
|
+
function validateCsvInput(csv, options) {
|
|
580
|
+
// Validate CSV input
|
|
581
|
+
if (typeof csv !== 'string') {
|
|
582
|
+
throw new ValidationError('Input must be a CSV string');
|
|
583
|
+
}
|
|
584
|
+
return validateCsvOptions(options);
|
|
585
|
+
}
|
|
586
|
+
|
|
560
587
|
/**
|
|
561
588
|
* Парсинг одной строки CSV с правильным экранированием
|
|
562
589
|
* @private
|
|
@@ -702,6 +729,78 @@
|
|
|
702
729
|
}
|
|
703
730
|
return result;
|
|
704
731
|
}
|
|
732
|
+
function isSimpleCsv(csv) {
|
|
733
|
+
return csv.indexOf('"') === -1 && csv.indexOf('\\') === -1;
|
|
734
|
+
}
|
|
735
|
+
function parseSimpleCsv(csv, delimiter, options) {
|
|
736
|
+
const {
|
|
737
|
+
hasHeaders = true,
|
|
738
|
+
renameMap = {},
|
|
739
|
+
trim = true,
|
|
740
|
+
parseNumbers = false,
|
|
741
|
+
parseBooleans = false,
|
|
742
|
+
maxRows
|
|
743
|
+
} = options;
|
|
744
|
+
const result = [];
|
|
745
|
+
let headers = null;
|
|
746
|
+
let fieldStart = 0;
|
|
747
|
+
let currentRow = [];
|
|
748
|
+
let rowHasData = false;
|
|
749
|
+
let rowCount = 0;
|
|
750
|
+
const finalizeRow = fields => {
|
|
751
|
+
if (fields.length === 1 && fields[0].trim() === '') {
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
754
|
+
if (!headers) {
|
|
755
|
+
if (hasHeaders) {
|
|
756
|
+
headers = fields.map(header => {
|
|
757
|
+
const trimmed = trim ? header.trim() : header;
|
|
758
|
+
return renameMap[trimmed] || trimmed;
|
|
759
|
+
});
|
|
760
|
+
return;
|
|
761
|
+
}
|
|
762
|
+
headers = fields.map((_, index) => `column${index + 1}`);
|
|
763
|
+
}
|
|
764
|
+
rowCount++;
|
|
765
|
+
if (maxRows && rowCount > maxRows) {
|
|
766
|
+
throw new LimitError(`CSV size exceeds maximum limit of ${maxRows} rows`, maxRows, rowCount);
|
|
767
|
+
}
|
|
768
|
+
const row = {};
|
|
769
|
+
const fieldCount = Math.min(fields.length, headers.length);
|
|
770
|
+
for (let i = 0; i < fieldCount; i++) {
|
|
771
|
+
row[headers[i]] = parseCsvValue(fields[i], {
|
|
772
|
+
trim,
|
|
773
|
+
parseNumbers,
|
|
774
|
+
parseBooleans
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
result.push(row);
|
|
778
|
+
};
|
|
779
|
+
let i = 0;
|
|
780
|
+
while (i <= csv.length) {
|
|
781
|
+
const char = i < csv.length ? csv[i] : '\n';
|
|
782
|
+
if (char !== '\r' && char !== '\n' && char !== ' ' && char !== '\t') {
|
|
783
|
+
rowHasData = true;
|
|
784
|
+
}
|
|
785
|
+
if (char === delimiter || char === '\n' || char === '\r' || i === csv.length) {
|
|
786
|
+
const field = csv.slice(fieldStart, i);
|
|
787
|
+
currentRow.push(field);
|
|
788
|
+
if (char === '\n' || char === '\r' || i === csv.length) {
|
|
789
|
+
if (rowHasData || currentRow.length > 1) {
|
|
790
|
+
finalizeRow(currentRow);
|
|
791
|
+
}
|
|
792
|
+
currentRow = [];
|
|
793
|
+
rowHasData = false;
|
|
794
|
+
}
|
|
795
|
+
if (char === '\r' && csv[i + 1] === '\n') {
|
|
796
|
+
i++;
|
|
797
|
+
}
|
|
798
|
+
fieldStart = i + 1;
|
|
799
|
+
}
|
|
800
|
+
i++;
|
|
801
|
+
}
|
|
802
|
+
return result;
|
|
803
|
+
}
|
|
705
804
|
|
|
706
805
|
/**
|
|
707
806
|
* Автоматическое определение разделителя CSV
|
|
@@ -777,7 +876,8 @@
|
|
|
777
876
|
trim = true,
|
|
778
877
|
parseNumbers = false,
|
|
779
878
|
parseBooleans = false,
|
|
780
|
-
maxRows
|
|
879
|
+
maxRows,
|
|
880
|
+
warnExtraFields = true
|
|
781
881
|
} = opts;
|
|
782
882
|
|
|
783
883
|
// Определение разделителя
|
|
@@ -791,6 +891,16 @@
|
|
|
791
891
|
if (csv.trim() === '') {
|
|
792
892
|
return [];
|
|
793
893
|
}
|
|
894
|
+
if (isSimpleCsv(csv)) {
|
|
895
|
+
return parseSimpleCsv(csv, finalDelimiter, {
|
|
896
|
+
hasHeaders,
|
|
897
|
+
renameMap,
|
|
898
|
+
trim,
|
|
899
|
+
parseNumbers,
|
|
900
|
+
parseBooleans,
|
|
901
|
+
maxRows
|
|
902
|
+
});
|
|
903
|
+
}
|
|
794
904
|
|
|
795
905
|
// Парсинг CSV с обработкой кавычек и переносов строк
|
|
796
906
|
const lines = [];
|
|
@@ -894,7 +1004,8 @@
|
|
|
894
1004
|
}
|
|
895
1005
|
|
|
896
1006
|
// Предупреждение о лишних полях
|
|
897
|
-
|
|
1007
|
+
const isDev = typeof process !== 'undefined' && process.env && process.env.NODE_ENV === 'development';
|
|
1008
|
+
if (fields.length > headers.length && warnExtraFields && isDev) {
|
|
898
1009
|
console.warn(`[jtcsv] Line ${i + 1}: ${fields.length - headers.length} extra fields ignored`);
|
|
899
1010
|
}
|
|
900
1011
|
result.push(row);
|
|
@@ -910,12 +1021,464 @@
|
|
|
910
1021
|
function: 'csvToJson'
|
|
911
1022
|
});
|
|
912
1023
|
}
|
|
1024
|
+
async function* csvToJsonIterator(input, options = {}) {
|
|
1025
|
+
const opts = options && typeof options === 'object' ? options : {};
|
|
1026
|
+
validateCsvOptions(opts);
|
|
1027
|
+
if (typeof input === 'string') {
|
|
1028
|
+
const rows = csvToJson(input, options);
|
|
1029
|
+
for (const row of rows) {
|
|
1030
|
+
yield row;
|
|
1031
|
+
}
|
|
1032
|
+
return;
|
|
1033
|
+
}
|
|
1034
|
+
const {
|
|
1035
|
+
delimiter,
|
|
1036
|
+
autoDetect = true,
|
|
1037
|
+
candidates = [';', ',', '\t', '|'],
|
|
1038
|
+
hasHeaders = true,
|
|
1039
|
+
renameMap = {},
|
|
1040
|
+
trim = true,
|
|
1041
|
+
parseNumbers = false,
|
|
1042
|
+
parseBooleans = false,
|
|
1043
|
+
maxRows
|
|
1044
|
+
} = opts;
|
|
1045
|
+
const stream = input instanceof Blob && input.stream ? input.stream() : input;
|
|
1046
|
+
if (!stream || typeof stream.getReader !== 'function') {
|
|
1047
|
+
throw new ValidationError('Input must be a CSV string, Blob/File, or ReadableStream');
|
|
1048
|
+
}
|
|
1049
|
+
const reader = stream.getReader();
|
|
1050
|
+
const decoder = new TextDecoder('utf-8');
|
|
1051
|
+
let buffer = '';
|
|
1052
|
+
let insideQuotes = false;
|
|
1053
|
+
let headers = null;
|
|
1054
|
+
let rowCount = 0;
|
|
1055
|
+
let lineNumber = 0;
|
|
1056
|
+
let finalDelimiter = delimiter;
|
|
1057
|
+
let delimiterResolved = Boolean(finalDelimiter);
|
|
1058
|
+
const processFields = fields => {
|
|
1059
|
+
if (fields.length === 1 && fields[0].trim() === '') {
|
|
1060
|
+
return null;
|
|
1061
|
+
}
|
|
1062
|
+
rowCount++;
|
|
1063
|
+
if (maxRows && rowCount > maxRows) {
|
|
1064
|
+
throw new LimitError(`CSV size exceeds maximum limit of ${maxRows} rows`, maxRows, rowCount);
|
|
1065
|
+
}
|
|
1066
|
+
const row = {};
|
|
1067
|
+
const fieldCount = Math.min(fields.length, headers.length);
|
|
1068
|
+
for (let j = 0; j < fieldCount; j++) {
|
|
1069
|
+
row[headers[j]] = parseCsvValue(fields[j], {
|
|
1070
|
+
trim,
|
|
1071
|
+
parseNumbers,
|
|
1072
|
+
parseBooleans
|
|
1073
|
+
});
|
|
1074
|
+
}
|
|
1075
|
+
return row;
|
|
1076
|
+
};
|
|
1077
|
+
const processLine = line => {
|
|
1078
|
+
lineNumber++;
|
|
1079
|
+
let cleanLine = line;
|
|
1080
|
+
if (cleanLine.endsWith('\r')) {
|
|
1081
|
+
cleanLine = cleanLine.slice(0, -1);
|
|
1082
|
+
}
|
|
1083
|
+
if (!delimiterResolved) {
|
|
1084
|
+
if (!finalDelimiter && autoDetect) {
|
|
1085
|
+
finalDelimiter = autoDetectDelimiter(cleanLine, candidates);
|
|
1086
|
+
}
|
|
1087
|
+
finalDelimiter = finalDelimiter || ';';
|
|
1088
|
+
delimiterResolved = true;
|
|
1089
|
+
}
|
|
1090
|
+
if (cleanLine.trim() === '') {
|
|
1091
|
+
return null;
|
|
1092
|
+
}
|
|
1093
|
+
if (!headers) {
|
|
1094
|
+
if (hasHeaders) {
|
|
1095
|
+
headers = parseCsvLine(cleanLine, lineNumber, finalDelimiter).map(header => {
|
|
1096
|
+
const trimmed = trim ? header.trim() : header;
|
|
1097
|
+
return renameMap[trimmed] || trimmed;
|
|
1098
|
+
});
|
|
1099
|
+
return null;
|
|
1100
|
+
}
|
|
1101
|
+
const fields = parseCsvLine(cleanLine, lineNumber, finalDelimiter);
|
|
1102
|
+
headers = fields.map((_, index) => `column${index + 1}`);
|
|
1103
|
+
return processFields(fields);
|
|
1104
|
+
}
|
|
1105
|
+
const fields = parseCsvLine(cleanLine, lineNumber, finalDelimiter);
|
|
1106
|
+
return processFields(fields);
|
|
1107
|
+
};
|
|
1108
|
+
while (true) {
|
|
1109
|
+
const {
|
|
1110
|
+
value,
|
|
1111
|
+
done
|
|
1112
|
+
} = await reader.read();
|
|
1113
|
+
if (done) {
|
|
1114
|
+
break;
|
|
1115
|
+
}
|
|
1116
|
+
buffer += decoder.decode(value, {
|
|
1117
|
+
stream: true
|
|
1118
|
+
});
|
|
1119
|
+
let start = 0;
|
|
1120
|
+
for (let i = 0; i < buffer.length; i++) {
|
|
1121
|
+
const char = buffer[i];
|
|
1122
|
+
if (char === '"') {
|
|
1123
|
+
if (insideQuotes && buffer[i + 1] === '"') {
|
|
1124
|
+
i++;
|
|
1125
|
+
continue;
|
|
1126
|
+
}
|
|
1127
|
+
insideQuotes = !insideQuotes;
|
|
1128
|
+
continue;
|
|
1129
|
+
}
|
|
1130
|
+
if (char === '\n' && !insideQuotes) {
|
|
1131
|
+
const line = buffer.slice(start, i);
|
|
1132
|
+
start = i + 1;
|
|
1133
|
+
const row = processLine(line);
|
|
1134
|
+
if (row) {
|
|
1135
|
+
yield row;
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
buffer = buffer.slice(start);
|
|
1140
|
+
}
|
|
1141
|
+
if (buffer.length > 0) {
|
|
1142
|
+
const row = processLine(buffer);
|
|
1143
|
+
if (row) {
|
|
1144
|
+
yield row;
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
if (insideQuotes) {
|
|
1148
|
+
throw new ParsingError('Unclosed quotes in CSV', lineNumber);
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
913
1151
|
|
|
914
1152
|
// Экспорт для Node.js совместимости
|
|
915
1153
|
if (typeof module !== 'undefined' && module.exports) {
|
|
916
1154
|
module.exports = {
|
|
917
1155
|
csvToJson,
|
|
918
|
-
autoDetectDelimiter
|
|
1156
|
+
autoDetectDelimiter,
|
|
1157
|
+
csvToJsonIterator
|
|
1158
|
+
};
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
const DEFAULT_MAX_CHUNK_SIZE = 64 * 1024;
|
|
1162
|
+
function isReadableStream(value) {
|
|
1163
|
+
return value && typeof value.getReader === 'function';
|
|
1164
|
+
}
|
|
1165
|
+
function isAsyncIterable(value) {
|
|
1166
|
+
return value && typeof value[Symbol.asyncIterator] === 'function';
|
|
1167
|
+
}
|
|
1168
|
+
function isIterable(value) {
|
|
1169
|
+
return value && typeof value[Symbol.iterator] === 'function';
|
|
1170
|
+
}
|
|
1171
|
+
function createReadableStreamFromIterator(iterator) {
|
|
1172
|
+
return new ReadableStream({
|
|
1173
|
+
async pull(controller) {
|
|
1174
|
+
try {
|
|
1175
|
+
const {
|
|
1176
|
+
value,
|
|
1177
|
+
done
|
|
1178
|
+
} = await iterator.next();
|
|
1179
|
+
if (done) {
|
|
1180
|
+
controller.close();
|
|
1181
|
+
return;
|
|
1182
|
+
}
|
|
1183
|
+
controller.enqueue(value);
|
|
1184
|
+
} catch (error) {
|
|
1185
|
+
controller.error(error);
|
|
1186
|
+
}
|
|
1187
|
+
},
|
|
1188
|
+
cancel() {
|
|
1189
|
+
if (iterator.return) {
|
|
1190
|
+
iterator.return();
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
});
|
|
1194
|
+
}
|
|
1195
|
+
function detectInputFormat(input, options) {
|
|
1196
|
+
if (options && options.inputFormat) {
|
|
1197
|
+
return options.inputFormat;
|
|
1198
|
+
}
|
|
1199
|
+
if (typeof input === 'string') {
|
|
1200
|
+
const trimmed = input.trim();
|
|
1201
|
+
if (trimmed.startsWith('[')) {
|
|
1202
|
+
return 'json-array';
|
|
1203
|
+
}
|
|
1204
|
+
if (trimmed.includes('\n')) {
|
|
1205
|
+
return 'ndjson';
|
|
1206
|
+
}
|
|
1207
|
+
return 'json-array';
|
|
1208
|
+
}
|
|
1209
|
+
if (input instanceof Blob || isReadableStream(input)) {
|
|
1210
|
+
return 'ndjson';
|
|
1211
|
+
}
|
|
1212
|
+
return 'json-array';
|
|
1213
|
+
}
|
|
1214
|
+
async function* parseNdjsonText(text) {
|
|
1215
|
+
const lines = text.split(/\r?\n/);
|
|
1216
|
+
for (const line of lines) {
|
|
1217
|
+
const trimmed = line.trim();
|
|
1218
|
+
if (!trimmed) {
|
|
1219
|
+
continue;
|
|
1220
|
+
}
|
|
1221
|
+
yield JSON.parse(trimmed);
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
async function* parseNdjsonStream(stream) {
|
|
1225
|
+
const reader = stream.getReader();
|
|
1226
|
+
const decoder = new TextDecoder('utf-8');
|
|
1227
|
+
let buffer = '';
|
|
1228
|
+
while (true) {
|
|
1229
|
+
const {
|
|
1230
|
+
value,
|
|
1231
|
+
done
|
|
1232
|
+
} = await reader.read();
|
|
1233
|
+
if (done) {
|
|
1234
|
+
break;
|
|
1235
|
+
}
|
|
1236
|
+
buffer += decoder.decode(value, {
|
|
1237
|
+
stream: true
|
|
1238
|
+
});
|
|
1239
|
+
const lines = buffer.split(/\r?\n/);
|
|
1240
|
+
buffer = lines.pop() || '';
|
|
1241
|
+
for (const line of lines) {
|
|
1242
|
+
const trimmed = line.trim();
|
|
1243
|
+
if (!trimmed) {
|
|
1244
|
+
continue;
|
|
1245
|
+
}
|
|
1246
|
+
yield JSON.parse(trimmed);
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
if (buffer.trim()) {
|
|
1250
|
+
yield JSON.parse(buffer.trim());
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
async function* normalizeJsonInput(input, options = {}) {
|
|
1254
|
+
const format = detectInputFormat(input, options);
|
|
1255
|
+
if (Array.isArray(input)) {
|
|
1256
|
+
for (const item of input) {
|
|
1257
|
+
yield item;
|
|
1258
|
+
}
|
|
1259
|
+
return;
|
|
1260
|
+
}
|
|
1261
|
+
if (isAsyncIterable(input)) {
|
|
1262
|
+
for await (const item of input) {
|
|
1263
|
+
yield item;
|
|
1264
|
+
}
|
|
1265
|
+
return;
|
|
1266
|
+
}
|
|
1267
|
+
if (isIterable(input)) {
|
|
1268
|
+
for (const item of input) {
|
|
1269
|
+
yield item;
|
|
1270
|
+
}
|
|
1271
|
+
return;
|
|
1272
|
+
}
|
|
1273
|
+
if (typeof input === 'string') {
|
|
1274
|
+
if (format === 'ndjson') {
|
|
1275
|
+
yield* parseNdjsonText(input);
|
|
1276
|
+
return;
|
|
1277
|
+
}
|
|
1278
|
+
const parsed = JSON.parse(input);
|
|
1279
|
+
if (Array.isArray(parsed)) {
|
|
1280
|
+
for (const item of parsed) {
|
|
1281
|
+
yield item;
|
|
1282
|
+
}
|
|
1283
|
+
return;
|
|
1284
|
+
}
|
|
1285
|
+
yield parsed;
|
|
1286
|
+
return;
|
|
1287
|
+
}
|
|
1288
|
+
if (input instanceof Blob) {
|
|
1289
|
+
if (format === 'ndjson') {
|
|
1290
|
+
yield* parseNdjsonStream(input.stream());
|
|
1291
|
+
return;
|
|
1292
|
+
}
|
|
1293
|
+
const text = await input.text();
|
|
1294
|
+
const parsed = JSON.parse(text);
|
|
1295
|
+
if (Array.isArray(parsed)) {
|
|
1296
|
+
for (const item of parsed) {
|
|
1297
|
+
yield item;
|
|
1298
|
+
}
|
|
1299
|
+
return;
|
|
1300
|
+
}
|
|
1301
|
+
yield parsed;
|
|
1302
|
+
return;
|
|
1303
|
+
}
|
|
1304
|
+
if (isReadableStream(input)) {
|
|
1305
|
+
if (format !== 'ndjson') {
|
|
1306
|
+
throw new ValidationError('ReadableStream input requires inputFormat="ndjson"');
|
|
1307
|
+
}
|
|
1308
|
+
yield* parseNdjsonStream(input);
|
|
1309
|
+
return;
|
|
1310
|
+
}
|
|
1311
|
+
throw new ValidationError('Input must be an array, iterable, string, Blob, or ReadableStream');
|
|
1312
|
+
}
|
|
1313
|
+
function validateStreamOptions(options) {
|
|
1314
|
+
if (options && typeof options !== 'object') {
|
|
1315
|
+
throw new ConfigurationError('Options must be an object');
|
|
1316
|
+
}
|
|
1317
|
+
if (options?.delimiter && typeof options.delimiter !== 'string') {
|
|
1318
|
+
throw new ConfigurationError('Delimiter must be a string');
|
|
1319
|
+
}
|
|
1320
|
+
if (options?.delimiter && options.delimiter.length !== 1) {
|
|
1321
|
+
throw new ConfigurationError('Delimiter must be a single character');
|
|
1322
|
+
}
|
|
1323
|
+
if (options?.renameMap && typeof options.renameMap !== 'object') {
|
|
1324
|
+
throw new ConfigurationError('renameMap must be an object');
|
|
1325
|
+
}
|
|
1326
|
+
if (options?.maxRecords !== undefined) {
|
|
1327
|
+
if (typeof options.maxRecords !== 'number' || options.maxRecords <= 0) {
|
|
1328
|
+
throw new ConfigurationError('maxRecords must be a positive number');
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
function escapeCsvValue(value, options) {
|
|
1333
|
+
const {
|
|
1334
|
+
delimiter,
|
|
1335
|
+
preventCsvInjection = true,
|
|
1336
|
+
rfc4180Compliant = true
|
|
1337
|
+
} = options;
|
|
1338
|
+
if (value === null || value === undefined || value === '') {
|
|
1339
|
+
return '';
|
|
1340
|
+
}
|
|
1341
|
+
const stringValue = String(value);
|
|
1342
|
+
let escapedValue = stringValue;
|
|
1343
|
+
if (preventCsvInjection && /^[=+\-@]/.test(stringValue)) {
|
|
1344
|
+
escapedValue = "'" + stringValue;
|
|
1345
|
+
}
|
|
1346
|
+
const needsQuoting = rfc4180Compliant ? escapedValue.includes(delimiter) || escapedValue.includes('"') || escapedValue.includes('\n') || escapedValue.includes('\r') : escapedValue.includes(delimiter) || escapedValue.includes('"') || escapedValue.includes('\n') || escapedValue.includes('\r');
|
|
1347
|
+
if (needsQuoting) {
|
|
1348
|
+
return `"${escapedValue.replace(/"/g, '""')}"`;
|
|
1349
|
+
}
|
|
1350
|
+
return escapedValue;
|
|
1351
|
+
}
|
|
1352
|
+
function buildHeaderState(keys, options) {
|
|
1353
|
+
const renameMap = options.renameMap || {};
|
|
1354
|
+
const template = options.template || {};
|
|
1355
|
+
const originalKeys = Array.isArray(options.headers) ? options.headers : keys;
|
|
1356
|
+
const headers = originalKeys.map(key => renameMap[key] || key);
|
|
1357
|
+
const reverseRenameMap = {};
|
|
1358
|
+
originalKeys.forEach((key, index) => {
|
|
1359
|
+
reverseRenameMap[headers[index]] = key;
|
|
1360
|
+
});
|
|
1361
|
+
let finalHeaders = headers;
|
|
1362
|
+
if (Object.keys(template).length > 0) {
|
|
1363
|
+
const templateHeaders = Object.keys(template).map(key => renameMap[key] || key);
|
|
1364
|
+
const extraHeaders = headers.filter(h => !templateHeaders.includes(h));
|
|
1365
|
+
finalHeaders = [...templateHeaders, ...extraHeaders];
|
|
1366
|
+
}
|
|
1367
|
+
return {
|
|
1368
|
+
headers: finalHeaders,
|
|
1369
|
+
reverseRenameMap
|
|
1370
|
+
};
|
|
1371
|
+
}
|
|
1372
|
+
async function* jsonToCsvChunkIterator(input, options = {}) {
|
|
1373
|
+
validateStreamOptions(options);
|
|
1374
|
+
const opts = options && typeof options === 'object' ? options : {};
|
|
1375
|
+
const {
|
|
1376
|
+
delimiter = ';',
|
|
1377
|
+
includeHeaders = true,
|
|
1378
|
+
maxRecords,
|
|
1379
|
+
maxChunkSize = DEFAULT_MAX_CHUNK_SIZE,
|
|
1380
|
+
headerMode
|
|
1381
|
+
} = opts;
|
|
1382
|
+
let headerState = null;
|
|
1383
|
+
let buffer = '';
|
|
1384
|
+
let recordCount = 0;
|
|
1385
|
+
const lineEnding = opts.rfc4180Compliant === false ? '\n' : '\r\n';
|
|
1386
|
+
if (Array.isArray(input) && !opts.headers && (!headerMode || headerMode === 'all')) {
|
|
1387
|
+
const allKeys = new Set();
|
|
1388
|
+
for (const item of input) {
|
|
1389
|
+
if (!item || typeof item !== 'object') {
|
|
1390
|
+
continue;
|
|
1391
|
+
}
|
|
1392
|
+
Object.keys(item).forEach(key => allKeys.add(key));
|
|
1393
|
+
}
|
|
1394
|
+
headerState = buildHeaderState(Array.from(allKeys), opts);
|
|
1395
|
+
if (includeHeaders && headerState.headers.length > 0) {
|
|
1396
|
+
buffer += headerState.headers.join(delimiter) + lineEnding;
|
|
1397
|
+
}
|
|
1398
|
+
} else if (Array.isArray(opts.headers)) {
|
|
1399
|
+
headerState = buildHeaderState(opts.headers, opts);
|
|
1400
|
+
if (includeHeaders && headerState.headers.length > 0) {
|
|
1401
|
+
buffer += headerState.headers.join(delimiter) + lineEnding;
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
for await (const item of normalizeJsonInput(input, opts)) {
|
|
1405
|
+
if (!item || typeof item !== 'object') {
|
|
1406
|
+
continue;
|
|
1407
|
+
}
|
|
1408
|
+
if (!headerState) {
|
|
1409
|
+
headerState = buildHeaderState(Object.keys(item), opts);
|
|
1410
|
+
if (includeHeaders && headerState.headers.length > 0) {
|
|
1411
|
+
buffer += headerState.headers.join(delimiter) + lineEnding;
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
recordCount += 1;
|
|
1415
|
+
if (maxRecords && recordCount > maxRecords) {
|
|
1416
|
+
throw new LimitError(`Data size exceeds maximum limit of ${maxRecords} records`, maxRecords, recordCount);
|
|
1417
|
+
}
|
|
1418
|
+
const row = headerState.headers.map(header => {
|
|
1419
|
+
const originalKey = headerState.reverseRenameMap[header] || header;
|
|
1420
|
+
return escapeCsvValue(item[originalKey], {
|
|
1421
|
+
delimiter,
|
|
1422
|
+
preventCsvInjection: opts.preventCsvInjection !== false,
|
|
1423
|
+
rfc4180Compliant: opts.rfc4180Compliant !== false
|
|
1424
|
+
});
|
|
1425
|
+
}).join(delimiter);
|
|
1426
|
+
buffer += row + lineEnding;
|
|
1427
|
+
if (buffer.length >= maxChunkSize) {
|
|
1428
|
+
yield buffer;
|
|
1429
|
+
buffer = '';
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
if (buffer.length > 0) {
|
|
1433
|
+
yield buffer;
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
async function* jsonToNdjsonChunkIterator(input, options = {}) {
|
|
1437
|
+
validateStreamOptions(options);
|
|
1438
|
+
for await (const item of normalizeJsonInput(input, options)) {
|
|
1439
|
+
if (item === undefined) {
|
|
1440
|
+
continue;
|
|
1441
|
+
}
|
|
1442
|
+
yield JSON.stringify(item) + '\n';
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
async function* csvToJsonChunkIterator(input, options = {}) {
|
|
1446
|
+
const outputFormat = options.outputFormat || 'ndjson';
|
|
1447
|
+
const asArray = outputFormat === 'json-array' || outputFormat === 'array' || outputFormat === 'json';
|
|
1448
|
+
let first = true;
|
|
1449
|
+
if (asArray) {
|
|
1450
|
+
yield '[';
|
|
1451
|
+
}
|
|
1452
|
+
for await (const row of csvToJsonIterator(input, options)) {
|
|
1453
|
+
const payload = JSON.stringify(row);
|
|
1454
|
+
if (asArray) {
|
|
1455
|
+
yield (first ? '' : ',') + payload;
|
|
1456
|
+
} else {
|
|
1457
|
+
yield payload + '\n';
|
|
1458
|
+
}
|
|
1459
|
+
first = false;
|
|
1460
|
+
}
|
|
1461
|
+
if (asArray) {
|
|
1462
|
+
yield ']';
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
function jsonToCsvStream(input, options = {}) {
|
|
1466
|
+
const iterator = jsonToCsvChunkIterator(input, options);
|
|
1467
|
+
return createReadableStreamFromIterator(iterator);
|
|
1468
|
+
}
|
|
1469
|
+
function jsonToNdjsonStream(input, options = {}) {
|
|
1470
|
+
const iterator = jsonToNdjsonChunkIterator(input, options);
|
|
1471
|
+
return createReadableStreamFromIterator(iterator);
|
|
1472
|
+
}
|
|
1473
|
+
function csvToJsonStream(input, options = {}) {
|
|
1474
|
+
const iterator = csvToJsonChunkIterator(input, options);
|
|
1475
|
+
return createReadableStreamFromIterator(iterator);
|
|
1476
|
+
}
|
|
1477
|
+
if (typeof module !== 'undefined' && module.exports) {
|
|
1478
|
+
module.exports = {
|
|
1479
|
+
jsonToCsvStream,
|
|
1480
|
+
jsonToNdjsonStream,
|
|
1481
|
+
csvToJsonStream
|
|
919
1482
|
};
|
|
920
1483
|
}
|
|
921
1484
|
|
|
@@ -1040,6 +1603,26 @@
|
|
|
1040
1603
|
});
|
|
1041
1604
|
}
|
|
1042
1605
|
|
|
1606
|
+
/**
|
|
1607
|
+
* Stream CSV file as async iterator without full buffering.
|
|
1608
|
+
*
|
|
1609
|
+
* @param {File} file - File selected from input
|
|
1610
|
+
* @param {Object} [options] - csvToJson options
|
|
1611
|
+
* @returns {AsyncGenerator<Object>} Async iterator of rows
|
|
1612
|
+
*/
|
|
1613
|
+
function parseCsvFileStream(file, options = {}) {
|
|
1614
|
+
if (typeof window === 'undefined') {
|
|
1615
|
+
throw new ValidationError('parseCsvFileStream() is browser-only. Use readCsvAsJson() in Node.js');
|
|
1616
|
+
}
|
|
1617
|
+
if (!(file instanceof File)) {
|
|
1618
|
+
throw new ValidationError('Input must be a File object');
|
|
1619
|
+
}
|
|
1620
|
+
if (!file.name.toLowerCase().endsWith('.csv')) {
|
|
1621
|
+
throw new ValidationError('File must have .csv extension');
|
|
1622
|
+
}
|
|
1623
|
+
return csvToJsonIterator(file, options);
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1043
1626
|
/**
|
|
1044
1627
|
* Создает CSV файл из JSON данных (альтернатива downloadAsCsv)
|
|
1045
1628
|
* Возвращает Blob вместо автоматического скачивания
|
|
@@ -1089,8 +1672,12 @@
|
|
|
1089
1672
|
module.exports = {
|
|
1090
1673
|
downloadAsCsv,
|
|
1091
1674
|
parseCsvFile,
|
|
1675
|
+
parseCsvFileStream,
|
|
1092
1676
|
createCsvBlob,
|
|
1093
|
-
parseCsvBlob
|
|
1677
|
+
parseCsvBlob,
|
|
1678
|
+
jsonToCsvStream,
|
|
1679
|
+
jsonToNdjsonStream,
|
|
1680
|
+
csvToJsonStream
|
|
1094
1681
|
};
|
|
1095
1682
|
}
|
|
1096
1683
|
|
|
@@ -1100,44 +1687,74 @@
|
|
|
1100
1687
|
|
|
1101
1688
|
// Проверка поддержки Web Workers
|
|
1102
1689
|
const WORKERS_SUPPORTED = typeof Worker !== 'undefined';
|
|
1690
|
+
function isTransferableBuffer(value) {
|
|
1691
|
+
if (!(value instanceof ArrayBuffer)) {
|
|
1692
|
+
return false;
|
|
1693
|
+
}
|
|
1694
|
+
if (typeof SharedArrayBuffer !== 'undefined' && value instanceof SharedArrayBuffer) {
|
|
1695
|
+
return false;
|
|
1696
|
+
}
|
|
1697
|
+
return true;
|
|
1698
|
+
}
|
|
1699
|
+
function collectTransferables(args) {
|
|
1700
|
+
const transferables = [];
|
|
1701
|
+
const collectFromValue = value => {
|
|
1702
|
+
if (!value) {
|
|
1703
|
+
return;
|
|
1704
|
+
}
|
|
1705
|
+
if (isTransferableBuffer(value)) {
|
|
1706
|
+
transferables.push(value);
|
|
1707
|
+
return;
|
|
1708
|
+
}
|
|
1709
|
+
if (ArrayBuffer.isView(value) && isTransferableBuffer(value.buffer)) {
|
|
1710
|
+
transferables.push(value.buffer);
|
|
1711
|
+
return;
|
|
1712
|
+
}
|
|
1713
|
+
if (Array.isArray(value)) {
|
|
1714
|
+
value.forEach(collectFromValue);
|
|
1715
|
+
}
|
|
1716
|
+
};
|
|
1717
|
+
args.forEach(collectFromValue);
|
|
1718
|
+
return transferables.length ? transferables : null;
|
|
1719
|
+
}
|
|
1103
1720
|
|
|
1104
|
-
/**
|
|
1105
|
-
* Опции для Worker Pool
|
|
1106
|
-
* @typedef {Object} WorkerPoolOptions
|
|
1107
|
-
* @property {number} [workerCount=4] - Количество workers в pool
|
|
1108
|
-
* @property {number} [maxQueueSize=100] - Максимальный размер очереди задач
|
|
1109
|
-
* @property {boolean} [autoScale=true] - Автоматическое масштабирование pool
|
|
1110
|
-
* @property {number} [idleTimeout=60000] - Таймаут простоя worker (мс)
|
|
1721
|
+
/**
|
|
1722
|
+
* Опции для Worker Pool
|
|
1723
|
+
* @typedef {Object} WorkerPoolOptions
|
|
1724
|
+
* @property {number} [workerCount=4] - Количество workers в pool
|
|
1725
|
+
* @property {number} [maxQueueSize=100] - Максимальный размер очереди задач
|
|
1726
|
+
* @property {boolean} [autoScale=true] - Автоматическое масштабирование pool
|
|
1727
|
+
* @property {number} [idleTimeout=60000] - Таймаут простоя worker (мс)
|
|
1111
1728
|
*/
|
|
1112
1729
|
|
|
1113
|
-
/**
|
|
1114
|
-
* Статистика Worker Pool
|
|
1115
|
-
* @typedef {Object} WorkerPoolStats
|
|
1116
|
-
* @property {number} totalWorkers - Всего workers
|
|
1117
|
-
* @property {number} activeWorkers - Активные workers
|
|
1118
|
-
* @property {number} idleWorkers - Простаивающие workers
|
|
1119
|
-
* @property {number} queueSize - Размер очереди
|
|
1120
|
-
* @property {number} tasksCompleted - Завершенные задачи
|
|
1121
|
-
* @property {number} tasksFailed - Неудачные задачи
|
|
1730
|
+
/**
|
|
1731
|
+
* Статистика Worker Pool
|
|
1732
|
+
* @typedef {Object} WorkerPoolStats
|
|
1733
|
+
* @property {number} totalWorkers - Всего workers
|
|
1734
|
+
* @property {number} activeWorkers - Активные workers
|
|
1735
|
+
* @property {number} idleWorkers - Простаивающие workers
|
|
1736
|
+
* @property {number} queueSize - Размер очереди
|
|
1737
|
+
* @property {number} tasksCompleted - Завершенные задачи
|
|
1738
|
+
* @property {number} tasksFailed - Неудачные задачи
|
|
1122
1739
|
*/
|
|
1123
1740
|
|
|
1124
|
-
/**
|
|
1125
|
-
* Прогресс обработки задачи
|
|
1126
|
-
* @typedef {Object} TaskProgress
|
|
1127
|
-
* @property {number} processed - Обработано элементов
|
|
1128
|
-
* @property {number} total - Всего элементов
|
|
1129
|
-
* @property {number} percentage - Процент выполнения
|
|
1130
|
-
* @property {number} speed - Скорость обработки (элементов/сек)
|
|
1741
|
+
/**
|
|
1742
|
+
* Прогресс обработки задачи
|
|
1743
|
+
* @typedef {Object} TaskProgress
|
|
1744
|
+
* @property {number} processed - Обработано элементов
|
|
1745
|
+
* @property {number} total - Всего элементов
|
|
1746
|
+
* @property {number} percentage - Процент выполнения
|
|
1747
|
+
* @property {number} speed - Скорость обработки (элементов/сек)
|
|
1131
1748
|
*/
|
|
1132
1749
|
|
|
1133
|
-
/**
|
|
1134
|
-
* Worker Pool для параллельной обработки CSV
|
|
1750
|
+
/**
|
|
1751
|
+
* Worker Pool для параллельной обработки CSV
|
|
1135
1752
|
*/
|
|
1136
1753
|
class WorkerPool {
|
|
1137
|
-
/**
|
|
1138
|
-
* Создает новый Worker Pool
|
|
1139
|
-
* @param {string} workerScript - URL скрипта worker
|
|
1140
|
-
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
1754
|
+
/**
|
|
1755
|
+
* Создает новый Worker Pool
|
|
1756
|
+
* @param {string} workerScript - URL скрипта worker
|
|
1757
|
+
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
1141
1758
|
*/
|
|
1142
1759
|
constructor(workerScript, options = {}) {
|
|
1143
1760
|
if (!WORKERS_SUPPORTED) {
|
|
@@ -1165,9 +1782,9 @@
|
|
|
1165
1782
|
this.initializeWorkers();
|
|
1166
1783
|
}
|
|
1167
1784
|
|
|
1168
|
-
/**
|
|
1169
|
-
* Инициализация workers
|
|
1170
|
-
* @private
|
|
1785
|
+
/**
|
|
1786
|
+
* Инициализация workers
|
|
1787
|
+
* @private
|
|
1171
1788
|
*/
|
|
1172
1789
|
initializeWorkers() {
|
|
1173
1790
|
const {
|
|
@@ -1179,9 +1796,9 @@
|
|
|
1179
1796
|
this.updateStats();
|
|
1180
1797
|
}
|
|
1181
1798
|
|
|
1182
|
-
/**
|
|
1183
|
-
* Создает нового worker
|
|
1184
|
-
* @private
|
|
1799
|
+
/**
|
|
1800
|
+
* Создает нового worker
|
|
1801
|
+
* @private
|
|
1185
1802
|
*/
|
|
1186
1803
|
createWorker() {
|
|
1187
1804
|
try {
|
|
@@ -1206,9 +1823,9 @@
|
|
|
1206
1823
|
}
|
|
1207
1824
|
}
|
|
1208
1825
|
|
|
1209
|
-
/**
|
|
1210
|
-
* Обработка сообщений от worker
|
|
1211
|
-
* @private
|
|
1826
|
+
/**
|
|
1827
|
+
* Обработка сообщений от worker
|
|
1828
|
+
* @private
|
|
1212
1829
|
*/
|
|
1213
1830
|
handleWorkerMessage(worker, event) {
|
|
1214
1831
|
const {
|
|
@@ -1223,9 +1840,9 @@
|
|
|
1223
1840
|
}
|
|
1224
1841
|
}
|
|
1225
1842
|
|
|
1226
|
-
/**
|
|
1227
|
-
* Обработка прогресса задачи
|
|
1228
|
-
* @private
|
|
1843
|
+
/**
|
|
1844
|
+
* Обработка прогресса задачи
|
|
1845
|
+
* @private
|
|
1229
1846
|
*/
|
|
1230
1847
|
handleProgress(worker, progressData) {
|
|
1231
1848
|
const taskId = worker.taskId;
|
|
@@ -1242,9 +1859,9 @@
|
|
|
1242
1859
|
}
|
|
1243
1860
|
}
|
|
1244
1861
|
|
|
1245
|
-
/**
|
|
1246
|
-
* Обработка результата задачи
|
|
1247
|
-
* @private
|
|
1862
|
+
/**
|
|
1863
|
+
* Обработка результата задачи
|
|
1864
|
+
* @private
|
|
1248
1865
|
*/
|
|
1249
1866
|
handleResult(worker, resultData) {
|
|
1250
1867
|
const taskId = worker.taskId;
|
|
@@ -1269,9 +1886,9 @@
|
|
|
1269
1886
|
}
|
|
1270
1887
|
}
|
|
1271
1888
|
|
|
1272
|
-
/**
|
|
1273
|
-
* Обработка ошибки задачи
|
|
1274
|
-
* @private
|
|
1889
|
+
/**
|
|
1890
|
+
* Обработка ошибки задачи
|
|
1891
|
+
* @private
|
|
1275
1892
|
*/
|
|
1276
1893
|
handleWorkerTaskError(worker, errorData) {
|
|
1277
1894
|
const taskId = worker.taskId;
|
|
@@ -1286,7 +1903,14 @@
|
|
|
1286
1903
|
this.stats.idleWorkers++;
|
|
1287
1904
|
|
|
1288
1905
|
// Завершение с ошибкой
|
|
1289
|
-
|
|
1906
|
+
const workerError = new Error(errorData.message || 'Ошибка в worker');
|
|
1907
|
+
if (errorData.code) {
|
|
1908
|
+
workerError.code = errorData.code;
|
|
1909
|
+
}
|
|
1910
|
+
if (errorData.details) {
|
|
1911
|
+
workerError.details = errorData.details;
|
|
1912
|
+
}
|
|
1913
|
+
task.reject(workerError);
|
|
1290
1914
|
this.activeTasks.delete(taskId);
|
|
1291
1915
|
this.stats.tasksFailed++;
|
|
1292
1916
|
|
|
@@ -1296,9 +1920,9 @@
|
|
|
1296
1920
|
}
|
|
1297
1921
|
}
|
|
1298
1922
|
|
|
1299
|
-
/**
|
|
1300
|
-
* Обработка ошибок worker
|
|
1301
|
-
* @private
|
|
1923
|
+
/**
|
|
1924
|
+
* Обработка ошибок worker
|
|
1925
|
+
* @private
|
|
1302
1926
|
*/
|
|
1303
1927
|
handleWorkerError(worker, error) {
|
|
1304
1928
|
console.error(`Worker ${worker.id} error:`, error);
|
|
@@ -1307,17 +1931,17 @@
|
|
|
1307
1931
|
this.restartWorker(worker);
|
|
1308
1932
|
}
|
|
1309
1933
|
|
|
1310
|
-
/**
|
|
1311
|
-
* Обработка ошибок сообщений
|
|
1312
|
-
* @private
|
|
1934
|
+
/**
|
|
1935
|
+
* Обработка ошибок сообщений
|
|
1936
|
+
* @private
|
|
1313
1937
|
*/
|
|
1314
1938
|
handleWorkerMessageError(worker, error) {
|
|
1315
1939
|
console.error(`Worker ${worker.id} message error:`, error);
|
|
1316
1940
|
}
|
|
1317
1941
|
|
|
1318
|
-
/**
|
|
1319
|
-
* Перезапуск worker
|
|
1320
|
-
* @private
|
|
1942
|
+
/**
|
|
1943
|
+
* Перезапуск worker
|
|
1944
|
+
* @private
|
|
1321
1945
|
*/
|
|
1322
1946
|
restartWorker(worker) {
|
|
1323
1947
|
const index = this.workers.indexOf(worker);
|
|
@@ -1345,9 +1969,9 @@
|
|
|
1345
1969
|
}
|
|
1346
1970
|
}
|
|
1347
1971
|
|
|
1348
|
-
/**
|
|
1349
|
-
* Выполнение задачи на worker
|
|
1350
|
-
* @private
|
|
1972
|
+
/**
|
|
1973
|
+
* Выполнение задачи на worker
|
|
1974
|
+
* @private
|
|
1351
1975
|
*/
|
|
1352
1976
|
executeTask(worker, task) {
|
|
1353
1977
|
worker.status = 'active';
|
|
@@ -1357,59 +1981,59 @@
|
|
|
1357
1981
|
this.stats.activeWorkers++;
|
|
1358
1982
|
|
|
1359
1983
|
// Отправка задачи в worker
|
|
1360
|
-
|
|
1984
|
+
const payload = {
|
|
1361
1985
|
type: 'EXECUTE',
|
|
1362
1986
|
taskId: task.id,
|
|
1363
1987
|
method: task.method,
|
|
1364
1988
|
args: task.args,
|
|
1365
1989
|
options: task.options
|
|
1366
|
-
}
|
|
1990
|
+
};
|
|
1991
|
+
if (task.transferList && task.transferList.length) {
|
|
1992
|
+
worker.postMessage(payload, task.transferList);
|
|
1993
|
+
} else {
|
|
1994
|
+
worker.postMessage(payload);
|
|
1995
|
+
}
|
|
1367
1996
|
}
|
|
1368
1997
|
|
|
1369
|
-
/**
|
|
1370
|
-
* Обработка очереди задач
|
|
1371
|
-
* @private
|
|
1998
|
+
/**
|
|
1999
|
+
* Обработка очереди задач
|
|
2000
|
+
* @private
|
|
1372
2001
|
*/
|
|
1373
2002
|
processQueue() {
|
|
1374
2003
|
if (this.taskQueue.length === 0) {
|
|
1375
2004
|
return;
|
|
1376
2005
|
}
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
2006
|
+
while (this.taskQueue.length > 0) {
|
|
2007
|
+
const idleWorker = this.workers.find(w => w.status === 'idle');
|
|
2008
|
+
if (!idleWorker) {
|
|
2009
|
+
if (this.options.autoScale && this.workers.length < this.options.maxQueueSize) {
|
|
2010
|
+
this.createWorker();
|
|
2011
|
+
continue;
|
|
2012
|
+
}
|
|
2013
|
+
break;
|
|
1385
2014
|
}
|
|
1386
|
-
|
|
2015
|
+
const task = this.taskQueue.shift();
|
|
2016
|
+
this.stats.queueSize--;
|
|
2017
|
+
this.executeTask(idleWorker, task);
|
|
1387
2018
|
}
|
|
1388
|
-
|
|
1389
|
-
// Получение задачи из очереди
|
|
1390
|
-
const task = this.taskQueue.shift();
|
|
1391
|
-
this.stats.queueSize--;
|
|
1392
|
-
|
|
1393
|
-
// Выполнение задачи
|
|
1394
|
-
this.executeTask(idleWorker, task);
|
|
1395
2019
|
this.updateStats();
|
|
1396
2020
|
}
|
|
1397
2021
|
|
|
1398
|
-
/**
|
|
1399
|
-
* Обновление статистики
|
|
1400
|
-
* @private
|
|
2022
|
+
/**
|
|
2023
|
+
* Обновление статистики
|
|
2024
|
+
* @private
|
|
1401
2025
|
*/
|
|
1402
2026
|
updateStats() {
|
|
1403
2027
|
this.stats.queueSize = this.taskQueue.length;
|
|
1404
2028
|
}
|
|
1405
2029
|
|
|
1406
|
-
/**
|
|
1407
|
-
* Выполнение задачи через pool
|
|
1408
|
-
* @param {string} method - Метод для вызова в worker
|
|
1409
|
-
* @param {Array} args - Аргументы метода
|
|
1410
|
-
* @param {Object} [options] - Опции задачи
|
|
1411
|
-
* @param {Function} [onProgress] - Callback прогресса
|
|
1412
|
-
* @returns {Promise<any>} Результат выполнения
|
|
2030
|
+
/**
|
|
2031
|
+
* Выполнение задачи через pool
|
|
2032
|
+
* @param {string} method - Метод для вызова в worker
|
|
2033
|
+
* @param {Array} args - Аргументы метода
|
|
2034
|
+
* @param {Object} [options] - Опции задачи
|
|
2035
|
+
* @param {Function} [onProgress] - Callback прогресса
|
|
2036
|
+
* @returns {Promise<any>} Результат выполнения
|
|
1413
2037
|
*/
|
|
1414
2038
|
async exec(method, args = [], options = {}, onProgress = null) {
|
|
1415
2039
|
return new Promise((resolve, reject) => {
|
|
@@ -1421,11 +2045,17 @@
|
|
|
1421
2045
|
|
|
1422
2046
|
// Создание задачи
|
|
1423
2047
|
const taskId = `task-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
2048
|
+
const {
|
|
2049
|
+
transfer,
|
|
2050
|
+
...taskOptions
|
|
2051
|
+
} = options || {};
|
|
2052
|
+
const transferList = transfer || collectTransferables(args);
|
|
1424
2053
|
const task = {
|
|
1425
2054
|
id: taskId,
|
|
1426
2055
|
method,
|
|
1427
2056
|
args,
|
|
1428
|
-
options,
|
|
2057
|
+
options: taskOptions,
|
|
2058
|
+
transferList,
|
|
1429
2059
|
onProgress,
|
|
1430
2060
|
resolve,
|
|
1431
2061
|
reject,
|
|
@@ -1442,9 +2072,9 @@
|
|
|
1442
2072
|
});
|
|
1443
2073
|
}
|
|
1444
2074
|
|
|
1445
|
-
/**
|
|
1446
|
-
* Получение статистики pool
|
|
1447
|
-
* @returns {WorkerPoolStats} Статистика
|
|
2075
|
+
/**
|
|
2076
|
+
* Получение статистики pool
|
|
2077
|
+
* @returns {WorkerPoolStats} Статистика
|
|
1448
2078
|
*/
|
|
1449
2079
|
getStats() {
|
|
1450
2080
|
return {
|
|
@@ -1452,8 +2082,8 @@
|
|
|
1452
2082
|
};
|
|
1453
2083
|
}
|
|
1454
2084
|
|
|
1455
|
-
/**
|
|
1456
|
-
* Очистка простаивающих workers
|
|
2085
|
+
/**
|
|
2086
|
+
* Очистка простаивающих workers
|
|
1457
2087
|
*/
|
|
1458
2088
|
cleanupIdleWorkers() {
|
|
1459
2089
|
const now = Date.now();
|
|
@@ -1474,8 +2104,8 @@
|
|
|
1474
2104
|
}
|
|
1475
2105
|
}
|
|
1476
2106
|
|
|
1477
|
-
/**
|
|
1478
|
-
* Завершение всех workers
|
|
2107
|
+
/**
|
|
2108
|
+
* Завершение всех workers
|
|
1479
2109
|
*/
|
|
1480
2110
|
terminate() {
|
|
1481
2111
|
this.workers.forEach(worker => {
|
|
@@ -1497,10 +2127,10 @@
|
|
|
1497
2127
|
}
|
|
1498
2128
|
}
|
|
1499
2129
|
|
|
1500
|
-
/**
|
|
1501
|
-
* Создает Worker Pool для обработки CSV
|
|
1502
|
-
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
1503
|
-
* @returns {WorkerPool} Worker Pool
|
|
2130
|
+
/**
|
|
2131
|
+
* Создает Worker Pool для обработки CSV
|
|
2132
|
+
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
2133
|
+
* @returns {WorkerPool} Worker Pool
|
|
1504
2134
|
*/
|
|
1505
2135
|
function createWorkerPool(options = {}) {
|
|
1506
2136
|
// Используем встроенный worker скрипт
|
|
@@ -1508,12 +2138,12 @@
|
|
|
1508
2138
|
return new WorkerPool(workerScript, options);
|
|
1509
2139
|
}
|
|
1510
2140
|
|
|
1511
|
-
/**
|
|
1512
|
-
* Парсит CSV с использованием Web Workers
|
|
1513
|
-
* @param {string|File} csvInput - CSV строка или File объект
|
|
1514
|
-
* @param {Object} [options] - Опции парсинга
|
|
1515
|
-
* @param {Function} [onProgress] - Callback прогресса
|
|
1516
|
-
* @returns {Promise<Array<Object>>} JSON данные
|
|
2141
|
+
/**
|
|
2142
|
+
* Парсит CSV с использованием Web Workers
|
|
2143
|
+
* @param {string|File} csvInput - CSV строка или File объект
|
|
2144
|
+
* @param {Object} [options] - Опции парсинга
|
|
2145
|
+
* @param {Function} [onProgress] - Callback прогресса
|
|
2146
|
+
* @returns {Promise<Array<Object>>} JSON данные
|
|
1517
2147
|
*/
|
|
1518
2148
|
async function parseCSVWithWorker(csvInput, options = {}, onProgress = null) {
|
|
1519
2149
|
// Создание pool если нужно
|
|
@@ -1523,29 +2153,42 @@
|
|
|
1523
2153
|
const pool = parseCSVWithWorker.pool;
|
|
1524
2154
|
|
|
1525
2155
|
// Подготовка CSV строки
|
|
1526
|
-
|
|
2156
|
+
// ?????????? CSV ??????
|
|
2157
|
+
let csvPayload = csvInput;
|
|
2158
|
+
let transfer = null;
|
|
1527
2159
|
if (csvInput instanceof File) {
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
} else {
|
|
1532
|
-
|
|
2160
|
+
const buffer = await readFileAsArrayBuffer(csvInput);
|
|
2161
|
+
csvPayload = new Uint8Array(buffer);
|
|
2162
|
+
transfer = [buffer];
|
|
2163
|
+
} else if (csvInput instanceof ArrayBuffer) {
|
|
2164
|
+
csvPayload = csvInput;
|
|
2165
|
+
transfer = [csvInput];
|
|
2166
|
+
} else if (ArrayBuffer.isView(csvInput)) {
|
|
2167
|
+
csvPayload = csvInput;
|
|
2168
|
+
if (csvInput.buffer instanceof ArrayBuffer) {
|
|
2169
|
+
transfer = [csvInput.buffer];
|
|
2170
|
+
}
|
|
2171
|
+
} else if (typeof csvInput !== 'string') {
|
|
2172
|
+
throw new ValidationError('Input must be a CSV string, File, or ArrayBuffer');
|
|
1533
2173
|
}
|
|
1534
2174
|
|
|
1535
|
-
//
|
|
1536
|
-
|
|
2175
|
+
// ????????? ?????? ????? pool
|
|
2176
|
+
const execOptions = transfer ? {
|
|
2177
|
+
transfer
|
|
2178
|
+
} : {};
|
|
2179
|
+
return pool.exec('parseCSV', [csvPayload, options], execOptions, onProgress);
|
|
1537
2180
|
}
|
|
1538
2181
|
|
|
1539
|
-
/**
|
|
1540
|
-
* Чтение файла как текст
|
|
1541
|
-
* @private
|
|
2182
|
+
/**
|
|
2183
|
+
* Чтение файла как текст
|
|
2184
|
+
* @private
|
|
1542
2185
|
*/
|
|
1543
|
-
async function
|
|
2186
|
+
async function readFileAsArrayBuffer(file) {
|
|
1544
2187
|
return new Promise((resolve, reject) => {
|
|
1545
2188
|
const reader = new FileReader();
|
|
1546
2189
|
reader.onload = event => resolve(event.target.result);
|
|
1547
2190
|
reader.onerror = error => reject(error);
|
|
1548
|
-
reader.
|
|
2191
|
+
reader.readAsArrayBuffer(file);
|
|
1549
2192
|
});
|
|
1550
2193
|
}
|
|
1551
2194
|
|
|
@@ -1558,9 +2201,24 @@
|
|
|
1558
2201
|
};
|
|
1559
2202
|
}
|
|
1560
2203
|
|
|
2204
|
+
var workerPool = /*#__PURE__*/Object.freeze({
|
|
2205
|
+
__proto__: null,
|
|
2206
|
+
WorkerPool: WorkerPool,
|
|
2207
|
+
createWorkerPool: createWorkerPool,
|
|
2208
|
+
parseCSVWithWorker: parseCSVWithWorker
|
|
2209
|
+
});
|
|
2210
|
+
|
|
1561
2211
|
// Браузерный entry point для jtcsv
|
|
1562
2212
|
// Экспортирует все функции с поддержкой браузера
|
|
1563
2213
|
|
|
2214
|
+
async function createWorkerPoolLazy(options = {}) {
|
|
2215
|
+
const mod = await Promise.resolve().then(function () { return workerPool; });
|
|
2216
|
+
return mod.createWorkerPool(options);
|
|
2217
|
+
}
|
|
2218
|
+
async function parseCSVWithWorkerLazy(csvInput, options = {}, onProgress = null) {
|
|
2219
|
+
const mod = await Promise.resolve().then(function () { return workerPool; });
|
|
2220
|
+
return mod.parseCSVWithWorker(csvInput, options, onProgress);
|
|
2221
|
+
}
|
|
1564
2222
|
|
|
1565
2223
|
// Основной экспорт
|
|
1566
2224
|
const jtcsv = {
|
|
@@ -1571,11 +2229,18 @@
|
|
|
1571
2229
|
deepUnwrap,
|
|
1572
2230
|
// CSV to JSON функции
|
|
1573
2231
|
csvToJson,
|
|
2232
|
+
csvToJsonIterator,
|
|
1574
2233
|
parseCsvFile,
|
|
2234
|
+
parseCsvFileStream,
|
|
2235
|
+
jsonToCsvStream,
|
|
2236
|
+
jsonToNdjsonStream,
|
|
2237
|
+
csvToJsonStream,
|
|
1575
2238
|
autoDetectDelimiter,
|
|
1576
2239
|
// Web Workers функции
|
|
1577
2240
|
createWorkerPool,
|
|
1578
2241
|
parseCSVWithWorker,
|
|
2242
|
+
createWorkerPoolLazy,
|
|
2243
|
+
parseCSVWithWorkerLazy,
|
|
1579
2244
|
// Error classes
|
|
1580
2245
|
ValidationError,
|
|
1581
2246
|
SecurityError,
|
|
@@ -1583,6 +2248,7 @@
|
|
|
1583
2248
|
ParsingError,
|
|
1584
2249
|
LimitError,
|
|
1585
2250
|
ConfigurationError,
|
|
2251
|
+
ERROR_CODES,
|
|
1586
2252
|
// Удобные алиасы
|
|
1587
2253
|
parse: csvToJson,
|
|
1588
2254
|
unparse: jsonToCsv,
|
|
@@ -1603,6 +2269,7 @@
|
|
|
1603
2269
|
}
|
|
1604
2270
|
|
|
1605
2271
|
exports.ConfigurationError = ConfigurationError;
|
|
2272
|
+
exports.ERROR_CODES = ERROR_CODES;
|
|
1606
2273
|
exports.FileSystemError = FileSystemError;
|
|
1607
2274
|
exports.LimitError = LimitError;
|
|
1608
2275
|
exports.ParsingError = ParsingError;
|
|
@@ -1610,13 +2277,20 @@
|
|
|
1610
2277
|
exports.ValidationError = ValidationError;
|
|
1611
2278
|
exports.autoDetectDelimiter = autoDetectDelimiter;
|
|
1612
2279
|
exports.createWorkerPool = createWorkerPool;
|
|
2280
|
+
exports.createWorkerPoolLazy = createWorkerPoolLazy;
|
|
1613
2281
|
exports.csvToJson = csvToJson;
|
|
2282
|
+
exports.csvToJsonIterator = csvToJsonIterator;
|
|
2283
|
+
exports.csvToJsonStream = csvToJsonStream;
|
|
1614
2284
|
exports.deepUnwrap = deepUnwrap;
|
|
1615
2285
|
exports.default = jtcsv;
|
|
1616
2286
|
exports.downloadAsCsv = downloadAsCsv;
|
|
1617
2287
|
exports.jsonToCsv = jsonToCsv;
|
|
2288
|
+
exports.jsonToCsvStream = jsonToCsvStream;
|
|
2289
|
+
exports.jsonToNdjsonStream = jsonToNdjsonStream;
|
|
1618
2290
|
exports.parseCSVWithWorker = parseCSVWithWorker;
|
|
2291
|
+
exports.parseCSVWithWorkerLazy = parseCSVWithWorkerLazy;
|
|
1619
2292
|
exports.parseCsvFile = parseCsvFile;
|
|
2293
|
+
exports.parseCsvFileStream = parseCsvFileStream;
|
|
1620
2294
|
exports.preprocessData = preprocessData;
|
|
1621
2295
|
|
|
1622
2296
|
Object.defineProperty(exports, '__esModule', { value: true });
|