jtcsv 2.2.7 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -1
- package/bin/jtcsv.js +891 -821
- package/bin/jtcsv.ts +2534 -0
- package/csv-to-json.js +168 -145
- package/dist/jtcsv-core.cjs.js +1407 -0
- package/dist/jtcsv-core.cjs.js.map +1 -0
- package/dist/jtcsv-core.esm.js +1379 -0
- package/dist/jtcsv-core.esm.js.map +1 -0
- package/dist/jtcsv-core.umd.js +1413 -0
- package/dist/jtcsv-core.umd.js.map +1 -0
- package/dist/jtcsv-full.cjs.js +1912 -0
- package/dist/jtcsv-full.cjs.js.map +1 -0
- package/dist/jtcsv-full.esm.js +1880 -0
- package/dist/jtcsv-full.esm.js.map +1 -0
- package/dist/jtcsv-full.umd.js +1918 -0
- package/dist/jtcsv-full.umd.js.map +1 -0
- package/dist/jtcsv-workers.esm.js +759 -0
- package/dist/jtcsv-workers.esm.js.map +1 -0
- package/dist/jtcsv-workers.umd.js +773 -0
- package/dist/jtcsv-workers.umd.js.map +1 -0
- package/dist/jtcsv.cjs.js +61 -19
- package/dist/jtcsv.cjs.js.map +1 -1
- package/dist/jtcsv.esm.js +61 -19
- package/dist/jtcsv.esm.js.map +1 -1
- package/dist/jtcsv.umd.js +61 -19
- package/dist/jtcsv.umd.js.map +1 -1
- package/errors.js +188 -2
- package/examples/advanced/conditional-transformations.js +446 -0
- package/examples/advanced/conditional-transformations.ts +446 -0
- package/examples/advanced/csv-parser.worker.js +89 -0
- package/examples/advanced/csv-parser.worker.ts +89 -0
- package/examples/advanced/nested-objects-example.js +306 -0
- package/examples/advanced/nested-objects-example.ts +306 -0
- package/examples/advanced/performance-optimization.js +504 -0
- package/examples/advanced/performance-optimization.ts +504 -0
- package/examples/advanced/run-demo-server.js +116 -0
- package/examples/advanced/run-demo-server.ts +116 -0
- package/examples/advanced/web-worker-usage.html +874 -0
- package/examples/async-multithreaded-example.ts +335 -0
- package/examples/cli-advanced-usage.md +288 -0
- package/examples/cli-batch-processing.ts +38 -0
- package/examples/cli-tool.js +0 -3
- package/examples/cli-tool.ts +183 -0
- package/examples/error-handling.js +21 -7
- package/examples/error-handling.ts +356 -0
- package/examples/express-api.js +0 -3
- package/examples/express-api.ts +164 -0
- package/examples/large-dataset-example.js +0 -3
- package/examples/large-dataset-example.ts +204 -0
- package/examples/ndjson-processing.js +1 -1
- package/examples/ndjson-processing.ts +456 -0
- package/examples/plugin-excel-exporter.js +3 -4
- package/examples/plugin-excel-exporter.ts +406 -0
- package/examples/react-integration.tsx +637 -0
- package/examples/schema-validation.ts +640 -0
- package/examples/simple-usage.js +254 -254
- package/examples/simple-usage.ts +194 -0
- package/examples/streaming-example.js +4 -5
- package/examples/streaming-example.ts +419 -0
- package/examples/web-workers-advanced.ts +28 -0
- package/index.d.ts +1 -3
- package/index.js +15 -1
- package/json-save.js +9 -3
- package/json-to-csv.js +168 -21
- package/package.json +69 -10
- package/plugins/express-middleware/README.md +21 -2
- package/plugins/express-middleware/example.js +3 -4
- package/plugins/express-middleware/example.ts +135 -0
- package/plugins/express-middleware/index.d.ts +1 -1
- package/plugins/express-middleware/index.js +270 -118
- package/plugins/express-middleware/index.ts +557 -0
- package/plugins/fastify-plugin/index.js +2 -4
- package/plugins/fastify-plugin/index.ts +443 -0
- package/plugins/hono/index.ts +226 -0
- package/plugins/nestjs/index.ts +201 -0
- package/plugins/nextjs-api/examples/ConverterComponent.tsx +386 -0
- package/plugins/nextjs-api/examples/api-convert.js +0 -2
- package/plugins/nextjs-api/examples/api-convert.ts +67 -0
- package/plugins/nextjs-api/index.tsx +339 -0
- package/plugins/nextjs-api/route.js +2 -3
- package/plugins/nextjs-api/route.ts +370 -0
- package/plugins/nuxt/index.ts +94 -0
- package/plugins/nuxt/runtime/composables/useJtcsv.ts +100 -0
- package/plugins/nuxt/runtime/plugin.ts +71 -0
- package/plugins/remix/index.js +1 -1
- package/plugins/remix/index.ts +260 -0
- package/plugins/sveltekit/index.js +1 -1
- package/plugins/sveltekit/index.ts +301 -0
- package/plugins/trpc/index.ts +267 -0
- package/src/browser/browser-functions.ts +402 -0
- package/src/browser/core.js +92 -0
- package/src/browser/core.ts +152 -0
- package/src/browser/csv-to-json-browser.d.ts +3 -0
- package/src/browser/csv-to-json-browser.js +36 -14
- package/src/browser/csv-to-json-browser.ts +264 -0
- package/src/browser/errors-browser.ts +303 -0
- package/src/browser/extensions/plugins.js +92 -0
- package/src/browser/extensions/plugins.ts +93 -0
- package/src/browser/extensions/workers.js +39 -0
- package/src/browser/extensions/workers.ts +39 -0
- package/src/browser/globals.d.ts +5 -0
- package/src/browser/index.ts +192 -0
- package/src/browser/json-to-csv-browser.d.ts +3 -0
- package/src/browser/json-to-csv-browser.js +13 -3
- package/src/browser/json-to-csv-browser.ts +262 -0
- package/src/browser/streams.js +12 -2
- package/src/browser/streams.ts +336 -0
- package/src/browser/workers/csv-parser.worker.ts +377 -0
- package/src/browser/workers/worker-pool.ts +548 -0
- package/src/core/delimiter-cache.js +22 -8
- package/src/core/delimiter-cache.ts +310 -0
- package/src/core/node-optimizations.ts +449 -0
- package/src/core/plugin-system.js +29 -11
- package/src/core/plugin-system.ts +400 -0
- package/src/core/transform-hooks.ts +558 -0
- package/src/engines/fast-path-engine-new.ts +347 -0
- package/src/engines/fast-path-engine.ts +854 -0
- package/src/errors.ts +72 -0
- package/src/formats/ndjson-parser.ts +469 -0
- package/src/formats/tsv-parser.ts +334 -0
- package/src/index-with-plugins.js +16 -9
- package/src/index-with-plugins.ts +395 -0
- package/src/types/index.ts +255 -0
- package/src/utils/bom-utils.js +259 -0
- package/src/utils/bom-utils.ts +373 -0
- package/src/utils/encoding-support.js +124 -0
- package/src/utils/encoding-support.ts +155 -0
- package/src/utils/schema-validator.js +19 -19
- package/src/utils/schema-validator.ts +819 -0
- package/src/utils/transform-loader.js +1 -1
- package/src/utils/transform-loader.ts +389 -0
- package/src/utils/zod-adapter.js +170 -0
- package/src/utils/zod-adapter.ts +280 -0
- package/src/web-server/index.js +10 -10
- package/src/web-server/index.ts +683 -0
- package/src/workers/csv-multithreaded.ts +310 -0
- package/src/workers/csv-parser.worker.ts +227 -0
- package/src/workers/worker-pool.ts +409 -0
- package/stream-csv-to-json.js +26 -8
- package/stream-json-to-csv.js +1 -0
|
@@ -0,0 +1,759 @@
|
|
|
1
|
+
// Система ошибок для браузерной версии jtcsv
|
|
2
|
+
// Адаптирована для работы без Node.js специфичных API
|
|
3
|
+
/**
|
|
4
|
+
* Базовый класс ошибки jtcsv
|
|
5
|
+
*/
|
|
6
|
+
class JTCSVError extends Error {
|
|
7
|
+
constructor(message, code = 'JTCSV_ERROR', details = {}) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.name = 'JTCSVError';
|
|
10
|
+
this.code = code;
|
|
11
|
+
this.details = details;
|
|
12
|
+
// Сохранение stack trace
|
|
13
|
+
if (Error.captureStackTrace) {
|
|
14
|
+
Error.captureStackTrace(this, JTCSVError);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Ошибка валидации
|
|
20
|
+
*/
|
|
21
|
+
class ValidationError extends JTCSVError {
|
|
22
|
+
constructor(message, details = {}) {
|
|
23
|
+
super(message, 'VALIDATION_ERROR', details);
|
|
24
|
+
this.name = 'ValidationError';
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Ошибка безопасности
|
|
29
|
+
*/
|
|
30
|
+
class SecurityError extends JTCSVError {
|
|
31
|
+
constructor(message, details = {}) {
|
|
32
|
+
super(message, 'SECURITY_ERROR', details);
|
|
33
|
+
this.name = 'SecurityError';
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Ошибка файловой системы (адаптирована для браузера)
|
|
38
|
+
*/
|
|
39
|
+
class FileSystemError extends JTCSVError {
|
|
40
|
+
constructor(message, originalError, details = {}) {
|
|
41
|
+
super(message, 'FILE_SYSTEM_ERROR', { ...details, originalError });
|
|
42
|
+
this.name = 'FileSystemError';
|
|
43
|
+
if (originalError && originalError.code) {
|
|
44
|
+
this.code = originalError.code;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Ошибка парсинга
|
|
50
|
+
*/
|
|
51
|
+
class ParsingError extends JTCSVError {
|
|
52
|
+
constructor(message, lineNumber, details = {}) {
|
|
53
|
+
super(message, 'PARSING_ERROR', { ...details, lineNumber });
|
|
54
|
+
this.name = 'ParsingError';
|
|
55
|
+
this.lineNumber = lineNumber;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Ошибка превышения лимита
|
|
60
|
+
*/
|
|
61
|
+
class LimitError extends JTCSVError {
|
|
62
|
+
constructor(message, limit, actual, details = {}) {
|
|
63
|
+
super(message, 'LIMIT_ERROR', { ...details, limit, actual });
|
|
64
|
+
this.name = 'LimitError';
|
|
65
|
+
this.limit = limit;
|
|
66
|
+
this.actual = actual;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Ошибка конфигурации
|
|
71
|
+
*/
|
|
72
|
+
class ConfigurationError extends JTCSVError {
|
|
73
|
+
constructor(message, details = {}) {
|
|
74
|
+
super(message, 'CONFIGURATION_ERROR', details);
|
|
75
|
+
this.name = 'ConfigurationError';
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Коды ошибок
|
|
80
|
+
*/
|
|
81
|
+
const ERROR_CODES = {
|
|
82
|
+
JTCSV_ERROR: 'JTCSV_ERROR',
|
|
83
|
+
VALIDATION_ERROR: 'VALIDATION_ERROR',
|
|
84
|
+
SECURITY_ERROR: 'SECURITY_ERROR',
|
|
85
|
+
FILE_SYSTEM_ERROR: 'FILE_SYSTEM_ERROR',
|
|
86
|
+
PARSING_ERROR: 'PARSING_ERROR',
|
|
87
|
+
LIMIT_ERROR: 'LIMIT_ERROR',
|
|
88
|
+
CONFIGURATION_ERROR: 'CONFIGURATION_ERROR',
|
|
89
|
+
INVALID_INPUT: 'INVALID_INPUT',
|
|
90
|
+
SECURITY_VIOLATION: 'SECURITY_VIOLATION',
|
|
91
|
+
FILE_NOT_FOUND: 'FILE_NOT_FOUND',
|
|
92
|
+
PARSE_FAILED: 'PARSE_FAILED',
|
|
93
|
+
SIZE_LIMIT: 'SIZE_LIMIT',
|
|
94
|
+
INVALID_CONFIG: 'INVALID_CONFIG',
|
|
95
|
+
UNKNOWN_ERROR: 'UNKNOWN_ERROR'
|
|
96
|
+
};
|
|
97
|
+
/**
|
|
98
|
+
* Безопасное выполнение функции с обработкой ошибок
|
|
99
|
+
*
|
|
100
|
+
* @param fn - Функция для выполнения
|
|
101
|
+
* @param errorCode - Код ошибки по умолчанию
|
|
102
|
+
* @param errorDetails - Детали ошибки
|
|
103
|
+
* @returns Результат выполнения функции
|
|
104
|
+
*/
|
|
105
|
+
function safeExecute(fn, errorCode = 'UNKNOWN_ERROR', errorDetails = {}) {
|
|
106
|
+
try {
|
|
107
|
+
if (typeof fn === 'function') {
|
|
108
|
+
return fn();
|
|
109
|
+
}
|
|
110
|
+
throw new ValidationError('Function expected');
|
|
111
|
+
}
|
|
112
|
+
catch (error) {
|
|
113
|
+
// Если ошибка уже является JTCSVError, перебросить её
|
|
114
|
+
if (error instanceof JTCSVError) {
|
|
115
|
+
throw error;
|
|
116
|
+
}
|
|
117
|
+
// Определить тип ошибки на основе сообщения или кода
|
|
118
|
+
let enhancedError;
|
|
119
|
+
const errorMessage = error.message || String(error);
|
|
120
|
+
if (errorMessage.includes('validation') || errorMessage.includes('Validation')) {
|
|
121
|
+
enhancedError = new ValidationError(errorMessage, { ...errorDetails, originalError: error });
|
|
122
|
+
}
|
|
123
|
+
else if (errorMessage.includes('security') || errorMessage.includes('Security')) {
|
|
124
|
+
enhancedError = new SecurityError(errorMessage, { ...errorDetails, originalError: error });
|
|
125
|
+
}
|
|
126
|
+
else if (errorMessage.includes('parsing') || errorMessage.includes('Parsing')) {
|
|
127
|
+
enhancedError = new ParsingError(errorMessage, undefined, { ...errorDetails, originalError: error });
|
|
128
|
+
}
|
|
129
|
+
else if (errorMessage.includes('limit') || errorMessage.includes('Limit')) {
|
|
130
|
+
enhancedError = new LimitError(errorMessage, null, null, { ...errorDetails, originalError: error });
|
|
131
|
+
}
|
|
132
|
+
else if (errorMessage.includes('configuration') || errorMessage.includes('Configuration')) {
|
|
133
|
+
enhancedError = new ConfigurationError(errorMessage, { ...errorDetails, originalError: error });
|
|
134
|
+
}
|
|
135
|
+
else if (errorMessage.includes('file') || errorMessage.includes('File')) {
|
|
136
|
+
enhancedError = new FileSystemError(errorMessage, error, errorDetails);
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
// Общая ошибка
|
|
140
|
+
enhancedError = new JTCSVError(errorMessage, errorCode, { ...errorDetails, originalError: error });
|
|
141
|
+
}
|
|
142
|
+
// Сохранить оригинальный stack trace если возможно
|
|
143
|
+
if (error.stack) {
|
|
144
|
+
enhancedError.stack = error.stack;
|
|
145
|
+
}
|
|
146
|
+
throw enhancedError;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Асинхронная версия safeExecute
|
|
151
|
+
*/
|
|
152
|
+
async function safeExecuteAsync(fn, errorCode = 'UNKNOWN_ERROR', errorDetails = {}) {
|
|
153
|
+
try {
|
|
154
|
+
if (typeof fn === 'function') {
|
|
155
|
+
return await fn();
|
|
156
|
+
}
|
|
157
|
+
throw new ValidationError('Function expected');
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
// Если ошибка уже является JTCSVError, перебросить её
|
|
161
|
+
if (error instanceof JTCSVError) {
|
|
162
|
+
throw error;
|
|
163
|
+
}
|
|
164
|
+
// Определить тип ошибки
|
|
165
|
+
let enhancedError;
|
|
166
|
+
const errorMessage = error.message || String(error);
|
|
167
|
+
if (errorMessage.includes('validation') || errorMessage.includes('Validation')) {
|
|
168
|
+
enhancedError = new ValidationError(errorMessage, { ...errorDetails, originalError: error });
|
|
169
|
+
}
|
|
170
|
+
else if (errorMessage.includes('security') || errorMessage.includes('Security')) {
|
|
171
|
+
enhancedError = new SecurityError(errorMessage, { ...errorDetails, originalError: error });
|
|
172
|
+
}
|
|
173
|
+
else if (errorMessage.includes('parsing') || errorMessage.includes('Parsing')) {
|
|
174
|
+
enhancedError = new ParsingError(errorMessage, undefined, { ...errorDetails, originalError: error });
|
|
175
|
+
}
|
|
176
|
+
else if (errorMessage.includes('limit') || errorMessage.includes('Limit')) {
|
|
177
|
+
enhancedError = new LimitError(errorMessage, null, null, { ...errorDetails, originalError: error });
|
|
178
|
+
}
|
|
179
|
+
else if (errorMessage.includes('configuration') || errorMessage.includes('Configuration')) {
|
|
180
|
+
enhancedError = new ConfigurationError(errorMessage, { ...errorDetails, originalError: error });
|
|
181
|
+
}
|
|
182
|
+
else if (errorMessage.includes('file') || errorMessage.includes('File')) {
|
|
183
|
+
enhancedError = new FileSystemError(errorMessage, error, errorDetails);
|
|
184
|
+
}
|
|
185
|
+
else {
|
|
186
|
+
enhancedError = new JTCSVError(errorMessage, errorCode, { ...errorDetails, originalError: error });
|
|
187
|
+
}
|
|
188
|
+
if (error.stack) {
|
|
189
|
+
enhancedError.stack = error.stack;
|
|
190
|
+
}
|
|
191
|
+
throw enhancedError;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Создать сообщение об ошибке
|
|
196
|
+
*/
|
|
197
|
+
function createErrorMessage(error, includeStack = false) {
|
|
198
|
+
let message = error.message || 'Unknown error';
|
|
199
|
+
if (error instanceof JTCSVError) {
|
|
200
|
+
message = `[${error.code}] ${message}`;
|
|
201
|
+
if (error instanceof ParsingError && error.lineNumber) {
|
|
202
|
+
message += ` (line ${error.lineNumber})`;
|
|
203
|
+
}
|
|
204
|
+
if (error instanceof LimitError && error.limit && error.actual) {
|
|
205
|
+
message += ` (limit: ${error.limit}, actual: ${error.actual})`;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
if (includeStack && error.stack) {
|
|
209
|
+
message += `\n${error.stack}`;
|
|
210
|
+
}
|
|
211
|
+
return message;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Обработка ошибки
|
|
215
|
+
*/
|
|
216
|
+
function handleError(error, options = {}) {
|
|
217
|
+
const { log = true, throw: shouldThrow = false, format = true } = options;
|
|
218
|
+
const message = format ? createErrorMessage(error) : error.message;
|
|
219
|
+
if (log) {
|
|
220
|
+
console.error(`[jtcsv] ${message}`);
|
|
221
|
+
if (error instanceof JTCSVError && error.details) {
|
|
222
|
+
console.error('Error details:', error.details);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (shouldThrow) {
|
|
226
|
+
throw error;
|
|
227
|
+
}
|
|
228
|
+
return message;
|
|
229
|
+
}
|
|
230
|
+
// Экспорт для Node.js совместимости
|
|
231
|
+
if (typeof module !== 'undefined' && module.exports) {
|
|
232
|
+
module.exports = {
|
|
233
|
+
JTCSVError,
|
|
234
|
+
ValidationError,
|
|
235
|
+
SecurityError,
|
|
236
|
+
FileSystemError,
|
|
237
|
+
ParsingError,
|
|
238
|
+
LimitError,
|
|
239
|
+
ConfigurationError,
|
|
240
|
+
ERROR_CODES,
|
|
241
|
+
safeExecute,
|
|
242
|
+
safeExecuteAsync,
|
|
243
|
+
createErrorMessage,
|
|
244
|
+
handleError
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Worker Pool для параллельной обработки CSV
|
|
249
|
+
// Использует Comlink для простой коммуникации с Web Workers
|
|
250
|
+
// Проверка поддержки Web Workers
|
|
251
|
+
const WORKERS_SUPPORTED = typeof Worker !== 'undefined';
|
|
252
|
+
function isTransferableBuffer(value) {
|
|
253
|
+
if (!(value instanceof ArrayBuffer)) {
|
|
254
|
+
return false;
|
|
255
|
+
}
|
|
256
|
+
if (typeof SharedArrayBuffer !== 'undefined' && value instanceof SharedArrayBuffer) {
|
|
257
|
+
return false;
|
|
258
|
+
}
|
|
259
|
+
return true;
|
|
260
|
+
}
|
|
261
|
+
function collectTransferables(args) {
|
|
262
|
+
const transferables = [];
|
|
263
|
+
const collectFromValue = (value) => {
|
|
264
|
+
if (!value) {
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
if (isTransferableBuffer(value)) {
|
|
268
|
+
transferables.push(value);
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
if (ArrayBuffer.isView(value) && isTransferableBuffer(value.buffer)) {
|
|
272
|
+
transferables.push(value.buffer);
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
if (Array.isArray(value)) {
|
|
276
|
+
value.forEach(collectFromValue);
|
|
277
|
+
}
|
|
278
|
+
};
|
|
279
|
+
args.forEach(collectFromValue);
|
|
280
|
+
return transferables.length ? transferables : null;
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Опции для Worker Pool
|
|
284
|
+
* @typedef {Object} WorkerPoolOptions
|
|
285
|
+
* @property {number} [workerCount=4] - Количество workers в pool
|
|
286
|
+
* @property {number} [maxQueueSize=100] - Максимальный размер очереди задач
|
|
287
|
+
* @property {boolean} [autoScale=true] - Автоматическое масштабирование pool
|
|
288
|
+
* @property {number} [idleTimeout=60000] - Таймаут простоя worker (мс)
|
|
289
|
+
*/
|
|
290
|
+
/**
|
|
291
|
+
* Статистика Worker Pool
|
|
292
|
+
* @typedef {Object} WorkerPoolStats
|
|
293
|
+
* @property {number} totalWorkers - Всего workers
|
|
294
|
+
* @property {number} activeWorkers - Активные workers
|
|
295
|
+
* @property {number} idleWorkers - Простаивающие workers
|
|
296
|
+
* @property {number} queueSize - Размер очереди
|
|
297
|
+
* @property {number} tasksCompleted - Завершенные задачи
|
|
298
|
+
* @property {number} tasksFailed - Неудачные задачи
|
|
299
|
+
*/
|
|
300
|
+
/**
|
|
301
|
+
* Прогресс обработки задачи
|
|
302
|
+
* @typedef {Object} TaskProgress
|
|
303
|
+
* @property {number} processed - Обработано элементов
|
|
304
|
+
* @property {number} total - Всего элементов
|
|
305
|
+
* @property {number} percentage - Процент выполнения
|
|
306
|
+
* @property {number} speed - Скорость обработки (элементов/сек)
|
|
307
|
+
*/
|
|
308
|
+
/**
|
|
309
|
+
* Worker Pool для параллельной обработки CSV
|
|
310
|
+
*/
|
|
311
|
+
class WorkerPool {
|
|
312
|
+
/**
|
|
313
|
+
* Создает новый Worker Pool
|
|
314
|
+
* @param {string} workerScript - URL скрипта worker
|
|
315
|
+
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
316
|
+
*/
|
|
317
|
+
constructor(workerScript, options = {}) {
|
|
318
|
+
if (!WORKERS_SUPPORTED) {
|
|
319
|
+
throw new ValidationError('Web Workers не поддерживаются в этом браузере');
|
|
320
|
+
}
|
|
321
|
+
this.workerScript = workerScript;
|
|
322
|
+
this.options = {
|
|
323
|
+
workerCount: 4,
|
|
324
|
+
maxQueueSize: 100,
|
|
325
|
+
autoScale: true,
|
|
326
|
+
idleTimeout: 60000,
|
|
327
|
+
...options
|
|
328
|
+
};
|
|
329
|
+
this.workers = [];
|
|
330
|
+
this.taskQueue = [];
|
|
331
|
+
this.activeTasks = new Map();
|
|
332
|
+
this.stats = {
|
|
333
|
+
totalWorkers: 0,
|
|
334
|
+
activeWorkers: 0,
|
|
335
|
+
idleWorkers: 0,
|
|
336
|
+
queueSize: 0,
|
|
337
|
+
tasksCompleted: 0,
|
|
338
|
+
tasksFailed: 0
|
|
339
|
+
};
|
|
340
|
+
this.initializeWorkers();
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Инициализация workers
|
|
344
|
+
* @private
|
|
345
|
+
*/
|
|
346
|
+
initializeWorkers() {
|
|
347
|
+
const { workerCount } = this.options;
|
|
348
|
+
for (let i = 0; i < workerCount; i++) {
|
|
349
|
+
this.createWorker();
|
|
350
|
+
}
|
|
351
|
+
this.updateStats();
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Создает нового worker
|
|
355
|
+
* @private
|
|
356
|
+
*/
|
|
357
|
+
createWorker() {
|
|
358
|
+
try {
|
|
359
|
+
const worker = new Worker(this.workerScript, { type: 'module' });
|
|
360
|
+
worker.id = `worker-${this.workers.length}`;
|
|
361
|
+
worker.status = 'idle';
|
|
362
|
+
worker.lastUsed = Date.now();
|
|
363
|
+
worker.taskId = null;
|
|
364
|
+
// Обработчики событий
|
|
365
|
+
worker.onmessage = (event) => this.handleWorkerMessage(worker, event);
|
|
366
|
+
worker.onerror = (error) => this.handleWorkerError(worker, error);
|
|
367
|
+
worker.onmessageerror = (error) => this.handleWorkerMessageError(worker, error);
|
|
368
|
+
this.workers.push(worker);
|
|
369
|
+
this.stats.totalWorkers++;
|
|
370
|
+
this.stats.idleWorkers++;
|
|
371
|
+
return worker;
|
|
372
|
+
}
|
|
373
|
+
catch (error) {
|
|
374
|
+
throw new ConfigurationError(`Не удалось создать worker: ${error.message}`);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Обработка сообщений от worker
|
|
379
|
+
* @private
|
|
380
|
+
*/
|
|
381
|
+
handleWorkerMessage(worker, event) {
|
|
382
|
+
const { data } = event;
|
|
383
|
+
if (data.type === 'PROGRESS') {
|
|
384
|
+
this.handleProgress(worker, data);
|
|
385
|
+
}
|
|
386
|
+
else if (data.type === 'RESULT') {
|
|
387
|
+
this.handleResult(worker, data);
|
|
388
|
+
}
|
|
389
|
+
else if (data.type === 'ERROR') {
|
|
390
|
+
this.handleWorkerTaskError(worker, data);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Обработка прогресса задачи
|
|
395
|
+
* @private
|
|
396
|
+
*/
|
|
397
|
+
handleProgress(worker, progressData) {
|
|
398
|
+
const taskId = worker.taskId;
|
|
399
|
+
if (taskId && this.activeTasks.has(taskId)) {
|
|
400
|
+
const task = this.activeTasks.get(taskId);
|
|
401
|
+
if (task.onProgress) {
|
|
402
|
+
task.onProgress({
|
|
403
|
+
processed: progressData.processed,
|
|
404
|
+
total: progressData.total,
|
|
405
|
+
percentage: (progressData.processed / progressData.total) * 100,
|
|
406
|
+
speed: progressData.speed || 0
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* Обработка результата задачи
|
|
413
|
+
* @private
|
|
414
|
+
*/
|
|
415
|
+
handleResult(worker, resultData) {
|
|
416
|
+
const taskId = worker.taskId;
|
|
417
|
+
if (taskId && this.activeTasks.has(taskId)) {
|
|
418
|
+
const task = this.activeTasks.get(taskId);
|
|
419
|
+
// Освобождение worker
|
|
420
|
+
worker.status = 'idle';
|
|
421
|
+
worker.lastUsed = Date.now();
|
|
422
|
+
worker.taskId = null;
|
|
423
|
+
this.stats.activeWorkers--;
|
|
424
|
+
this.stats.idleWorkers++;
|
|
425
|
+
// Завершение задачи
|
|
426
|
+
task.resolve(resultData.data);
|
|
427
|
+
this.activeTasks.delete(taskId);
|
|
428
|
+
this.stats.tasksCompleted++;
|
|
429
|
+
// Обработка следующей задачи в очереди
|
|
430
|
+
this.processQueue();
|
|
431
|
+
this.updateStats();
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Обработка ошибки задачи
|
|
436
|
+
* @private
|
|
437
|
+
*/
|
|
438
|
+
handleWorkerTaskError(worker, errorData) {
|
|
439
|
+
const taskId = worker.taskId;
|
|
440
|
+
if (taskId && this.activeTasks.has(taskId)) {
|
|
441
|
+
const task = this.activeTasks.get(taskId);
|
|
442
|
+
// Освобождение worker
|
|
443
|
+
worker.status = 'idle';
|
|
444
|
+
worker.lastUsed = Date.now();
|
|
445
|
+
worker.taskId = null;
|
|
446
|
+
this.stats.activeWorkers--;
|
|
447
|
+
this.stats.idleWorkers++;
|
|
448
|
+
// Завершение с ошибкой
|
|
449
|
+
const workerError = new Error(errorData.message || 'Ошибка в worker');
|
|
450
|
+
if (errorData.code) {
|
|
451
|
+
workerError.code = errorData.code;
|
|
452
|
+
}
|
|
453
|
+
if (errorData.details) {
|
|
454
|
+
workerError.details = errorData.details;
|
|
455
|
+
}
|
|
456
|
+
task.reject(workerError);
|
|
457
|
+
this.activeTasks.delete(taskId);
|
|
458
|
+
this.stats.tasksFailed++;
|
|
459
|
+
// Обработка следующей задачи
|
|
460
|
+
this.processQueue();
|
|
461
|
+
this.updateStats();
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
/**
|
|
465
|
+
* Обработка ошибок worker
|
|
466
|
+
* @private
|
|
467
|
+
*/
|
|
468
|
+
handleWorkerError(worker, error) {
|
|
469
|
+
console.error(`Worker ${worker.id} error:`, error);
|
|
470
|
+
// Перезапуск worker
|
|
471
|
+
this.restartWorker(worker);
|
|
472
|
+
}
|
|
473
|
+
/**
|
|
474
|
+
* Обработка ошибок сообщений
|
|
475
|
+
* @private
|
|
476
|
+
*/
|
|
477
|
+
handleWorkerMessageError(worker, error) {
|
|
478
|
+
console.error(`Worker ${worker.id} message error:`, error);
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Перезапуск worker
|
|
482
|
+
* @private
|
|
483
|
+
*/
|
|
484
|
+
restartWorker(worker) {
|
|
485
|
+
const index = this.workers.indexOf(worker);
|
|
486
|
+
if (index !== -1) {
|
|
487
|
+
// Завершение старого worker
|
|
488
|
+
worker.terminate();
|
|
489
|
+
// Удаление из статистики
|
|
490
|
+
if (worker.status === 'active') {
|
|
491
|
+
this.stats.activeWorkers--;
|
|
492
|
+
}
|
|
493
|
+
else {
|
|
494
|
+
this.stats.idleWorkers--;
|
|
495
|
+
}
|
|
496
|
+
this.stats.totalWorkers--;
|
|
497
|
+
// Создание нового worker
|
|
498
|
+
const newWorker = this.createWorker();
|
|
499
|
+
this.workers[index] = newWorker;
|
|
500
|
+
// Перезапуск задачи если была активна
|
|
501
|
+
if (worker.taskId && this.activeTasks.has(worker.taskId)) {
|
|
502
|
+
const task = this.activeTasks.get(worker.taskId);
|
|
503
|
+
this.executeTask(newWorker, task);
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
/**
|
|
508
|
+
* Выполнение задачи на worker
|
|
509
|
+
* @private
|
|
510
|
+
*/
|
|
511
|
+
executeTask(worker, task) {
|
|
512
|
+
worker.status = 'active';
|
|
513
|
+
worker.lastUsed = Date.now();
|
|
514
|
+
worker.taskId = task.id;
|
|
515
|
+
this.stats.idleWorkers--;
|
|
516
|
+
this.stats.activeWorkers++;
|
|
517
|
+
// Отправка задачи в worker
|
|
518
|
+
const payload = {
|
|
519
|
+
type: 'EXECUTE',
|
|
520
|
+
taskId: task.id,
|
|
521
|
+
method: task.method,
|
|
522
|
+
args: task.args,
|
|
523
|
+
options: task.options
|
|
524
|
+
};
|
|
525
|
+
if (task.transferList && task.transferList.length) {
|
|
526
|
+
worker.postMessage(payload, task.transferList);
|
|
527
|
+
}
|
|
528
|
+
else {
|
|
529
|
+
worker.postMessage(payload);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* Обработка очереди задач
|
|
534
|
+
* @private
|
|
535
|
+
*/
|
|
536
|
+
processQueue() {
|
|
537
|
+
if (this.taskQueue.length === 0) {
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
while (this.taskQueue.length > 0) {
|
|
541
|
+
const idleWorker = this.workers.find(w => w.status === 'idle');
|
|
542
|
+
if (!idleWorker) {
|
|
543
|
+
if (this.options.autoScale && this.workers.length < this.options.maxQueueSize) {
|
|
544
|
+
this.createWorker();
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
break;
|
|
548
|
+
}
|
|
549
|
+
const task = this.taskQueue.shift();
|
|
550
|
+
this.stats.queueSize--;
|
|
551
|
+
this.executeTask(idleWorker, task);
|
|
552
|
+
}
|
|
553
|
+
this.updateStats();
|
|
554
|
+
}
|
|
555
|
+
/**
|
|
556
|
+
* Обновление статистики
|
|
557
|
+
* @private
|
|
558
|
+
*/
|
|
559
|
+
updateStats() {
|
|
560
|
+
this.stats.queueSize = this.taskQueue.length;
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* Выполнение задачи через pool
|
|
564
|
+
* @param {string} method - Метод для вызова в worker
|
|
565
|
+
* @param {Array} args - Аргументы метода
|
|
566
|
+
* @param {Object} [options] - Опции задачи
|
|
567
|
+
* @param {Function} [onProgress] - Callback прогресса
|
|
568
|
+
* @returns {Promise<unknown>} Результат выполнения
|
|
569
|
+
*/
|
|
570
|
+
async exec(method, args = [], options = {}, onProgress = null) {
|
|
571
|
+
return new Promise((resolve, reject) => {
|
|
572
|
+
// Проверка размера очереди
|
|
573
|
+
if (this.taskQueue.length >= this.options.maxQueueSize) {
|
|
574
|
+
reject(new Error('Очередь задач переполнена'));
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
// Создание задачи
|
|
578
|
+
const taskId = `task-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
579
|
+
const { transfer, ...taskOptions } = options || {};
|
|
580
|
+
const transferList = transfer || collectTransferables(args);
|
|
581
|
+
const task = {
|
|
582
|
+
id: taskId,
|
|
583
|
+
method,
|
|
584
|
+
args,
|
|
585
|
+
options: taskOptions,
|
|
586
|
+
transferList,
|
|
587
|
+
onProgress,
|
|
588
|
+
resolve,
|
|
589
|
+
reject,
|
|
590
|
+
createdAt: Date.now()
|
|
591
|
+
};
|
|
592
|
+
// Добавление в очередь
|
|
593
|
+
this.taskQueue.push(task);
|
|
594
|
+
this.stats.queueSize++;
|
|
595
|
+
// Запуск обработки очереди
|
|
596
|
+
this.processQueue();
|
|
597
|
+
this.updateStats();
|
|
598
|
+
});
|
|
599
|
+
}
|
|
600
|
+
/**
|
|
601
|
+
* Получение статистики pool
|
|
602
|
+
* @returns {WorkerPoolStats} Статистика
|
|
603
|
+
*/
|
|
604
|
+
getStats() {
|
|
605
|
+
return { ...this.stats };
|
|
606
|
+
}
|
|
607
|
+
/**
|
|
608
|
+
* Очистка простаивающих workers
|
|
609
|
+
*/
|
|
610
|
+
cleanupIdleWorkers() {
|
|
611
|
+
const now = Date.now();
|
|
612
|
+
const { idleTimeout } = this.options;
|
|
613
|
+
for (let i = this.workers.length - 1; i >= 0; i--) {
|
|
614
|
+
const worker = this.workers[i];
|
|
615
|
+
if (worker.status === 'idle' && (now - worker.lastUsed) > idleTimeout) {
|
|
616
|
+
// Сохранение минимального количества workers
|
|
617
|
+
if (this.workers.length > 1) {
|
|
618
|
+
worker.terminate();
|
|
619
|
+
this.workers.splice(i, 1);
|
|
620
|
+
this.stats.totalWorkers--;
|
|
621
|
+
this.stats.idleWorkers--;
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
/**
|
|
627
|
+
* Завершение всех workers
|
|
628
|
+
*/
|
|
629
|
+
terminate() {
|
|
630
|
+
this.workers.forEach(worker => {
|
|
631
|
+
worker.terminate();
|
|
632
|
+
});
|
|
633
|
+
this.workers = [];
|
|
634
|
+
this.taskQueue = [];
|
|
635
|
+
this.activeTasks.clear();
|
|
636
|
+
// Сброс статистики
|
|
637
|
+
this.stats = {
|
|
638
|
+
totalWorkers: 0,
|
|
639
|
+
activeWorkers: 0,
|
|
640
|
+
idleWorkers: 0,
|
|
641
|
+
queueSize: 0,
|
|
642
|
+
tasksCompleted: 0,
|
|
643
|
+
tasksFailed: 0
|
|
644
|
+
};
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
/**
|
|
648
|
+
* Создает Worker Pool для обработки CSV
|
|
649
|
+
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
650
|
+
* @returns {WorkerPool} Worker Pool
|
|
651
|
+
*/
|
|
652
|
+
function createWorkerPool(options = {}) {
|
|
653
|
+
// Используем встроенный worker скрипт
|
|
654
|
+
const baseUrl = typeof document !== 'undefined'
|
|
655
|
+
? document.baseURI
|
|
656
|
+
: (typeof self !== 'undefined' && self.location
|
|
657
|
+
? self.location.href
|
|
658
|
+
: '');
|
|
659
|
+
const workerScript = new URL('./csv-parser.worker.js', baseUrl).href;
|
|
660
|
+
return new WorkerPool(workerScript, options);
|
|
661
|
+
}
|
|
662
|
+
/**
|
|
663
|
+
* Парсит CSV с использованием Web Workers
|
|
664
|
+
* @param {string|File} csvInput - CSV строка или File объект
|
|
665
|
+
* @param {Object} [options] - Опции парсинга
|
|
666
|
+
* @param {Function} [onProgress] - Callback прогресса
|
|
667
|
+
* @returns {Promise<Array<Object>>} JSON данные
|
|
668
|
+
*/
|
|
669
|
+
async function parseCSVWithWorker(csvInput, options = {}, onProgress = null) {
|
|
670
|
+
// Создание pool если нужно
|
|
671
|
+
const poolHolder = parseCSVWithWorker;
|
|
672
|
+
if (!poolHolder.pool) {
|
|
673
|
+
poolHolder.pool = createWorkerPool();
|
|
674
|
+
}
|
|
675
|
+
const pool = poolHolder.pool;
|
|
676
|
+
// Подготовка CSV строки
|
|
677
|
+
// ?????????? CSV ??????
|
|
678
|
+
let csvPayload = csvInput;
|
|
679
|
+
let transfer = null;
|
|
680
|
+
if (csvInput instanceof File) {
|
|
681
|
+
const buffer = await readFileAsArrayBuffer(csvInput);
|
|
682
|
+
csvPayload = new Uint8Array(buffer);
|
|
683
|
+
transfer = [buffer];
|
|
684
|
+
}
|
|
685
|
+
else if (csvInput instanceof ArrayBuffer) {
|
|
686
|
+
csvPayload = csvInput;
|
|
687
|
+
transfer = [csvInput];
|
|
688
|
+
}
|
|
689
|
+
else if (ArrayBuffer.isView(csvInput)) {
|
|
690
|
+
csvPayload = csvInput;
|
|
691
|
+
if (csvInput.buffer instanceof ArrayBuffer) {
|
|
692
|
+
transfer = [csvInput.buffer];
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
else if (typeof csvInput !== 'string') {
|
|
696
|
+
throw new ValidationError('Input must be a CSV string, File, or ArrayBuffer');
|
|
697
|
+
}
|
|
698
|
+
// ????????? ?????? ????? pool
|
|
699
|
+
const execOptions = transfer ? { transfer } : {};
|
|
700
|
+
return pool.exec('parseCSV', [csvPayload, options], execOptions, onProgress);
|
|
701
|
+
}
|
|
702
|
+
/**
|
|
703
|
+
* Чтение файла как текст
|
|
704
|
+
* @private
|
|
705
|
+
*/
|
|
706
|
+
async function readFileAsArrayBuffer(file) {
|
|
707
|
+
return new Promise((resolve, reject) => {
|
|
708
|
+
const reader = new FileReader();
|
|
709
|
+
reader.onload = (event) => resolve(event.target.result);
|
|
710
|
+
reader.onerror = (error) => reject(error);
|
|
711
|
+
reader.readAsArrayBuffer(file);
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
// Экспорт для Node.js совместимости
|
|
715
|
+
if (typeof module !== 'undefined' && module.exports) {
|
|
716
|
+
module.exports = {
|
|
717
|
+
WorkerPool,
|
|
718
|
+
createWorkerPool,
|
|
719
|
+
parseCSVWithWorker
|
|
720
|
+
};
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
var workerPool = /*#__PURE__*/Object.freeze({
|
|
724
|
+
__proto__: null,
|
|
725
|
+
WorkerPool: WorkerPool,
|
|
726
|
+
createWorkerPool: createWorkerPool,
|
|
727
|
+
parseCSVWithWorker: parseCSVWithWorker
|
|
728
|
+
});
|
|
729
|
+
|
|
730
|
+
// Расширение Web Workers для jtcsv
|
|
731
|
+
// Дополнительный модуль для параллельной обработки больших CSV
|
|
732
|
+
async function createWorkerPoolLazy(options = {}) {
|
|
733
|
+
const mod = await Promise.resolve().then(function () { return workerPool; });
|
|
734
|
+
return mod.createWorkerPool(options);
|
|
735
|
+
}
|
|
736
|
+
async function parseCSVWithWorkerLazy(csvInput, options = {}, onProgress = null) {
|
|
737
|
+
const mod = await Promise.resolve().then(function () { return workerPool; });
|
|
738
|
+
return mod.parseCSVWithWorker(csvInput, options, onProgress);
|
|
739
|
+
}
|
|
740
|
+
const jtcsvWorkers = {
|
|
741
|
+
createWorkerPool,
|
|
742
|
+
parseCSVWithWorker,
|
|
743
|
+
createWorkerPoolLazy,
|
|
744
|
+
parseCSVWithWorkerLazy
|
|
745
|
+
};
|
|
746
|
+
// Экспорт
|
|
747
|
+
if (typeof module !== 'undefined' && module.exports) {
|
|
748
|
+
module.exports = jtcsvWorkers;
|
|
749
|
+
}
|
|
750
|
+
else if (typeof define === 'function' && define.amd) {
|
|
751
|
+
define([], () => jtcsvWorkers);
|
|
752
|
+
}
|
|
753
|
+
else if (typeof window !== 'undefined' && window.jtcsv) {
|
|
754
|
+
// Расширяем глобальный jtcsv, если он существует
|
|
755
|
+
Object.assign(window.jtcsv, jtcsvWorkers);
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
export { createWorkerPool, createWorkerPoolLazy, jtcsvWorkers as default, parseCSVWithWorker, parseCSVWithWorkerLazy };
|
|
759
|
+
//# sourceMappingURL=jtcsv-workers.esm.js.map
|