jtcsv 2.1.3 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +60 -341
- package/bin/jtcsv.js +2462 -1372
- package/csv-to-json.js +35 -26
- package/dist/jtcsv.cjs.js +807 -133
- package/dist/jtcsv.cjs.js.map +1 -1
- package/dist/jtcsv.esm.js +800 -134
- package/dist/jtcsv.esm.js.map +1 -1
- package/dist/jtcsv.umd.js +807 -133
- package/dist/jtcsv.umd.js.map +1 -1
- package/errors.js +20 -0
- package/examples/browser-vanilla.html +37 -0
- package/examples/cli-batch-processing.js +38 -0
- package/examples/error-handling.js +324 -0
- package/examples/ndjson-processing.js +434 -0
- package/examples/react-integration.jsx +637 -0
- package/examples/schema-validation.js +640 -0
- package/examples/simple-usage.js +10 -7
- package/examples/typescript-example.ts +486 -0
- package/examples/web-workers-advanced.js +28 -0
- package/index.d.ts +2 -0
- package/json-save.js +2 -1
- package/json-to-csv.js +171 -131
- package/package.json +20 -4
- package/plugins/README.md +41 -467
- package/plugins/express-middleware/README.md +32 -274
- package/plugins/hono/README.md +16 -13
- package/plugins/nestjs/README.md +13 -11
- package/plugins/nextjs-api/README.md +28 -423
- package/plugins/nextjs-api/index.js +1 -2
- package/plugins/nextjs-api/route.js +1 -2
- package/plugins/nuxt/README.md +6 -7
- package/plugins/remix/README.md +9 -9
- package/plugins/sveltekit/README.md +8 -8
- package/plugins/trpc/README.md +8 -5
- package/src/browser/browser-functions.js +33 -3
- package/src/browser/csv-to-json-browser.js +269 -11
- package/src/browser/errors-browser.js +19 -1
- package/src/browser/index.js +39 -5
- package/src/browser/streams.js +393 -0
- package/src/browser/workers/csv-parser.worker.js +20 -2
- package/src/browser/workers/worker-pool.js +507 -447
- package/src/core/plugin-system.js +4 -0
- package/src/engines/fast-path-engine.js +31 -23
- package/src/errors.js +26 -0
- package/src/formats/ndjson-parser.js +54 -5
- package/src/formats/tsv-parser.js +4 -1
- package/src/utils/schema-validator.js +594 -0
- package/src/utils/transform-loader.js +205 -0
- package/src/web-server/index.js +683 -0
- package/stream-csv-to-json.js +16 -87
- package/stream-json-to-csv.js +18 -86
|
@@ -1,467 +1,527 @@
|
|
|
1
|
-
// Worker Pool для параллельной обработки CSV
|
|
2
|
-
// Использует Comlink для простой коммуникации с Web Workers
|
|
3
|
-
|
|
4
|
-
import { ValidationError, ConfigurationError } from '../errors-browser.js';
|
|
5
|
-
|
|
6
|
-
// Проверка поддержки Web Workers
|
|
7
|
-
const WORKERS_SUPPORTED = typeof Worker !== 'undefined';
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
* @property {number} [workerCount=4] - Количество workers в pool
|
|
13
|
-
* @property {number} [maxQueueSize=100] - Максимальный размер очереди задач
|
|
14
|
-
* @property {boolean} [autoScale=true] - Автоматическое масштабирование pool
|
|
15
|
-
* @property {number} [idleTimeout=60000] - Таймаут простоя worker (мс)
|
|
16
|
-
*/
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* Статистика Worker Pool
|
|
20
|
-
* @typedef {Object} WorkerPoolStats
|
|
21
|
-
* @property {number} totalWorkers - Всего workers
|
|
22
|
-
* @property {number} activeWorkers - Активные workers
|
|
23
|
-
* @property {number} idleWorkers - Простаивающие workers
|
|
24
|
-
* @property {number} queueSize - Размер очереди
|
|
25
|
-
* @property {number} tasksCompleted - Завершенные задачи
|
|
26
|
-
* @property {number} tasksFailed - Неудачные задачи
|
|
27
|
-
*/
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Прогресс обработки задачи
|
|
31
|
-
* @typedef {Object} TaskProgress
|
|
32
|
-
* @property {number} processed - Обработано элементов
|
|
33
|
-
* @property {number} total - Всего элементов
|
|
34
|
-
* @property {number} percentage - Процент выполнения
|
|
35
|
-
* @property {number} speed - Скорость обработки (элементов/сек)
|
|
36
|
-
*/
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Worker Pool для параллельной обработки CSV
|
|
40
|
-
*/
|
|
41
|
-
export class WorkerPool {
|
|
42
|
-
/**
|
|
43
|
-
* Создает новый Worker Pool
|
|
44
|
-
* @param {string} workerScript - URL скрипта worker
|
|
45
|
-
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
46
|
-
*/
|
|
47
|
-
constructor(workerScript, options = {}) {
|
|
48
|
-
if (!WORKERS_SUPPORTED) {
|
|
49
|
-
throw new ValidationError('Web Workers не поддерживаются в этом браузере');
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
this.workerScript = workerScript;
|
|
53
|
-
this.options = {
|
|
54
|
-
workerCount: 4,
|
|
55
|
-
maxQueueSize: 100,
|
|
56
|
-
autoScale: true,
|
|
57
|
-
idleTimeout: 60000,
|
|
58
|
-
...options
|
|
59
|
-
};
|
|
60
|
-
|
|
61
|
-
this.workers = [];
|
|
62
|
-
this.taskQueue = [];
|
|
63
|
-
this.activeTasks = new Map();
|
|
64
|
-
this.stats = {
|
|
65
|
-
totalWorkers: 0,
|
|
66
|
-
activeWorkers: 0,
|
|
67
|
-
idleWorkers: 0,
|
|
68
|
-
queueSize: 0,
|
|
69
|
-
tasksCompleted: 0,
|
|
70
|
-
tasksFailed: 0
|
|
71
|
-
};
|
|
72
|
-
|
|
73
|
-
this.initializeWorkers();
|
|
1
|
+
// Worker Pool для параллельной обработки CSV
|
|
2
|
+
// Использует Comlink для простой коммуникации с Web Workers
|
|
3
|
+
|
|
4
|
+
import { ValidationError, ConfigurationError } from '../errors-browser.js';
|
|
5
|
+
|
|
6
|
+
// Проверка поддержки Web Workers
|
|
7
|
+
const WORKERS_SUPPORTED = typeof Worker !== 'undefined';
|
|
8
|
+
|
|
9
|
+
function isTransferableBuffer(value) {
|
|
10
|
+
if (!(value instanceof ArrayBuffer)) {
|
|
11
|
+
return false;
|
|
74
12
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
* Инициализация workers
|
|
78
|
-
* @private
|
|
79
|
-
*/
|
|
80
|
-
initializeWorkers() {
|
|
81
|
-
const { workerCount } = this.options;
|
|
82
|
-
|
|
83
|
-
for (let i = 0; i < workerCount; i++) {
|
|
84
|
-
this.createWorker();
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
this.updateStats();
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* Создает нового worker
|
|
92
|
-
* @private
|
|
93
|
-
*/
|
|
94
|
-
createWorker() {
|
|
95
|
-
try {
|
|
96
|
-
const worker = new Worker(this.workerScript, { type: 'module' });
|
|
97
|
-
|
|
98
|
-
worker.id = `worker-${this.workers.length}`;
|
|
99
|
-
worker.status = 'idle';
|
|
100
|
-
worker.lastUsed = Date.now();
|
|
101
|
-
worker.taskId = null;
|
|
102
|
-
|
|
103
|
-
// Обработчики событий
|
|
104
|
-
worker.onmessage = (event) => this.handleWorkerMessage(worker, event);
|
|
105
|
-
worker.onerror = (error) => this.handleWorkerError(worker, error);
|
|
106
|
-
worker.onmessageerror = (error) => this.handleWorkerMessageError(worker, error);
|
|
107
|
-
|
|
108
|
-
this.workers.push(worker);
|
|
109
|
-
this.stats.totalWorkers++;
|
|
110
|
-
this.stats.idleWorkers++;
|
|
111
|
-
|
|
112
|
-
return worker;
|
|
113
|
-
} catch (error) {
|
|
114
|
-
throw new ConfigurationError(`Не удалось создать worker: ${error.message}`);
|
|
115
|
-
}
|
|
13
|
+
if (typeof SharedArrayBuffer !== 'undefined' && value instanceof SharedArrayBuffer) {
|
|
14
|
+
return false;
|
|
116
15
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
*
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
worker.
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
16
|
+
return true;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function collectTransferables(args) {
|
|
20
|
+
const transferables = [];
|
|
21
|
+
|
|
22
|
+
const collectFromValue = (value) => {
|
|
23
|
+
if (!value) {
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
if (isTransferableBuffer(value)) {
|
|
27
|
+
transferables.push(value);
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
if (ArrayBuffer.isView(value) && isTransferableBuffer(value.buffer)) {
|
|
31
|
+
transferables.push(value.buffer);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (Array.isArray(value)) {
|
|
35
|
+
value.forEach(collectFromValue);
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
args.forEach(collectFromValue);
|
|
40
|
+
return transferables.length ? transferables : null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Опции для Worker Pool
|
|
45
|
+
* @typedef {Object} WorkerPoolOptions
|
|
46
|
+
* @property {number} [workerCount=4] - Количество workers в pool
|
|
47
|
+
* @property {number} [maxQueueSize=100] - Максимальный размер очереди задач
|
|
48
|
+
* @property {boolean} [autoScale=true] - Автоматическое масштабирование pool
|
|
49
|
+
* @property {number} [idleTimeout=60000] - Таймаут простоя worker (мс)
|
|
50
|
+
*/
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Статистика Worker Pool
|
|
54
|
+
* @typedef {Object} WorkerPoolStats
|
|
55
|
+
* @property {number} totalWorkers - Всего workers
|
|
56
|
+
* @property {number} activeWorkers - Активные workers
|
|
57
|
+
* @property {number} idleWorkers - Простаивающие workers
|
|
58
|
+
* @property {number} queueSize - Размер очереди
|
|
59
|
+
* @property {number} tasksCompleted - Завершенные задачи
|
|
60
|
+
* @property {number} tasksFailed - Неудачные задачи
|
|
61
|
+
*/
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Прогресс обработки задачи
|
|
65
|
+
* @typedef {Object} TaskProgress
|
|
66
|
+
* @property {number} processed - Обработано элементов
|
|
67
|
+
* @property {number} total - Всего элементов
|
|
68
|
+
* @property {number} percentage - Процент выполнения
|
|
69
|
+
* @property {number} speed - Скорость обработки (элементов/сек)
|
|
70
|
+
*/
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Worker Pool для параллельной обработки CSV
|
|
74
|
+
*/
|
|
75
|
+
export class WorkerPool {
|
|
76
|
+
/**
|
|
77
|
+
* Создает новый Worker Pool
|
|
78
|
+
* @param {string} workerScript - URL скрипта worker
|
|
79
|
+
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
80
|
+
*/
|
|
81
|
+
constructor(workerScript, options = {}) {
|
|
82
|
+
if (!WORKERS_SUPPORTED) {
|
|
83
|
+
throw new ValidationError('Web Workers не поддерживаются в этом браузере');
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
this.workerScript = workerScript;
|
|
87
|
+
this.options = {
|
|
88
|
+
workerCount: 4,
|
|
89
|
+
maxQueueSize: 100,
|
|
90
|
+
autoScale: true,
|
|
91
|
+
idleTimeout: 60000,
|
|
92
|
+
...options
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
this.workers = [];
|
|
96
|
+
this.taskQueue = [];
|
|
97
|
+
this.activeTasks = new Map();
|
|
98
|
+
this.stats = {
|
|
99
|
+
totalWorkers: 0,
|
|
100
|
+
activeWorkers: 0,
|
|
101
|
+
idleWorkers: 0,
|
|
102
|
+
queueSize: 0,
|
|
103
|
+
tasksCompleted: 0,
|
|
104
|
+
tasksFailed: 0
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
this.initializeWorkers();
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Инициализация workers
|
|
112
|
+
* @private
|
|
113
|
+
*/
|
|
114
|
+
initializeWorkers() {
|
|
115
|
+
const { workerCount } = this.options;
|
|
116
|
+
|
|
117
|
+
for (let i = 0; i < workerCount; i++) {
|
|
118
|
+
this.createWorker();
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
this.updateStats();
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Создает нового worker
|
|
126
|
+
* @private
|
|
127
|
+
*/
|
|
128
|
+
createWorker() {
|
|
129
|
+
try {
|
|
130
|
+
const worker = new Worker(this.workerScript, { type: 'module' });
|
|
131
|
+
|
|
132
|
+
worker.id = `worker-${this.workers.length}`;
|
|
133
|
+
worker.status = 'idle';
|
|
134
|
+
worker.lastUsed = Date.now();
|
|
135
|
+
worker.taskId = null;
|
|
136
|
+
|
|
137
|
+
// Обработчики событий
|
|
138
|
+
worker.onmessage = (event) => this.handleWorkerMessage(worker, event);
|
|
139
|
+
worker.onerror = (error) => this.handleWorkerError(worker, error);
|
|
140
|
+
worker.onmessageerror = (error) => this.handleWorkerMessageError(worker, error);
|
|
141
|
+
|
|
142
|
+
this.workers.push(worker);
|
|
143
|
+
this.stats.totalWorkers++;
|
|
144
|
+
this.stats.idleWorkers++;
|
|
145
|
+
|
|
146
|
+
return worker;
|
|
147
|
+
} catch (error) {
|
|
148
|
+
throw new ConfigurationError(`Не удалось создать worker: ${error.message}`);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Обработка сообщений от worker
|
|
154
|
+
* @private
|
|
155
|
+
*/
|
|
156
|
+
handleWorkerMessage(worker, event) {
|
|
157
|
+
const { data } = event;
|
|
158
|
+
|
|
159
|
+
if (data.type === 'PROGRESS') {
|
|
160
|
+
this.handleProgress(worker, data);
|
|
161
|
+
} else if (data.type === 'RESULT') {
|
|
162
|
+
this.handleResult(worker, data);
|
|
163
|
+
} else if (data.type === 'ERROR') {
|
|
164
|
+
this.handleWorkerTaskError(worker, data);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Обработка прогресса задачи
|
|
170
|
+
* @private
|
|
171
|
+
*/
|
|
172
|
+
handleProgress(worker, progressData) {
|
|
173
|
+
const taskId = worker.taskId;
|
|
174
|
+
if (taskId && this.activeTasks.has(taskId)) {
|
|
175
|
+
const task = this.activeTasks.get(taskId);
|
|
176
|
+
if (task.onProgress) {
|
|
177
|
+
task.onProgress({
|
|
178
|
+
processed: progressData.processed,
|
|
179
|
+
total: progressData.total,
|
|
180
|
+
percentage: (progressData.processed / progressData.total) * 100,
|
|
181
|
+
speed: progressData.speed || 0
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Обработка результата задачи
|
|
189
|
+
* @private
|
|
190
|
+
*/
|
|
191
|
+
handleResult(worker, resultData) {
|
|
192
|
+
const taskId = worker.taskId;
|
|
193
|
+
if (taskId && this.activeTasks.has(taskId)) {
|
|
194
|
+
const task = this.activeTasks.get(taskId);
|
|
195
|
+
|
|
196
|
+
// Освобождение worker
|
|
197
|
+
worker.status = 'idle';
|
|
198
|
+
worker.lastUsed = Date.now();
|
|
199
|
+
worker.taskId = null;
|
|
200
|
+
this.stats.activeWorkers--;
|
|
201
|
+
this.stats.idleWorkers++;
|
|
202
|
+
|
|
203
|
+
// Завершение задачи
|
|
204
|
+
task.resolve(resultData.data);
|
|
205
|
+
this.activeTasks.delete(taskId);
|
|
206
|
+
this.stats.tasksCompleted++;
|
|
207
|
+
|
|
208
|
+
// Обработка следующей задачи в очереди
|
|
209
|
+
this.processQueue();
|
|
210
|
+
this.updateStats();
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Обработка ошибки задачи
|
|
216
|
+
* @private
|
|
217
|
+
*/
|
|
218
|
+
handleWorkerTaskError(worker, errorData) {
|
|
219
|
+
const taskId = worker.taskId;
|
|
220
|
+
if (taskId && this.activeTasks.has(taskId)) {
|
|
221
|
+
const task = this.activeTasks.get(taskId);
|
|
222
|
+
|
|
223
|
+
// Освобождение worker
|
|
224
|
+
worker.status = 'idle';
|
|
225
|
+
worker.lastUsed = Date.now();
|
|
226
|
+
worker.taskId = null;
|
|
227
|
+
this.stats.activeWorkers--;
|
|
228
|
+
this.stats.idleWorkers++;
|
|
229
|
+
|
|
230
|
+
// Завершение с ошибкой
|
|
231
|
+
const workerError = new Error(errorData.message || 'Ошибка в worker');
|
|
232
|
+
if (errorData.code) {
|
|
233
|
+
workerError.code = errorData.code;
|
|
234
|
+
}
|
|
235
|
+
if (errorData.details) {
|
|
236
|
+
workerError.details = errorData.details;
|
|
237
|
+
}
|
|
238
|
+
task.reject(workerError);
|
|
239
|
+
this.activeTasks.delete(taskId);
|
|
240
|
+
this.stats.tasksFailed++;
|
|
241
|
+
|
|
242
|
+
// Обработка следующей задачи
|
|
243
|
+
this.processQueue();
|
|
244
|
+
this.updateStats();
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Обработка ошибок worker
|
|
250
|
+
* @private
|
|
251
|
+
*/
|
|
252
|
+
handleWorkerError(worker, error) {
|
|
253
|
+
console.error(`Worker ${worker.id} error:`, error);
|
|
254
|
+
|
|
255
|
+
// Перезапуск worker
|
|
256
|
+
this.restartWorker(worker);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Обработка ошибок сообщений
|
|
261
|
+
* @private
|
|
262
|
+
*/
|
|
263
|
+
handleWorkerMessageError(worker, error) {
|
|
264
|
+
console.error(`Worker ${worker.id} message error:`, error);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Перезапуск worker
|
|
269
|
+
* @private
|
|
270
|
+
*/
|
|
271
|
+
restartWorker(worker) {
|
|
272
|
+
const index = this.workers.indexOf(worker);
|
|
273
|
+
if (index !== -1) {
|
|
274
|
+
// Завершение старого worker
|
|
275
|
+
worker.terminate();
|
|
276
|
+
|
|
277
|
+
// Удаление из статистики
|
|
278
|
+
if (worker.status === 'active') {
|
|
279
|
+
this.stats.activeWorkers--;
|
|
280
|
+
} else {
|
|
281
|
+
this.stats.idleWorkers--;
|
|
282
|
+
}
|
|
283
|
+
this.stats.totalWorkers--;
|
|
284
|
+
|
|
285
|
+
// Создание нового worker
|
|
286
|
+
const newWorker = this.createWorker();
|
|
287
|
+
this.workers[index] = newWorker;
|
|
288
|
+
|
|
289
|
+
// Перезапуск задачи если была активна
|
|
290
|
+
if (worker.taskId && this.activeTasks.has(worker.taskId)) {
|
|
291
|
+
const task = this.activeTasks.get(worker.taskId);
|
|
292
|
+
this.executeTask(newWorker, task);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* Выполнение задачи на worker
|
|
299
|
+
* @private
|
|
300
|
+
*/
|
|
301
|
+
executeTask(worker, task) {
|
|
302
|
+
worker.status = 'active';
|
|
303
|
+
worker.lastUsed = Date.now();
|
|
304
|
+
worker.taskId = task.id;
|
|
305
|
+
|
|
306
|
+
this.stats.idleWorkers--;
|
|
307
|
+
this.stats.activeWorkers++;
|
|
308
|
+
|
|
309
|
+
// Отправка задачи в worker
|
|
310
|
+
const payload = {
|
|
311
|
+
type: 'EXECUTE',
|
|
312
|
+
taskId: task.id,
|
|
313
|
+
method: task.method,
|
|
314
|
+
args: task.args,
|
|
315
|
+
options: task.options
|
|
316
|
+
};
|
|
317
|
+
|
|
318
|
+
if (task.transferList && task.transferList.length) {
|
|
319
|
+
worker.postMessage(payload, task.transferList);
|
|
320
|
+
} else {
|
|
321
|
+
worker.postMessage(payload);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Обработка очереди задач
|
|
327
|
+
* @private
|
|
328
|
+
*/
|
|
282
329
|
processQueue() {
|
|
283
330
|
if (this.taskQueue.length === 0) {
|
|
284
331
|
return;
|
|
285
332
|
}
|
|
286
333
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
this.processQueue();
|
|
294
|
-
}
|
|
295
|
-
return;
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
// Получение задачи из очереди
|
|
299
|
-
const task = this.taskQueue.shift();
|
|
300
|
-
this.stats.queueSize--;
|
|
301
|
-
|
|
302
|
-
// Выполнение задачи
|
|
303
|
-
this.executeTask(idleWorker, task);
|
|
304
|
-
this.updateStats();
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
/**
|
|
308
|
-
* Обновление статистики
|
|
309
|
-
* @private
|
|
310
|
-
*/
|
|
311
|
-
updateStats() {
|
|
312
|
-
this.stats.queueSize = this.taskQueue.length;
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
/**
|
|
316
|
-
* Выполнение задачи через pool
|
|
317
|
-
* @param {string} method - Метод для вызова в worker
|
|
318
|
-
* @param {Array} args - Аргументы метода
|
|
319
|
-
* @param {Object} [options] - Опции задачи
|
|
320
|
-
* @param {Function} [onProgress] - Callback прогресса
|
|
321
|
-
* @returns {Promise<any>} Результат выполнения
|
|
322
|
-
*/
|
|
323
|
-
async exec(method, args = [], options = {}, onProgress = null) {
|
|
324
|
-
return new Promise((resolve, reject) => {
|
|
325
|
-
// Проверка размера очереди
|
|
326
|
-
if (this.taskQueue.length >= this.options.maxQueueSize) {
|
|
327
|
-
reject(new Error('Очередь задач переполнена'));
|
|
328
|
-
return;
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
// Создание задачи
|
|
332
|
-
const taskId = `task-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
333
|
-
const task = {
|
|
334
|
-
id: taskId,
|
|
335
|
-
method,
|
|
336
|
-
args,
|
|
337
|
-
options,
|
|
338
|
-
onProgress,
|
|
339
|
-
resolve,
|
|
340
|
-
reject,
|
|
341
|
-
createdAt: Date.now()
|
|
342
|
-
};
|
|
343
|
-
|
|
344
|
-
// Добавление в очередь
|
|
345
|
-
this.taskQueue.push(task);
|
|
346
|
-
this.stats.queueSize++;
|
|
347
|
-
|
|
348
|
-
// Запуск обработки очереди
|
|
349
|
-
this.processQueue();
|
|
350
|
-
this.updateStats();
|
|
351
|
-
});
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
/**
|
|
355
|
-
* Получение статистики pool
|
|
356
|
-
* @returns {WorkerPoolStats} Статистика
|
|
357
|
-
*/
|
|
358
|
-
getStats() {
|
|
359
|
-
return { ...this.stats };
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
/**
|
|
363
|
-
* Очистка простаивающих workers
|
|
364
|
-
*/
|
|
365
|
-
cleanupIdleWorkers() {
|
|
366
|
-
const now = Date.now();
|
|
367
|
-
const { idleTimeout } = this.options;
|
|
368
|
-
|
|
369
|
-
for (let i = this.workers.length - 1; i >= 0; i--) {
|
|
370
|
-
const worker = this.workers[i];
|
|
371
|
-
if (worker.status === 'idle' && (now - worker.lastUsed) > idleTimeout) {
|
|
372
|
-
// Сохранение минимального количества workers
|
|
373
|
-
if (this.workers.length > 1) {
|
|
374
|
-
worker.terminate();
|
|
375
|
-
this.workers.splice(i, 1);
|
|
376
|
-
this.stats.totalWorkers--;
|
|
377
|
-
this.stats.idleWorkers--;
|
|
334
|
+
while (this.taskQueue.length > 0) {
|
|
335
|
+
const idleWorker = this.workers.find(w => w.status === 'idle');
|
|
336
|
+
if (!idleWorker) {
|
|
337
|
+
if (this.options.autoScale && this.workers.length < this.options.maxQueueSize) {
|
|
338
|
+
this.createWorker();
|
|
339
|
+
continue;
|
|
378
340
|
}
|
|
341
|
+
break;
|
|
379
342
|
}
|
|
343
|
+
|
|
344
|
+
const task = this.taskQueue.shift();
|
|
345
|
+
this.stats.queueSize--;
|
|
346
|
+
this.executeTask(idleWorker, task);
|
|
380
347
|
}
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
/**
|
|
384
|
-
* Завершение всех workers
|
|
385
|
-
*/
|
|
386
|
-
terminate() {
|
|
387
|
-
this.workers.forEach(worker => {
|
|
388
|
-
worker.terminate();
|
|
389
|
-
});
|
|
390
|
-
|
|
391
|
-
this.workers = [];
|
|
392
|
-
this.taskQueue = [];
|
|
393
|
-
this.activeTasks.clear();
|
|
394
348
|
|
|
395
|
-
|
|
396
|
-
this.stats = {
|
|
397
|
-
totalWorkers: 0,
|
|
398
|
-
activeWorkers: 0,
|
|
399
|
-
idleWorkers: 0,
|
|
400
|
-
queueSize: 0,
|
|
401
|
-
tasksCompleted: 0,
|
|
402
|
-
tasksFailed: 0
|
|
403
|
-
};
|
|
404
|
-
}
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
/**
|
|
408
|
-
* Создает Worker Pool для обработки CSV
|
|
409
|
-
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
410
|
-
* @returns {WorkerPool} Worker Pool
|
|
411
|
-
*/
|
|
412
|
-
export function createWorkerPool(options = {}) {
|
|
413
|
-
// Используем встроенный worker скрипт
|
|
414
|
-
const workerScript = new URL('./csv-parser.worker.js', import.meta.url).href;
|
|
415
|
-
return new WorkerPool(workerScript, options);
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
/**
|
|
419
|
-
* Парсит CSV с использованием Web Workers
|
|
420
|
-
* @param {string|File} csvInput - CSV строка или File объект
|
|
421
|
-
* @param {Object} [options] - Опции парсинга
|
|
422
|
-
* @param {Function} [onProgress] - Callback прогресса
|
|
423
|
-
* @returns {Promise<Array<Object>>} JSON данные
|
|
424
|
-
*/
|
|
425
|
-
export async function parseCSVWithWorker(csvInput, options = {}, onProgress = null) {
|
|
426
|
-
// Создание pool если нужно
|
|
427
|
-
if (!parseCSVWithWorker.pool) {
|
|
428
|
-
parseCSVWithWorker.pool = createWorkerPool();
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
const pool = parseCSVWithWorker.pool;
|
|
432
|
-
|
|
433
|
-
// Подготовка CSV строки
|
|
434
|
-
let csvString;
|
|
435
|
-
if (csvInput instanceof File) {
|
|
436
|
-
csvString = await readFileAsText(csvInput);
|
|
437
|
-
} else if (typeof csvInput === 'string') {
|
|
438
|
-
csvString = csvInput;
|
|
439
|
-
} else {
|
|
440
|
-
throw new ValidationError('Input must be a CSV string or File object');
|
|
349
|
+
this.updateStats();
|
|
441
350
|
}
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Обновление статистики
|
|
354
|
+
* @private
|
|
355
|
+
*/
|
|
356
|
+
updateStats() {
|
|
357
|
+
this.stats.queueSize = this.taskQueue.length;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Выполнение задачи через pool
|
|
362
|
+
* @param {string} method - Метод для вызова в worker
|
|
363
|
+
* @param {Array} args - Аргументы метода
|
|
364
|
+
* @param {Object} [options] - Опции задачи
|
|
365
|
+
* @param {Function} [onProgress] - Callback прогресса
|
|
366
|
+
* @returns {Promise<any>} Результат выполнения
|
|
367
|
+
*/
|
|
368
|
+
async exec(method, args = [], options = {}, onProgress = null) {
|
|
369
|
+
return new Promise((resolve, reject) => {
|
|
370
|
+
// Проверка размера очереди
|
|
371
|
+
if (this.taskQueue.length >= this.options.maxQueueSize) {
|
|
372
|
+
reject(new Error('Очередь задач переполнена'));
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// Создание задачи
|
|
377
|
+
const taskId = `task-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
378
|
+
const { transfer, ...taskOptions } = options || {};
|
|
379
|
+
const transferList = transfer || collectTransferables(args);
|
|
380
|
+
const task = {
|
|
381
|
+
id: taskId,
|
|
382
|
+
method,
|
|
383
|
+
args,
|
|
384
|
+
options: taskOptions,
|
|
385
|
+
transferList,
|
|
386
|
+
onProgress,
|
|
387
|
+
resolve,
|
|
388
|
+
reject,
|
|
389
|
+
createdAt: Date.now()
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
// Добавление в очередь
|
|
393
|
+
this.taskQueue.push(task);
|
|
394
|
+
this.stats.queueSize++;
|
|
395
|
+
|
|
396
|
+
// Запуск обработки очереди
|
|
397
|
+
this.processQueue();
|
|
398
|
+
this.updateStats();
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/**
|
|
403
|
+
* Получение статистики pool
|
|
404
|
+
* @returns {WorkerPoolStats} Статистика
|
|
405
|
+
*/
|
|
406
|
+
getStats() {
|
|
407
|
+
return { ...this.stats };
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* Очистка простаивающих workers
|
|
412
|
+
*/
|
|
413
|
+
cleanupIdleWorkers() {
|
|
414
|
+
const now = Date.now();
|
|
415
|
+
const { idleTimeout } = this.options;
|
|
416
|
+
|
|
417
|
+
for (let i = this.workers.length - 1; i >= 0; i--) {
|
|
418
|
+
const worker = this.workers[i];
|
|
419
|
+
if (worker.status === 'idle' && (now - worker.lastUsed) > idleTimeout) {
|
|
420
|
+
// Сохранение минимального количества workers
|
|
421
|
+
if (this.workers.length > 1) {
|
|
422
|
+
worker.terminate();
|
|
423
|
+
this.workers.splice(i, 1);
|
|
424
|
+
this.stats.totalWorkers--;
|
|
425
|
+
this.stats.idleWorkers--;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/**
|
|
432
|
+
* Завершение всех workers
|
|
433
|
+
*/
|
|
434
|
+
terminate() {
|
|
435
|
+
this.workers.forEach(worker => {
|
|
436
|
+
worker.terminate();
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
this.workers = [];
|
|
440
|
+
this.taskQueue = [];
|
|
441
|
+
this.activeTasks.clear();
|
|
442
|
+
|
|
443
|
+
// Сброс статистики
|
|
444
|
+
this.stats = {
|
|
445
|
+
totalWorkers: 0,
|
|
446
|
+
activeWorkers: 0,
|
|
447
|
+
idleWorkers: 0,
|
|
448
|
+
queueSize: 0,
|
|
449
|
+
tasksCompleted: 0,
|
|
450
|
+
tasksFailed: 0
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/**
|
|
456
|
+
* Создает Worker Pool для обработки CSV
|
|
457
|
+
* @param {WorkerPoolOptions} [options] - Опции pool
|
|
458
|
+
* @returns {WorkerPool} Worker Pool
|
|
459
|
+
*/
|
|
460
|
+
export function createWorkerPool(options = {}) {
|
|
461
|
+
// Используем встроенный worker скрипт
|
|
462
|
+
const workerScript = new URL('./csv-parser.worker.js', import.meta.url).href;
|
|
463
|
+
return new WorkerPool(workerScript, options);
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Парсит CSV с использованием Web Workers
|
|
468
|
+
* @param {string|File} csvInput - CSV строка или File объект
|
|
469
|
+
* @param {Object} [options] - Опции парсинга
|
|
470
|
+
* @param {Function} [onProgress] - Callback прогресса
|
|
471
|
+
* @returns {Promise<Array<Object>>} JSON данные
|
|
472
|
+
*/
|
|
473
|
+
export async function parseCSVWithWorker(csvInput, options = {}, onProgress = null) {
|
|
474
|
+
// Создание pool если нужно
|
|
475
|
+
if (!parseCSVWithWorker.pool) {
|
|
476
|
+
parseCSVWithWorker.pool = createWorkerPool();
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
const pool = parseCSVWithWorker.pool;
|
|
480
|
+
|
|
481
|
+
// Подготовка CSV строки
|
|
482
|
+
// ?????????? CSV ??????
|
|
483
|
+
let csvPayload = csvInput;
|
|
484
|
+
let transfer = null;
|
|
485
|
+
|
|
486
|
+
if (csvInput instanceof File) {
|
|
487
|
+
const buffer = await readFileAsArrayBuffer(csvInput);
|
|
488
|
+
csvPayload = new Uint8Array(buffer);
|
|
489
|
+
transfer = [buffer];
|
|
490
|
+
} else if (csvInput instanceof ArrayBuffer) {
|
|
491
|
+
csvPayload = csvInput;
|
|
492
|
+
transfer = [csvInput];
|
|
493
|
+
} else if (ArrayBuffer.isView(csvInput)) {
|
|
494
|
+
csvPayload = csvInput;
|
|
495
|
+
if (csvInput.buffer instanceof ArrayBuffer) {
|
|
496
|
+
transfer = [csvInput.buffer];
|
|
497
|
+
}
|
|
498
|
+
} else if (typeof csvInput !== 'string') {
|
|
499
|
+
throw new ValidationError('Input must be a CSV string, File, or ArrayBuffer');
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// ????????? ?????? ????? pool
|
|
503
|
+
const execOptions = transfer ? { transfer } : {};
|
|
504
|
+
return pool.exec('parseCSV', [csvPayload, options], execOptions, onProgress);
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* Чтение файла как текст
|
|
509
|
+
* @private
|
|
510
|
+
*/
|
|
511
|
+
async function readFileAsArrayBuffer(file) {
|
|
452
512
|
return new Promise((resolve, reject) => {
|
|
453
513
|
const reader = new FileReader();
|
|
454
514
|
reader.onload = (event) => resolve(event.target.result);
|
|
455
515
|
reader.onerror = (error) => reject(error);
|
|
456
|
-
reader.
|
|
516
|
+
reader.readAsArrayBuffer(file);
|
|
457
517
|
});
|
|
458
518
|
}
|
|
459
519
|
|
|
460
|
-
// Экспорт для Node.js совместимости
|
|
461
|
-
if (typeof module !== 'undefined' && module.exports) {
|
|
462
|
-
module.exports = {
|
|
463
|
-
WorkerPool,
|
|
464
|
-
createWorkerPool,
|
|
465
|
-
parseCSVWithWorker
|
|
466
|
-
};
|
|
520
|
+
// Экспорт для Node.js совместимости
|
|
521
|
+
if (typeof module !== 'undefined' && module.exports) {
|
|
522
|
+
module.exports = {
|
|
523
|
+
WorkerPool,
|
|
524
|
+
createWorkerPool,
|
|
525
|
+
parseCSVWithWorker
|
|
526
|
+
};
|
|
467
527
|
}
|