@ruvector/edge-net 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +12 -1
- package/real-agents.js +381 -12
- package/real-workers.js +970 -0
- package/real-workflows.js +739 -0
package/real-workers.js
ADDED
|
@@ -0,0 +1,970 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @ruvector/edge-net REAL Worker System
|
|
3
|
+
*
|
|
4
|
+
* Actually functional distributed workers with:
|
|
5
|
+
* - Real Node.js worker_threads for parallel execution
|
|
6
|
+
* - Real WebSocket relay for task distribution
|
|
7
|
+
* - Real result collection and aggregation
|
|
8
|
+
* - Real resource management
|
|
9
|
+
*
|
|
10
|
+
* @module @ruvector/edge-net/real-workers
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { EventEmitter } from 'events';
|
|
14
|
+
import { Worker, isMainThread, parentPort, workerData } from 'worker_threads';
|
|
15
|
+
import { randomBytes, createHash } from 'crypto';
|
|
16
|
+
import { cpus } from 'os';
|
|
17
|
+
import { fileURLToPath } from 'url';
|
|
18
|
+
import { dirname, join } from 'path';
|
|
19
|
+
|
|
20
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
21
|
+
const __dirname = dirname(__filename);
|
|
22
|
+
|
|
23
|
+
// ============================================
|
|
24
|
+
// WORKER TASK TYPES
|
|
25
|
+
// ============================================
|
|
26
|
+
|
|
27
|
+
export const WorkerTaskTypes = {
|
|
28
|
+
EMBED: 'embed',
|
|
29
|
+
PROCESS: 'process',
|
|
30
|
+
ANALYZE: 'analyze',
|
|
31
|
+
TRANSFORM: 'transform',
|
|
32
|
+
COMPUTE: 'compute',
|
|
33
|
+
AGGREGATE: 'aggregate',
|
|
34
|
+
CUSTOM: 'custom',
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
// ============================================
|
|
38
|
+
// INLINE WORKER CODE
|
|
39
|
+
// ============================================
|
|
40
|
+
|
|
41
|
+
const WORKER_CODE = `
|
|
42
|
+
const { parentPort, workerData } = require('worker_threads');
|
|
43
|
+
const crypto = require('crypto');
|
|
44
|
+
|
|
45
|
+
// Simple hash-based embedding (for fallback)
|
|
46
|
+
function hashEmbed(text, dims = 384) {
|
|
47
|
+
const hash = crypto.createHash('sha256').update(String(text)).digest();
|
|
48
|
+
const embedding = new Float32Array(dims);
|
|
49
|
+
for (let i = 0; i < dims; i++) {
|
|
50
|
+
embedding[i] = (hash[i % 32] - 128) / 128;
|
|
51
|
+
}
|
|
52
|
+
return Array.from(embedding);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Task handlers
|
|
56
|
+
const handlers = {
|
|
57
|
+
embed: (data) => {
|
|
58
|
+
if (Array.isArray(data)) {
|
|
59
|
+
return data.map(item => ({
|
|
60
|
+
text: String(item).slice(0, 100),
|
|
61
|
+
embedding: hashEmbed(item),
|
|
62
|
+
dimensions: 384,
|
|
63
|
+
}));
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
text: String(data).slice(0, 100),
|
|
67
|
+
embedding: hashEmbed(data),
|
|
68
|
+
dimensions: 384,
|
|
69
|
+
};
|
|
70
|
+
},
|
|
71
|
+
|
|
72
|
+
process: (data, options = {}) => {
|
|
73
|
+
const processor = options.processor || 'default';
|
|
74
|
+
if (Array.isArray(data)) {
|
|
75
|
+
return data.map((item, i) => ({
|
|
76
|
+
index: i,
|
|
77
|
+
processed: true,
|
|
78
|
+
result: typeof item === 'object' ? { ...item, _processed: true } : { value: item, _processed: true },
|
|
79
|
+
processor,
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
processed: true,
|
|
84
|
+
result: typeof data === 'object' ? { ...data, _processed: true } : { value: data, _processed: true },
|
|
85
|
+
processor,
|
|
86
|
+
};
|
|
87
|
+
},
|
|
88
|
+
|
|
89
|
+
analyze: (data, options = {}) => {
|
|
90
|
+
const items = Array.isArray(data) ? data : [data];
|
|
91
|
+
const stats = {
|
|
92
|
+
count: items.length,
|
|
93
|
+
types: {},
|
|
94
|
+
sizes: [],
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
for (const item of items) {
|
|
98
|
+
const type = typeof item;
|
|
99
|
+
stats.types[type] = (stats.types[type] || 0) + 1;
|
|
100
|
+
if (typeof item === 'string') {
|
|
101
|
+
stats.sizes.push(item.length);
|
|
102
|
+
} else if (typeof item === 'object' && item !== null) {
|
|
103
|
+
stats.sizes.push(JSON.stringify(item).length);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
stats.avgSize = stats.sizes.length > 0
|
|
108
|
+
? stats.sizes.reduce((a, b) => a + b, 0) / stats.sizes.length
|
|
109
|
+
: 0;
|
|
110
|
+
stats.minSize = stats.sizes.length > 0 ? Math.min(...stats.sizes) : 0;
|
|
111
|
+
stats.maxSize = stats.sizes.length > 0 ? Math.max(...stats.sizes) : 0;
|
|
112
|
+
|
|
113
|
+
return {
|
|
114
|
+
analyzed: true,
|
|
115
|
+
stats,
|
|
116
|
+
timestamp: Date.now(),
|
|
117
|
+
};
|
|
118
|
+
},
|
|
119
|
+
|
|
120
|
+
transform: (data, options = {}) => {
|
|
121
|
+
const transform = options.transform || 'identity';
|
|
122
|
+
const transforms = {
|
|
123
|
+
identity: (x) => x,
|
|
124
|
+
uppercase: (x) => typeof x === 'string' ? x.toUpperCase() : x,
|
|
125
|
+
lowercase: (x) => typeof x === 'string' ? x.toLowerCase() : x,
|
|
126
|
+
reverse: (x) => typeof x === 'string' ? x.split('').reverse().join('') : x,
|
|
127
|
+
hash: (x) => crypto.createHash('sha256').update(String(x)).digest('hex'),
|
|
128
|
+
json: (x) => JSON.stringify(x),
|
|
129
|
+
length: (x) => typeof x === 'string' ? x.length : JSON.stringify(x).length,
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
const fn = transforms[transform] || transforms.identity;
|
|
133
|
+
|
|
134
|
+
if (Array.isArray(data)) {
|
|
135
|
+
return data.map(item => ({
|
|
136
|
+
original: item,
|
|
137
|
+
transformed: fn(item),
|
|
138
|
+
transform,
|
|
139
|
+
}));
|
|
140
|
+
}
|
|
141
|
+
return {
|
|
142
|
+
original: data,
|
|
143
|
+
transformed: fn(data),
|
|
144
|
+
transform,
|
|
145
|
+
};
|
|
146
|
+
},
|
|
147
|
+
|
|
148
|
+
compute: (data, options = {}) => {
|
|
149
|
+
const operation = options.operation || 'sum';
|
|
150
|
+
const items = Array.isArray(data) ? data : [data];
|
|
151
|
+
const numbers = items.map(x => typeof x === 'number' ? x : parseFloat(x) || 0);
|
|
152
|
+
|
|
153
|
+
const operations = {
|
|
154
|
+
sum: () => numbers.reduce((a, b) => a + b, 0),
|
|
155
|
+
product: () => numbers.reduce((a, b) => a * b, 1),
|
|
156
|
+
mean: () => numbers.reduce((a, b) => a + b, 0) / numbers.length,
|
|
157
|
+
min: () => Math.min(...numbers),
|
|
158
|
+
max: () => Math.max(...numbers),
|
|
159
|
+
variance: () => {
|
|
160
|
+
const mean = numbers.reduce((a, b) => a + b, 0) / numbers.length;
|
|
161
|
+
return numbers.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / numbers.length;
|
|
162
|
+
},
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
return {
|
|
166
|
+
computed: true,
|
|
167
|
+
operation,
|
|
168
|
+
result: (operations[operation] || operations.sum)(),
|
|
169
|
+
inputCount: numbers.length,
|
|
170
|
+
};
|
|
171
|
+
},
|
|
172
|
+
|
|
173
|
+
aggregate: (data, options = {}) => {
|
|
174
|
+
const items = Array.isArray(data) ? data : [data];
|
|
175
|
+
const groupBy = options.groupBy;
|
|
176
|
+
|
|
177
|
+
if (groupBy && typeof items[0] === 'object') {
|
|
178
|
+
const groups = {};
|
|
179
|
+
for (const item of items) {
|
|
180
|
+
const key = item[groupBy] || 'undefined';
|
|
181
|
+
if (!groups[key]) groups[key] = [];
|
|
182
|
+
groups[key].push(item);
|
|
183
|
+
}
|
|
184
|
+
return {
|
|
185
|
+
aggregated: true,
|
|
186
|
+
groupBy,
|
|
187
|
+
groups: Object.keys(groups).map(key => ({
|
|
188
|
+
key,
|
|
189
|
+
count: groups[key].length,
|
|
190
|
+
items: groups[key],
|
|
191
|
+
})),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
return {
|
|
196
|
+
aggregated: true,
|
|
197
|
+
count: items.length,
|
|
198
|
+
items,
|
|
199
|
+
};
|
|
200
|
+
},
|
|
201
|
+
|
|
202
|
+
custom: (data, options = {}) => {
|
|
203
|
+
// For custom tasks, just return with metadata
|
|
204
|
+
return {
|
|
205
|
+
custom: true,
|
|
206
|
+
data,
|
|
207
|
+
options,
|
|
208
|
+
timestamp: Date.now(),
|
|
209
|
+
};
|
|
210
|
+
},
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
// Handle messages from main thread
|
|
214
|
+
parentPort.on('message', (message) => {
|
|
215
|
+
const { taskId, type, data, options } = message;
|
|
216
|
+
|
|
217
|
+
try {
|
|
218
|
+
const handler = handlers[type] || handlers.custom;
|
|
219
|
+
const result = handler(data, options);
|
|
220
|
+
|
|
221
|
+
parentPort.postMessage({
|
|
222
|
+
taskId,
|
|
223
|
+
success: true,
|
|
224
|
+
result,
|
|
225
|
+
});
|
|
226
|
+
} catch (error) {
|
|
227
|
+
parentPort.postMessage({
|
|
228
|
+
taskId,
|
|
229
|
+
success: false,
|
|
230
|
+
error: error.message,
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
});
|
|
234
|
+
`;
|
|
235
|
+
|
|
236
|
+
// ============================================
|
|
237
|
+
// REAL WORKER THREAD POOL
|
|
238
|
+
// ============================================
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Real worker pool using Node.js worker_threads
|
|
242
|
+
*/
|
|
243
|
+
export class RealWorkerPool extends EventEmitter {
|
|
244
|
+
constructor(options = {}) {
|
|
245
|
+
super();
|
|
246
|
+
this.id = `pool-${randomBytes(6).toString('hex')}`;
|
|
247
|
+
this.size = options.size || Math.max(2, cpus().length - 1);
|
|
248
|
+
this.maxQueueSize = options.maxQueueSize || 1000;
|
|
249
|
+
|
|
250
|
+
this.workers = [];
|
|
251
|
+
this.taskQueue = [];
|
|
252
|
+
this.activeTasks = new Map();
|
|
253
|
+
this.taskIdCounter = 0;
|
|
254
|
+
|
|
255
|
+
this.status = 'created';
|
|
256
|
+
this.stats = {
|
|
257
|
+
tasksCompleted: 0,
|
|
258
|
+
tasksFailed: 0,
|
|
259
|
+
totalProcessingTime: 0,
|
|
260
|
+
avgProcessingTime: 0,
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Initialize the worker pool
|
|
266
|
+
*/
|
|
267
|
+
async initialize() {
|
|
268
|
+
this.status = 'initializing';
|
|
269
|
+
this.emit('status', 'Initializing worker pool...');
|
|
270
|
+
|
|
271
|
+
for (let i = 0; i < this.size; i++) {
|
|
272
|
+
await this.spawnWorker(i);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
this.status = 'ready';
|
|
276
|
+
this.emit('ready', {
|
|
277
|
+
poolId: this.id,
|
|
278
|
+
workers: this.workers.length,
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
return this;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Spawn a worker thread
|
|
286
|
+
*/
|
|
287
|
+
async spawnWorker(index) {
|
|
288
|
+
return new Promise((resolve, reject) => {
|
|
289
|
+
try {
|
|
290
|
+
const worker = new Worker(WORKER_CODE, { eval: true });
|
|
291
|
+
|
|
292
|
+
const workerInfo = {
|
|
293
|
+
id: `worker-${index}`,
|
|
294
|
+
worker,
|
|
295
|
+
status: 'idle',
|
|
296
|
+
tasksCompleted: 0,
|
|
297
|
+
currentTask: null,
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
worker.on('message', (msg) => {
|
|
301
|
+
this.handleWorkerMessage(workerInfo, msg);
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
worker.on('error', (err) => {
|
|
305
|
+
console.error(`[Worker ${index}] Error:`, err.message);
|
|
306
|
+
this.handleWorkerError(workerInfo, err);
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
worker.on('exit', (code) => {
|
|
310
|
+
if (code !== 0) {
|
|
311
|
+
console.error(`[Worker ${index}] Exited with code ${code}`);
|
|
312
|
+
// Respawn worker
|
|
313
|
+
const idx = this.workers.indexOf(workerInfo);
|
|
314
|
+
if (idx >= 0 && this.status === 'ready') {
|
|
315
|
+
this.spawnWorker(index).then(w => {
|
|
316
|
+
this.workers[idx] = w;
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
this.workers.push(workerInfo);
|
|
323
|
+
resolve(workerInfo);
|
|
324
|
+
} catch (error) {
|
|
325
|
+
reject(error);
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* Handle message from worker
|
|
332
|
+
*/
|
|
333
|
+
handleWorkerMessage(workerInfo, msg) {
|
|
334
|
+
const { taskId, success, result, error } = msg;
|
|
335
|
+
const taskInfo = this.activeTasks.get(taskId);
|
|
336
|
+
|
|
337
|
+
if (!taskInfo) return;
|
|
338
|
+
|
|
339
|
+
const duration = Date.now() - taskInfo.startTime;
|
|
340
|
+
this.stats.totalProcessingTime += duration;
|
|
341
|
+
|
|
342
|
+
if (success) {
|
|
343
|
+
this.stats.tasksCompleted++;
|
|
344
|
+
workerInfo.tasksCompleted++;
|
|
345
|
+
taskInfo.resolve(result);
|
|
346
|
+
} else {
|
|
347
|
+
this.stats.tasksFailed++;
|
|
348
|
+
taskInfo.reject(new Error(error));
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
this.stats.avgProcessingTime = this.stats.totalProcessingTime /
|
|
352
|
+
(this.stats.tasksCompleted + this.stats.tasksFailed);
|
|
353
|
+
|
|
354
|
+
this.activeTasks.delete(taskId);
|
|
355
|
+
workerInfo.status = 'idle';
|
|
356
|
+
workerInfo.currentTask = null;
|
|
357
|
+
|
|
358
|
+
this.emit('task-complete', { taskId, success, duration });
|
|
359
|
+
|
|
360
|
+
// Process next queued task
|
|
361
|
+
this.processQueue();
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* Handle worker error
|
|
366
|
+
*/
|
|
367
|
+
handleWorkerError(workerInfo, error) {
|
|
368
|
+
if (workerInfo.currentTask) {
|
|
369
|
+
const taskInfo = this.activeTasks.get(workerInfo.currentTask);
|
|
370
|
+
if (taskInfo) {
|
|
371
|
+
taskInfo.reject(error);
|
|
372
|
+
this.activeTasks.delete(workerInfo.currentTask);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
workerInfo.status = 'error';
|
|
376
|
+
this.emit('worker-error', { workerId: workerInfo.id, error: error.message });
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Execute a single task
|
|
381
|
+
*/
|
|
382
|
+
async execute(type, data, options = {}) {
|
|
383
|
+
if (this.status !== 'ready') {
|
|
384
|
+
throw new Error('Worker pool not ready');
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
const taskId = `task-${++this.taskIdCounter}`;
|
|
388
|
+
|
|
389
|
+
return new Promise((resolve, reject) => {
|
|
390
|
+
const taskInfo = {
|
|
391
|
+
taskId,
|
|
392
|
+
type,
|
|
393
|
+
data,
|
|
394
|
+
options,
|
|
395
|
+
resolve,
|
|
396
|
+
reject,
|
|
397
|
+
startTime: null,
|
|
398
|
+
queuedAt: Date.now(),
|
|
399
|
+
};
|
|
400
|
+
|
|
401
|
+
// Find idle worker
|
|
402
|
+
const worker = this.findIdleWorker();
|
|
403
|
+
|
|
404
|
+
if (worker) {
|
|
405
|
+
this.dispatchTask(worker, taskInfo);
|
|
406
|
+
} else {
|
|
407
|
+
if (this.taskQueue.length >= this.maxQueueSize) {
|
|
408
|
+
reject(new Error('Task queue full'));
|
|
409
|
+
return;
|
|
410
|
+
}
|
|
411
|
+
this.taskQueue.push(taskInfo);
|
|
412
|
+
this.emit('task-queued', { taskId, queueLength: this.taskQueue.length });
|
|
413
|
+
}
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Execute batch of tasks in parallel
|
|
419
|
+
*/
|
|
420
|
+
async executeBatch(type, dataArray, options = {}) {
|
|
421
|
+
if (!Array.isArray(dataArray)) {
|
|
422
|
+
return this.execute(type, dataArray, options);
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
const batchId = `batch-${randomBytes(4).toString('hex')}`;
|
|
426
|
+
const startTime = Date.now();
|
|
427
|
+
|
|
428
|
+
this.emit('batch-start', { batchId, count: dataArray.length });
|
|
429
|
+
|
|
430
|
+
const promises = dataArray.map(data =>
|
|
431
|
+
this.execute(type, data, options)
|
|
432
|
+
);
|
|
433
|
+
|
|
434
|
+
const results = await Promise.allSettled(promises);
|
|
435
|
+
|
|
436
|
+
const duration = Date.now() - startTime;
|
|
437
|
+
const succeeded = results.filter(r => r.status === 'fulfilled').length;
|
|
438
|
+
const failed = results.filter(r => r.status === 'rejected').length;
|
|
439
|
+
|
|
440
|
+
this.emit('batch-complete', { batchId, duration, succeeded, failed });
|
|
441
|
+
|
|
442
|
+
return results.map(r =>
|
|
443
|
+
r.status === 'fulfilled' ? r.value : { error: r.reason.message }
|
|
444
|
+
);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* Map operation across data array
|
|
449
|
+
*/
|
|
450
|
+
async map(type, dataArray, options = {}) {
|
|
451
|
+
return this.executeBatch(type, dataArray, options);
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Reduce operation with aggregation
|
|
456
|
+
*/
|
|
457
|
+
async reduce(type, dataArray, options = {}) {
|
|
458
|
+
const mapped = await this.executeBatch(type, dataArray, options);
|
|
459
|
+
return this.execute('aggregate', mapped, options);
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/**
|
|
463
|
+
* Find an idle worker
|
|
464
|
+
*/
|
|
465
|
+
findIdleWorker() {
|
|
466
|
+
return this.workers.find(w => w.status === 'idle');
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* Dispatch task to worker
|
|
471
|
+
*/
|
|
472
|
+
dispatchTask(workerInfo, taskInfo) {
|
|
473
|
+
workerInfo.status = 'busy';
|
|
474
|
+
workerInfo.currentTask = taskInfo.taskId;
|
|
475
|
+
taskInfo.startTime = Date.now();
|
|
476
|
+
|
|
477
|
+
this.activeTasks.set(taskInfo.taskId, taskInfo);
|
|
478
|
+
|
|
479
|
+
workerInfo.worker.postMessage({
|
|
480
|
+
taskId: taskInfo.taskId,
|
|
481
|
+
type: taskInfo.type,
|
|
482
|
+
data: taskInfo.data,
|
|
483
|
+
options: taskInfo.options,
|
|
484
|
+
});
|
|
485
|
+
|
|
486
|
+
this.emit('task-dispatched', {
|
|
487
|
+
taskId: taskInfo.taskId,
|
|
488
|
+
workerId: workerInfo.id,
|
|
489
|
+
queueTime: taskInfo.startTime - taskInfo.queuedAt,
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/**
|
|
494
|
+
* Process queued tasks
|
|
495
|
+
*/
|
|
496
|
+
processQueue() {
|
|
497
|
+
while (this.taskQueue.length > 0) {
|
|
498
|
+
const worker = this.findIdleWorker();
|
|
499
|
+
if (!worker) break;
|
|
500
|
+
|
|
501
|
+
const taskInfo = this.taskQueue.shift();
|
|
502
|
+
this.dispatchTask(worker, taskInfo);
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* Get pool status
|
|
508
|
+
*/
|
|
509
|
+
getStatus() {
|
|
510
|
+
return {
|
|
511
|
+
poolId: this.id,
|
|
512
|
+
status: this.status,
|
|
513
|
+
workers: {
|
|
514
|
+
total: this.workers.length,
|
|
515
|
+
idle: this.workers.filter(w => w.status === 'idle').length,
|
|
516
|
+
busy: this.workers.filter(w => w.status === 'busy').length,
|
|
517
|
+
},
|
|
518
|
+
queue: {
|
|
519
|
+
size: this.taskQueue.length,
|
|
520
|
+
maxSize: this.maxQueueSize,
|
|
521
|
+
},
|
|
522
|
+
activeTasks: this.activeTasks.size,
|
|
523
|
+
stats: this.stats,
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* Shutdown the pool
|
|
529
|
+
*/
|
|
530
|
+
async shutdown() {
|
|
531
|
+
this.status = 'shutting_down';
|
|
532
|
+
this.emit('shutdown-start');
|
|
533
|
+
|
|
534
|
+
// Wait for active tasks with timeout
|
|
535
|
+
const timeout = Date.now() + 10000;
|
|
536
|
+
while (this.activeTasks.size > 0 && Date.now() < timeout) {
|
|
537
|
+
await new Promise(r => setTimeout(r, 100));
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
// Terminate workers
|
|
541
|
+
for (const workerInfo of this.workers) {
|
|
542
|
+
await workerInfo.worker.terminate();
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
this.workers = [];
|
|
546
|
+
this.taskQueue = [];
|
|
547
|
+
this.activeTasks.clear();
|
|
548
|
+
this.status = 'shutdown';
|
|
549
|
+
|
|
550
|
+
this.emit('shutdown-complete');
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// Alias for shutdown
|
|
554
|
+
async close() {
|
|
555
|
+
return this.shutdown();
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// ============================================
|
|
560
|
+
// DISTRIBUTED TASK CLIENT
|
|
561
|
+
// ============================================
|
|
562
|
+
|
|
563
|
+
/**
|
|
564
|
+
* Client for distributed task execution via relay
|
|
565
|
+
*/
|
|
566
|
+
export class DistributedTaskClient extends EventEmitter {
|
|
567
|
+
constructor(options = {}) {
|
|
568
|
+
super();
|
|
569
|
+
this.relayUrl = options.relayUrl || 'ws://localhost:8080';
|
|
570
|
+
this.nodeId = options.nodeId || `client-${randomBytes(8).toString('hex')}`;
|
|
571
|
+
this.ws = null;
|
|
572
|
+
this.connected = false;
|
|
573
|
+
|
|
574
|
+
this.pendingTasks = new Map();
|
|
575
|
+
this.completedTasks = new Map();
|
|
576
|
+
this.taskIdCounter = 0;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
/**
|
|
580
|
+
* Connect to relay server
|
|
581
|
+
*/
|
|
582
|
+
async connect() {
|
|
583
|
+
return new Promise(async (resolve, reject) => {
|
|
584
|
+
try {
|
|
585
|
+
let WebSocket;
|
|
586
|
+
if (typeof globalThis.WebSocket !== 'undefined') {
|
|
587
|
+
WebSocket = globalThis.WebSocket;
|
|
588
|
+
} else {
|
|
589
|
+
const ws = await import('ws');
|
|
590
|
+
WebSocket = ws.default || ws.WebSocket;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
this.ws = new WebSocket(this.relayUrl);
|
|
594
|
+
|
|
595
|
+
const timeout = setTimeout(() => {
|
|
596
|
+
reject(new Error('Connection timeout'));
|
|
597
|
+
}, 10000);
|
|
598
|
+
|
|
599
|
+
this.ws.onopen = () => {
|
|
600
|
+
clearTimeout(timeout);
|
|
601
|
+
this.connected = true;
|
|
602
|
+
|
|
603
|
+
// Register as task client
|
|
604
|
+
this.send({
|
|
605
|
+
type: 'register',
|
|
606
|
+
nodeId: this.nodeId,
|
|
607
|
+
capabilities: ['task_client'],
|
|
608
|
+
});
|
|
609
|
+
|
|
610
|
+
this.emit('connected');
|
|
611
|
+
resolve(true);
|
|
612
|
+
};
|
|
613
|
+
|
|
614
|
+
this.ws.onmessage = (event) => {
|
|
615
|
+
this.handleMessage(JSON.parse(event.data));
|
|
616
|
+
};
|
|
617
|
+
|
|
618
|
+
this.ws.onclose = () => {
|
|
619
|
+
this.connected = false;
|
|
620
|
+
this.emit('disconnected');
|
|
621
|
+
};
|
|
622
|
+
|
|
623
|
+
this.ws.onerror = (error) => {
|
|
624
|
+
clearTimeout(timeout);
|
|
625
|
+
reject(error);
|
|
626
|
+
};
|
|
627
|
+
|
|
628
|
+
} catch (error) {
|
|
629
|
+
reject(error);
|
|
630
|
+
}
|
|
631
|
+
});
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* Handle incoming message
|
|
636
|
+
*/
|
|
637
|
+
handleMessage(message) {
|
|
638
|
+
switch (message.type) {
|
|
639
|
+
case 'welcome':
|
|
640
|
+
this.emit('registered', message);
|
|
641
|
+
break;
|
|
642
|
+
|
|
643
|
+
case 'task_result':
|
|
644
|
+
this.handleTaskResult(message);
|
|
645
|
+
break;
|
|
646
|
+
|
|
647
|
+
case 'task_progress':
|
|
648
|
+
this.handleTaskProgress(message);
|
|
649
|
+
break;
|
|
650
|
+
|
|
651
|
+
case 'task_accepted':
|
|
652
|
+
this.emit('task-accepted', message);
|
|
653
|
+
break;
|
|
654
|
+
|
|
655
|
+
default:
|
|
656
|
+
this.emit('message', message);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
/**
|
|
661
|
+
* Handle task result
|
|
662
|
+
*/
|
|
663
|
+
handleTaskResult(message) {
|
|
664
|
+
const taskInfo = this.pendingTasks.get(message.taskId);
|
|
665
|
+
if (!taskInfo) return;
|
|
666
|
+
|
|
667
|
+
const duration = Date.now() - taskInfo.startTime;
|
|
668
|
+
|
|
669
|
+
if (message.success) {
|
|
670
|
+
taskInfo.resolve({
|
|
671
|
+
taskId: message.taskId,
|
|
672
|
+
result: message.result,
|
|
673
|
+
processedBy: message.processedBy,
|
|
674
|
+
duration,
|
|
675
|
+
});
|
|
676
|
+
} else {
|
|
677
|
+
taskInfo.reject(new Error(message.error || 'Task failed'));
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
this.pendingTasks.delete(message.taskId);
|
|
681
|
+
this.completedTasks.set(message.taskId, {
|
|
682
|
+
...message,
|
|
683
|
+
duration,
|
|
684
|
+
});
|
|
685
|
+
|
|
686
|
+
this.emit('task-complete', { taskId: message.taskId, duration });
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Handle task progress update
|
|
691
|
+
*/
|
|
692
|
+
handleTaskProgress(message) {
|
|
693
|
+
this.emit('task-progress', message);
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* Send message to relay
|
|
698
|
+
*/
|
|
699
|
+
send(message) {
|
|
700
|
+
if (this.connected && this.ws?.readyState === 1) {
|
|
701
|
+
this.ws.send(JSON.stringify(message));
|
|
702
|
+
return true;
|
|
703
|
+
}
|
|
704
|
+
return false;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
/**
|
|
708
|
+
* Submit task for distributed execution
|
|
709
|
+
*/
|
|
710
|
+
async submitTask(task, options = {}) {
|
|
711
|
+
if (!this.connected) {
|
|
712
|
+
throw new Error('Not connected to relay');
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
const taskId = `dtask-${++this.taskIdCounter}-${Date.now()}`;
|
|
716
|
+
|
|
717
|
+
return new Promise((resolve, reject) => {
|
|
718
|
+
const taskInfo = {
|
|
719
|
+
taskId,
|
|
720
|
+
task,
|
|
721
|
+
options,
|
|
722
|
+
resolve,
|
|
723
|
+
reject,
|
|
724
|
+
startTime: Date.now(),
|
|
725
|
+
};
|
|
726
|
+
|
|
727
|
+
this.pendingTasks.set(taskId, taskInfo);
|
|
728
|
+
|
|
729
|
+
// Set timeout
|
|
730
|
+
const timeout = options.timeout || 60000;
|
|
731
|
+
setTimeout(() => {
|
|
732
|
+
if (this.pendingTasks.has(taskId)) {
|
|
733
|
+
this.pendingTasks.delete(taskId);
|
|
734
|
+
reject(new Error('Task timeout'));
|
|
735
|
+
}
|
|
736
|
+
}, timeout);
|
|
737
|
+
|
|
738
|
+
// Submit to relay
|
|
739
|
+
this.send({
|
|
740
|
+
type: 'task_submit',
|
|
741
|
+
task: {
|
|
742
|
+
id: taskId,
|
|
743
|
+
type: task.type || 'compute',
|
|
744
|
+
data: task.data,
|
|
745
|
+
options: task.options,
|
|
746
|
+
priority: options.priority || 'medium',
|
|
747
|
+
},
|
|
748
|
+
});
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
/**
|
|
753
|
+
* Submit batch of tasks
|
|
754
|
+
*/
|
|
755
|
+
async submitBatch(tasks, options = {}) {
|
|
756
|
+
const promises = tasks.map(task =>
|
|
757
|
+
this.submitTask(task, options)
|
|
758
|
+
);
|
|
759
|
+
return Promise.allSettled(promises);
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/**
|
|
763
|
+
* Close connection
|
|
764
|
+
*/
|
|
765
|
+
close() {
|
|
766
|
+
if (this.ws) {
|
|
767
|
+
this.ws.close();
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
// ============================================
|
|
773
|
+
// DISTRIBUTED TASK WORKER
|
|
774
|
+
// ============================================
|
|
775
|
+
|
|
776
|
+
/**
|
|
777
|
+
* Worker that processes distributed tasks from relay
|
|
778
|
+
*/
|
|
779
|
+
export class DistributedTaskWorker extends EventEmitter {
|
|
780
|
+
constructor(options = {}) {
|
|
781
|
+
super();
|
|
782
|
+
this.relayUrl = options.relayUrl || 'ws://localhost:8080';
|
|
783
|
+
this.nodeId = options.nodeId || `worker-${randomBytes(8).toString('hex')}`;
|
|
784
|
+
this.capabilities = options.capabilities || ['compute', 'embed', 'process'];
|
|
785
|
+
this.ws = null;
|
|
786
|
+
this.connected = false;
|
|
787
|
+
|
|
788
|
+
this.localPool = null;
|
|
789
|
+
this.activeTasks = new Map();
|
|
790
|
+
|
|
791
|
+
this.stats = {
|
|
792
|
+
tasksProcessed: 0,
|
|
793
|
+
tasksFailed: 0,
|
|
794
|
+
creditsEarned: 0,
|
|
795
|
+
};
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/**
|
|
799
|
+
* Initialize worker with local pool
|
|
800
|
+
*/
|
|
801
|
+
async initialize() {
|
|
802
|
+
this.localPool = new RealWorkerPool({ size: 2 });
|
|
803
|
+
await this.localPool.initialize();
|
|
804
|
+
return this;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
/**
|
|
808
|
+
* Connect to relay and start processing
|
|
809
|
+
*/
|
|
810
|
+
async connect() {
|
|
811
|
+
return new Promise(async (resolve, reject) => {
|
|
812
|
+
try {
|
|
813
|
+
let WebSocket;
|
|
814
|
+
if (typeof globalThis.WebSocket !== 'undefined') {
|
|
815
|
+
WebSocket = globalThis.WebSocket;
|
|
816
|
+
} else {
|
|
817
|
+
const ws = await import('ws');
|
|
818
|
+
WebSocket = ws.default || ws.WebSocket;
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
this.ws = new WebSocket(this.relayUrl);
|
|
822
|
+
|
|
823
|
+
const timeout = setTimeout(() => {
|
|
824
|
+
reject(new Error('Connection timeout'));
|
|
825
|
+
}, 10000);
|
|
826
|
+
|
|
827
|
+
this.ws.onopen = () => {
|
|
828
|
+
clearTimeout(timeout);
|
|
829
|
+
this.connected = true;
|
|
830
|
+
|
|
831
|
+
// Register as task worker
|
|
832
|
+
this.send({
|
|
833
|
+
type: 'register',
|
|
834
|
+
nodeId: this.nodeId,
|
|
835
|
+
capabilities: this.capabilities,
|
|
836
|
+
workerType: 'task_processor',
|
|
837
|
+
});
|
|
838
|
+
|
|
839
|
+
this.emit('connected');
|
|
840
|
+
resolve(true);
|
|
841
|
+
};
|
|
842
|
+
|
|
843
|
+
this.ws.onmessage = (event) => {
|
|
844
|
+
this.handleMessage(JSON.parse(event.data));
|
|
845
|
+
};
|
|
846
|
+
|
|
847
|
+
this.ws.onclose = () => {
|
|
848
|
+
this.connected = false;
|
|
849
|
+
this.emit('disconnected');
|
|
850
|
+
};
|
|
851
|
+
|
|
852
|
+
this.ws.onerror = (error) => {
|
|
853
|
+
clearTimeout(timeout);
|
|
854
|
+
reject(error);
|
|
855
|
+
};
|
|
856
|
+
|
|
857
|
+
} catch (error) {
|
|
858
|
+
reject(error);
|
|
859
|
+
}
|
|
860
|
+
});
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
/**
|
|
864
|
+
* Handle incoming message
|
|
865
|
+
*/
|
|
866
|
+
handleMessage(message) {
|
|
867
|
+
switch (message.type) {
|
|
868
|
+
case 'welcome':
|
|
869
|
+
console.log(`[Worker] Registered: ${this.nodeId}`);
|
|
870
|
+
this.emit('registered', message);
|
|
871
|
+
break;
|
|
872
|
+
|
|
873
|
+
case 'task_assignment':
|
|
874
|
+
this.processTask(message.task);
|
|
875
|
+
break;
|
|
876
|
+
|
|
877
|
+
default:
|
|
878
|
+
this.emit('message', message);
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
/**
|
|
883
|
+
* Process assigned task
|
|
884
|
+
*/
|
|
885
|
+
async processTask(task) {
|
|
886
|
+
const startTime = Date.now();
|
|
887
|
+
this.activeTasks.set(task.id, task);
|
|
888
|
+
|
|
889
|
+
console.log(`[Worker] Processing task: ${task.id}`);
|
|
890
|
+
this.emit('task-start', { taskId: task.id });
|
|
891
|
+
|
|
892
|
+
try {
|
|
893
|
+
// Execute using local pool
|
|
894
|
+
const result = await this.localPool.execute(
|
|
895
|
+
task.type || 'process',
|
|
896
|
+
task.data,
|
|
897
|
+
task.options
|
|
898
|
+
);
|
|
899
|
+
|
|
900
|
+
const duration = Date.now() - startTime;
|
|
901
|
+
this.stats.tasksProcessed++;
|
|
902
|
+
|
|
903
|
+
// Report result
|
|
904
|
+
this.send({
|
|
905
|
+
type: 'task_complete',
|
|
906
|
+
taskId: task.id,
|
|
907
|
+
submitterId: task.submitter,
|
|
908
|
+
result,
|
|
909
|
+
reward: Math.ceil(1000000 * (duration / 1000)), // 0.001 rUv per second
|
|
910
|
+
success: true,
|
|
911
|
+
});
|
|
912
|
+
|
|
913
|
+
this.emit('task-complete', { taskId: task.id, duration, result });
|
|
914
|
+
|
|
915
|
+
} catch (error) {
|
|
916
|
+
this.stats.tasksFailed++;
|
|
917
|
+
|
|
918
|
+
this.send({
|
|
919
|
+
type: 'task_complete',
|
|
920
|
+
taskId: task.id,
|
|
921
|
+
submitterId: task.submitter,
|
|
922
|
+
error: error.message,
|
|
923
|
+
success: false,
|
|
924
|
+
});
|
|
925
|
+
|
|
926
|
+
this.emit('task-error', { taskId: task.id, error: error.message });
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
this.activeTasks.delete(task.id);
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
/**
|
|
933
|
+
* Send message to relay
|
|
934
|
+
*/
|
|
935
|
+
send(message) {
|
|
936
|
+
if (this.connected && this.ws?.readyState === 1) {
|
|
937
|
+
this.ws.send(JSON.stringify(message));
|
|
938
|
+
return true;
|
|
939
|
+
}
|
|
940
|
+
return false;
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/**
|
|
944
|
+
* Get worker stats
|
|
945
|
+
*/
|
|
946
|
+
getStats() {
|
|
947
|
+
return {
|
|
948
|
+
nodeId: this.nodeId,
|
|
949
|
+
connected: this.connected,
|
|
950
|
+
activeTasks: this.activeTasks.size,
|
|
951
|
+
...this.stats,
|
|
952
|
+
poolStatus: this.localPool?.getStatus(),
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
/**
|
|
957
|
+
* Stop worker
|
|
958
|
+
*/
|
|
959
|
+
async stop() {
|
|
960
|
+
if (this.localPool) {
|
|
961
|
+
await this.localPool.shutdown();
|
|
962
|
+
}
|
|
963
|
+
if (this.ws) {
|
|
964
|
+
this.ws.close();
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
// Default export
|
|
970
|
+
export default RealWorkerPool;
|