digitaltwin-core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +125 -0
- package/dist/components/assets_manager.d.ts +384 -0
- package/dist/components/assets_manager.d.ts.map +1 -0
- package/dist/components/assets_manager.js +637 -0
- package/dist/components/assets_manager.js.map +1 -0
- package/dist/components/collector.d.ts +62 -0
- package/dist/components/collector.d.ts.map +1 -0
- package/dist/components/collector.js +80 -0
- package/dist/components/collector.js.map +1 -0
- package/dist/components/global_assets_handler.d.ts +63 -0
- package/dist/components/global_assets_handler.d.ts.map +1 -0
- package/dist/components/global_assets_handler.js +127 -0
- package/dist/components/global_assets_handler.js.map +1 -0
- package/dist/components/handler.d.ts +33 -0
- package/dist/components/handler.d.ts.map +1 -0
- package/dist/components/handler.js +27 -0
- package/dist/components/handler.js.map +1 -0
- package/dist/components/harvester.d.ts +54 -0
- package/dist/components/harvester.d.ts.map +1 -0
- package/dist/components/harvester.js +253 -0
- package/dist/components/harvester.js.map +1 -0
- package/dist/components/index.d.ts +8 -0
- package/dist/components/index.d.ts.map +1 -0
- package/dist/components/index.js +6 -0
- package/dist/components/index.js.map +1 -0
- package/dist/components/interfaces.d.ts +18 -0
- package/dist/components/interfaces.d.ts.map +1 -0
- package/dist/components/interfaces.js +2 -0
- package/dist/components/interfaces.js.map +1 -0
- package/dist/components/types.d.ts +47 -0
- package/dist/components/types.d.ts.map +1 -0
- package/dist/components/types.js +2 -0
- package/dist/components/types.js.map +1 -0
- package/dist/database/adapters/knex_database_adapter.d.ts +47 -0
- package/dist/database/adapters/knex_database_adapter.d.ts.map +1 -0
- package/dist/database/adapters/knex_database_adapter.js +180 -0
- package/dist/database/adapters/knex_database_adapter.js.map +1 -0
- package/dist/database/database_adapter.d.ts +92 -0
- package/dist/database/database_adapter.d.ts.map +1 -0
- package/dist/database/database_adapter.js +6 -0
- package/dist/database/database_adapter.js.map +1 -0
- package/dist/engine/digital_twin_engine.d.ts +237 -0
- package/dist/engine/digital_twin_engine.d.ts.map +1 -0
- package/dist/engine/digital_twin_engine.js +574 -0
- package/dist/engine/digital_twin_engine.js.map +1 -0
- package/dist/engine/endpoints.d.ts +7 -0
- package/dist/engine/endpoints.d.ts.map +1 -0
- package/dist/engine/endpoints.js +18 -0
- package/dist/engine/endpoints.js.map +1 -0
- package/dist/engine/events.d.ts +13 -0
- package/dist/engine/events.d.ts.map +1 -0
- package/dist/engine/events.js +11 -0
- package/dist/engine/events.js.map +1 -0
- package/dist/engine/initializer.d.ts +8 -0
- package/dist/engine/initializer.d.ts.map +1 -0
- package/dist/engine/initializer.js +23 -0
- package/dist/engine/initializer.js.map +1 -0
- package/dist/engine/queue_manager.d.ts +78 -0
- package/dist/engine/queue_manager.d.ts.map +1 -0
- package/dist/engine/queue_manager.js +167 -0
- package/dist/engine/queue_manager.js.map +1 -0
- package/dist/engine/scheduler.d.ts +30 -0
- package/dist/engine/scheduler.d.ts.map +1 -0
- package/dist/engine/scheduler.js +378 -0
- package/dist/engine/scheduler.js.map +1 -0
- package/dist/env/env.d.ts +25 -0
- package/dist/env/env.d.ts.map +1 -0
- package/dist/env/env.js +55 -0
- package/dist/env/env.js.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +31 -0
- package/dist/index.js.map +1 -0
- package/dist/storage/adapters/local_storage_service.d.ts +29 -0
- package/dist/storage/adapters/local_storage_service.d.ts.map +1 -0
- package/dist/storage/adapters/local_storage_service.js +50 -0
- package/dist/storage/adapters/local_storage_service.js.map +1 -0
- package/dist/storage/adapters/ovh_storage_service.d.ts +32 -0
- package/dist/storage/adapters/ovh_storage_service.d.ts.map +1 -0
- package/dist/storage/adapters/ovh_storage_service.js +70 -0
- package/dist/storage/adapters/ovh_storage_service.js.map +1 -0
- package/dist/storage/storage_factory.d.ts +14 -0
- package/dist/storage/storage_factory.d.ts.map +1 -0
- package/dist/storage/storage_factory.js +36 -0
- package/dist/storage/storage_factory.js.map +1 -0
- package/dist/storage/storage_service.d.ts +26 -0
- package/dist/storage/storage_service.d.ts.map +1 -0
- package/dist/storage/storage_service.js +7 -0
- package/dist/storage/storage_service.js.map +1 -0
- package/dist/types/data_record.d.ts +17 -0
- package/dist/types/data_record.d.ts.map +1 -0
- package/dist/types/data_record.js +2 -0
- package/dist/types/data_record.js.map +1 -0
- package/dist/utils/logger.d.ts +17 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +35 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/map_to_data_record.d.ts +10 -0
- package/dist/utils/map_to_data_record.d.ts.map +1 -0
- package/dist/utils/map_to_data_record.js +21 -0
- package/dist/utils/map_to_data_record.js.map +1 -0
- package/dist/utils/servable_endpoint.d.ts +6 -0
- package/dist/utils/servable_endpoint.d.ts.map +1 -0
- package/dist/utils/servable_endpoint.js +15 -0
- package/dist/utils/servable_endpoint.js.map +1 -0
- package/package.json +95 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Collector } from '../components/collector.js';
|
|
2
|
+
import { Harvester } from '../components/harvester.js';
|
|
3
|
+
import { AssetsManager } from '../components/assets_manager.js';
|
|
4
|
+
import { DatabaseAdapter } from '../database/database_adapter.js';
|
|
5
|
+
import { StorageService } from '../storage/storage_service.js';
|
|
6
|
+
export declare function initializeComponents(components: Array<Collector | Harvester>, database: DatabaseAdapter, storage: StorageService): Promise<void>;
|
|
7
|
+
export declare function initializeAssetsManagers(assetsManagers: AssetsManager[], database: DatabaseAdapter, storage: StorageService): Promise<void>;
|
|
8
|
+
//# sourceMappingURL=initializer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"initializer.d.ts","sourceRoot":"","sources":["../../src/engine/initializer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAA;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAA;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAA;AAE9D,wBAAsB,oBAAoB,CACtC,UAAU,EAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,EACxC,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,cAAc,GACxB,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAsB,wBAAwB,CAC1C,cAAc,EAAE,aAAa,EAAE,EAC/B,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,cAAc,GACxB,OAAO,CAAC,IAAI,CAAC,CAMf"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export async function initializeComponents(components, database, storage) {
|
|
2
|
+
for (const comp of components) {
|
|
3
|
+
const config = comp.getConfiguration();
|
|
4
|
+
await ensureTableExists(database, config.name);
|
|
5
|
+
comp.setDependencies(database, storage);
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
export async function initializeAssetsManagers(assetsManagers, database, storage) {
|
|
9
|
+
for (const manager of assetsManagers) {
|
|
10
|
+
const config = manager.getConfiguration();
|
|
11
|
+
await ensureTableExists(database, config.name);
|
|
12
|
+
manager.setDependencies(database, storage);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
async function ensureTableExists(database, tableName) {
|
|
16
|
+
const exists = await database.doesTableExists(tableName);
|
|
17
|
+
if (!exists) {
|
|
18
|
+
console.log(`Creating table for component "${tableName}"...`);
|
|
19
|
+
await database.createTable(tableName);
|
|
20
|
+
console.log(`✅ Table "${tableName}" created successfully`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=initializer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"initializer.js","sourceRoot":"","sources":["../../src/engine/initializer.ts"],"names":[],"mappings":"AAMA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,UAAwC,EACxC,QAAyB,EACzB,OAAuB;IAEvB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAA;QACtC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC3C,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,cAA+B,EAC/B,QAAyB,EACzB,OAAuB;IAEvB,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;QACzC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;QAC9C,OAAO,CAAC,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC9C,CAAC;AACL,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAyB,EAAE,SAAiB;IACzE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;IACxD,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,iCAAiC,SAAS,MAAM,CAAC,CAAA;QAC7D,MAAM,QAAQ,CAAC,WAAW,CAAC,SAAS,CAAC,CAAA;QACrC,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,wBAAwB,CAAC,CAAA;IAC9D,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { Queue, QueueOptions, ConnectionOptions } from 'bullmq';
|
|
2
|
+
/**
|
|
3
|
+
* Configuration options for the Queue Manager
|
|
4
|
+
*/
|
|
5
|
+
export interface QueueConfig {
|
|
6
|
+
/** Number of workers for collectors (data collection) */
|
|
7
|
+
collectorWorkers?: number;
|
|
8
|
+
/** Number of workers for harvesters (data processing) */
|
|
9
|
+
harvesterWorkers?: number;
|
|
10
|
+
/** Redis connection configuration */
|
|
11
|
+
redis?: ConnectionOptions;
|
|
12
|
+
/** Advanced options for each queue type */
|
|
13
|
+
queueOptions?: {
|
|
14
|
+
collectors?: Partial<QueueOptions>;
|
|
15
|
+
harvesters?: Partial<QueueOptions>;
|
|
16
|
+
priority?: Partial<QueueOptions>;
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Queue Manager - Manages BullMQ queues for different component types
|
|
21
|
+
*
|
|
22
|
+
* Handles three types of queues:
|
|
23
|
+
* - Collector queue: High priority, fast execution for data collection
|
|
24
|
+
* - Harvester queue: Medium priority, slower processing for data transformation
|
|
25
|
+
* - Priority queue: Urgent/manual jobs with highest priority
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```typescript
|
|
29
|
+
* const queueManager = new QueueManager({
|
|
30
|
+
* redis: { host: 'localhost', port: 6379 },
|
|
31
|
+
* collectorWorkers: 3,
|
|
32
|
+
* harvesterWorkers: 2
|
|
33
|
+
* })
|
|
34
|
+
*
|
|
35
|
+
* await queueManager.close()
|
|
36
|
+
* ```
|
|
37
|
+
*/
|
|
38
|
+
export declare class QueueManager {
|
|
39
|
+
#private;
|
|
40
|
+
readonly collectorQueue: Queue;
|
|
41
|
+
readonly harvesterQueue: Queue;
|
|
42
|
+
readonly priorityQueue: Queue;
|
|
43
|
+
/**
|
|
44
|
+
* Creates a new Queue Manager instance
|
|
45
|
+
* @param config - Queue configuration options
|
|
46
|
+
*/
|
|
47
|
+
constructor(config?: QueueConfig);
|
|
48
|
+
/**
|
|
49
|
+
* Closes all queues gracefully
|
|
50
|
+
* @returns Promise that resolves when all queues are closed
|
|
51
|
+
*/
|
|
52
|
+
close(): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Gets statistics for all queues
|
|
55
|
+
* @returns Object containing stats for each queue type
|
|
56
|
+
*/
|
|
57
|
+
getQueueStats(): Promise<{
|
|
58
|
+
collectors: {
|
|
59
|
+
waiting: number;
|
|
60
|
+
active: number;
|
|
61
|
+
completed: number;
|
|
62
|
+
failed: number;
|
|
63
|
+
};
|
|
64
|
+
harvesters: {
|
|
65
|
+
waiting: number;
|
|
66
|
+
active: number;
|
|
67
|
+
completed: number;
|
|
68
|
+
failed: number;
|
|
69
|
+
};
|
|
70
|
+
priority: {
|
|
71
|
+
waiting: number;
|
|
72
|
+
active: number;
|
|
73
|
+
completed: number;
|
|
74
|
+
failed: number;
|
|
75
|
+
};
|
|
76
|
+
}>;
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=queue_manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue_manager.d.ts","sourceRoot":"","sources":["../../src/engine/queue_manager.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAA;AAE/D;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB,yDAAyD;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,yDAAyD;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,qCAAqC;IACrC,KAAK,CAAC,EAAE,iBAAiB,CAAA;IACzB,2CAA2C;IAC3C,YAAY,CAAC,EAAE;QACX,UAAU,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;QAClC,UAAU,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;QAClC,QAAQ,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;KACnC,CAAA;CACJ;AAkCD;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,YAAY;;IACrB,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAA;IAC9B,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAA;IAC9B,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAA;IAE7B;;;OAGG;gBACS,MAAM,GAAE,WAAgB;IA2DpC;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAuB5B;;;OAGG;IACG,aAAa;;;;;;;;;;;;;;;;;;;;CA4BtB"}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
// src/engine/queue_manager.ts
|
|
2
|
+
import { Queue } from 'bullmq';
|
|
3
|
+
/**
|
|
4
|
+
* Queue configuration constants
|
|
5
|
+
*/
|
|
6
|
+
const QUEUE_DEFAULTS = {
|
|
7
|
+
REDIS: {
|
|
8
|
+
host: 'localhost',
|
|
9
|
+
port: 6379,
|
|
10
|
+
maxRetriesPerRequest: null,
|
|
11
|
+
enableReadyCheck: true,
|
|
12
|
+
retryStrategy: (times) => Math.min(times * 50, 2000)
|
|
13
|
+
},
|
|
14
|
+
COLLECTORS: {
|
|
15
|
+
name: 'dt-collectors',
|
|
16
|
+
attempts: 3,
|
|
17
|
+
backoffDelay: 2000,
|
|
18
|
+
keepCompleted: 100,
|
|
19
|
+
keepFailed: 50
|
|
20
|
+
},
|
|
21
|
+
HARVESTERS: {
|
|
22
|
+
name: 'dt-harvesters',
|
|
23
|
+
attempts: 5,
|
|
24
|
+
backoffDelay: 5000,
|
|
25
|
+
keepCompleted: 50,
|
|
26
|
+
keepFailed: 100
|
|
27
|
+
},
|
|
28
|
+
PRIORITY: {
|
|
29
|
+
name: 'dt-priority',
|
|
30
|
+
attempts: 2,
|
|
31
|
+
priority: 1
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Queue Manager - Manages BullMQ queues for different component types
|
|
36
|
+
*
|
|
37
|
+
* Handles three types of queues:
|
|
38
|
+
* - Collector queue: High priority, fast execution for data collection
|
|
39
|
+
* - Harvester queue: Medium priority, slower processing for data transformation
|
|
40
|
+
* - Priority queue: Urgent/manual jobs with highest priority
|
|
41
|
+
*
|
|
42
|
+
* @example
|
|
43
|
+
* ```typescript
|
|
44
|
+
* const queueManager = new QueueManager({
|
|
45
|
+
* redis: { host: 'localhost', port: 6379 },
|
|
46
|
+
* collectorWorkers: 3,
|
|
47
|
+
* harvesterWorkers: 2
|
|
48
|
+
* })
|
|
49
|
+
*
|
|
50
|
+
* await queueManager.close()
|
|
51
|
+
* ```
|
|
52
|
+
*/
|
|
53
|
+
export class QueueManager {
|
|
54
|
+
/**
|
|
55
|
+
* Creates a new Queue Manager instance
|
|
56
|
+
* @param config - Queue configuration options
|
|
57
|
+
*/
|
|
58
|
+
constructor(config = {}) {
|
|
59
|
+
const baseConnection = config.redis || QUEUE_DEFAULTS.REDIS;
|
|
60
|
+
this.collectorQueue = this.#createCollectorQueue(baseConnection, config.queueOptions?.collectors);
|
|
61
|
+
this.harvesterQueue = this.#createHarvesterQueue(baseConnection, config.queueOptions?.harvesters);
|
|
62
|
+
this.priorityQueue = this.#createPriorityQueue(baseConnection, config.queueOptions?.priority);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Creates collector queue with optimized settings for data collection
|
|
66
|
+
* @private
|
|
67
|
+
*/
|
|
68
|
+
#createCollectorQueue(connection, options) {
|
|
69
|
+
return new Queue(QUEUE_DEFAULTS.COLLECTORS.name, {
|
|
70
|
+
connection,
|
|
71
|
+
defaultJobOptions: {
|
|
72
|
+
attempts: QUEUE_DEFAULTS.COLLECTORS.attempts,
|
|
73
|
+
backoff: { type: 'exponential', delay: QUEUE_DEFAULTS.COLLECTORS.backoffDelay },
|
|
74
|
+
removeOnComplete: { count: QUEUE_DEFAULTS.COLLECTORS.keepCompleted },
|
|
75
|
+
removeOnFail: { count: QUEUE_DEFAULTS.COLLECTORS.keepFailed }
|
|
76
|
+
},
|
|
77
|
+
...options
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Creates harvester queue with settings optimized for data processing
|
|
82
|
+
* @private
|
|
83
|
+
*/
|
|
84
|
+
#createHarvesterQueue(connection, options) {
|
|
85
|
+
return new Queue(QUEUE_DEFAULTS.HARVESTERS.name, {
|
|
86
|
+
connection,
|
|
87
|
+
defaultJobOptions: {
|
|
88
|
+
attempts: QUEUE_DEFAULTS.HARVESTERS.attempts,
|
|
89
|
+
backoff: { type: 'exponential', delay: QUEUE_DEFAULTS.HARVESTERS.backoffDelay },
|
|
90
|
+
removeOnComplete: { count: QUEUE_DEFAULTS.HARVESTERS.keepCompleted },
|
|
91
|
+
removeOnFail: { count: QUEUE_DEFAULTS.HARVESTERS.keepFailed }
|
|
92
|
+
},
|
|
93
|
+
...options
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Creates priority queue for urgent/manual jobs
|
|
98
|
+
* @private
|
|
99
|
+
*/
|
|
100
|
+
#createPriorityQueue(connection, options) {
|
|
101
|
+
return new Queue(QUEUE_DEFAULTS.PRIORITY.name, {
|
|
102
|
+
connection,
|
|
103
|
+
defaultJobOptions: {
|
|
104
|
+
priority: QUEUE_DEFAULTS.PRIORITY.priority,
|
|
105
|
+
attempts: QUEUE_DEFAULTS.PRIORITY.attempts,
|
|
106
|
+
removeOnComplete: true,
|
|
107
|
+
removeOnFail: false
|
|
108
|
+
},
|
|
109
|
+
...options
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Closes all queues gracefully
|
|
114
|
+
* @returns Promise that resolves when all queues are closed
|
|
115
|
+
*/
|
|
116
|
+
async close() {
|
|
117
|
+
// Close all queues and their Redis connections
|
|
118
|
+
await Promise.all([
|
|
119
|
+
this.collectorQueue.close(),
|
|
120
|
+
this.harvesterQueue.close(),
|
|
121
|
+
this.priorityQueue.close()
|
|
122
|
+
]);
|
|
123
|
+
// Force close any lingering Redis connections
|
|
124
|
+
try {
|
|
125
|
+
// Get the Redis client from the queue and close it
|
|
126
|
+
const redisClient = this.collectorQueue.redisConnection;
|
|
127
|
+
if (redisClient && typeof redisClient.disconnect === 'function') {
|
|
128
|
+
await redisClient.disconnect();
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
catch (error) {
|
|
132
|
+
// Ignore errors during forced cleanup
|
|
133
|
+
}
|
|
134
|
+
// Wait for Redis connections to fully close
|
|
135
|
+
await new Promise(resolve => setTimeout(resolve, 200));
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Gets statistics for all queues
|
|
139
|
+
* @returns Object containing stats for each queue type
|
|
140
|
+
*/
|
|
141
|
+
async getQueueStats() {
|
|
142
|
+
const [collectorStats, harvesterStats, priorityStats] = await Promise.all([
|
|
143
|
+
this.#getStats(this.collectorQueue),
|
|
144
|
+
this.#getStats(this.harvesterQueue),
|
|
145
|
+
this.#getStats(this.priorityQueue)
|
|
146
|
+
]);
|
|
147
|
+
return {
|
|
148
|
+
collectors: collectorStats,
|
|
149
|
+
harvesters: harvesterStats,
|
|
150
|
+
priority: priorityStats
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Gets statistics for a specific queue
|
|
155
|
+
* @private
|
|
156
|
+
*/
|
|
157
|
+
async #getStats(queue) {
|
|
158
|
+
const [waiting, active, completed, failed] = await Promise.all([
|
|
159
|
+
queue.getWaitingCount(),
|
|
160
|
+
queue.getActiveCount(),
|
|
161
|
+
queue.getCompletedCount(),
|
|
162
|
+
queue.getFailedCount()
|
|
163
|
+
]);
|
|
164
|
+
return { waiting, active, completed, failed };
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=queue_manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue_manager.js","sourceRoot":"","sources":["../../src/engine/queue_manager.ts"],"names":[],"mappings":"AAAA,8BAA8B;AAC9B,OAAO,EAAE,KAAK,EAAmC,MAAM,QAAQ,CAAA;AAoB/D;;GAEG;AACH,MAAM,cAAc,GAAG;IACnB,KAAK,EAAE;QACH,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,IAAI;QACV,oBAAoB,EAAE,IAAI;QAC1B,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,EAAE,IAAI,CAAC;KAC/D;IACD,UAAU,EAAE;QACR,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,CAAC;QACX,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,GAAG;QAClB,UAAU,EAAE,EAAE;KACjB;IACD,UAAU,EAAE;QACR,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,CAAC;QACX,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,EAAE;QACjB,UAAU,EAAE,GAAG;KAClB;IACD,QAAQ,EAAE;QACN,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;KACd;CACK,CAAA;AAEV;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,YAAY;IAKrB;;;OAGG;IACH,YAAY,SAAsB,EAAE;QAChC,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,IAAI,cAAc,CAAC,KAAK,CAAA;QAE3D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;QACjG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;QACjG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IACjG,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,UAA6B,EAAE,OAA+B;QAChF,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,EAAE;YAC7C,UAAU;YACV,iBAAiB,EAAE;gBACf,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,QAAQ;gBAC5C,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,YAAY,EAAE;gBAC/E,gBAAgB,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,aAAa,EAAE;gBACpE,YAAY,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,UAAU,EAAE;aAChE;YACD,GAAG,OAAO;SACb,CAAC,CAAA;IACN,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,UAA6B,EAAE,OAA+B;QAChF,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,EAAE;YAC7C,UAAU;YACV,iBAAiB,EAAE;gBACf,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,QAAQ;gBAC5C,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,YAAY,EAAE;gBAC/E,gBAAgB,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,aAAa,EAAE;gBACpE,YAAY,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,UAAU,EAAE;aAChE;YACD,GAAG,OAAO;SACb,CAAC,CAAA;IACN,CAAC;IAED;;;OAGG;IACH,oBAAoB,CAAC,UAA6B,EAAE,OAA+B;QAC/E,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE;YAC3C,UAAU;YACV,iBAAiB,EAAE;gBACf,QAAQ,EAAE,cAAc,CAAC,QAAQ,CAAC,QAAQ;gBAC1C,QAAQ,EAAE,cAAc,CAAC,QAAQ,CAAC,QAAQ;gBAC1C,gBAAgB,EAAE,IAAI;gBACtB,YAAY,EAAE,KAAK;aACtB;YACD,GAAG,OAAO;SACb,CAAC,CAAA;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,+CAA+C;QAC/C,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE;YAC3B,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE;YAC3B,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE;SAC7B,CAAC,CAAA;QAEF,8CAA8C;QAC9C,IAAI,CAAC;YACD,mDAAmD;YACnD,MAAM,WAAW,GAAI,IAAI,CAAC,cAAsB,CAAC,eAAe,CAAA;YAChE,IAAI,WAAW,IAAI,OAAO,WAAW,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBAC9D,MAAM,WAAW,CAAC,UAAU,EAAE,CAAA;YAClC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,sCAAsC;QAC1C,CAAC;QAED,4CAA4C;QAC5C,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;IAC1D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,CAAC,cAAc,EAAE,cAAc,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACtE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC;SACrC,CAAC,CAAA;QAEF,OAAO;YACH,UAAU,EAAE,cAAc;YAC1B,UAAU,EAAE,cAAc;YAC1B,QAAQ,EAAE,aAAa;SAC1B,CAAA;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,KAAY;QACxB,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,KAAK,CAAC,eAAe,EAAE;YACvB,KAAK,CAAC,cAAc,EAAE;YACtB,KAAK,CAAC,iBAAiB,EAAE;YACzB,KAAK,CAAC,cAAc,EAAE;SACzB,CAAC,CAAA;QAEF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAA;IACjD,CAAC;CACJ"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { Collector } from '../components/collector.js';
|
|
2
|
+
import { Harvester } from '../components/harvester.js';
|
|
3
|
+
import { Worker } from 'bullmq';
|
|
4
|
+
import { QueueManager } from './queue_manager.js';
|
|
5
|
+
import { LogLevel } from '../utils/logger.js';
|
|
6
|
+
/**
|
|
7
|
+
* Schedules components for execution using the queue manager
|
|
8
|
+
*
|
|
9
|
+
* This function creates a scheduler instance and sets up:
|
|
10
|
+
* - Job scheduling based on component schedules
|
|
11
|
+
* - Event-driven harvester triggers
|
|
12
|
+
* - Workers for processing jobs
|
|
13
|
+
*
|
|
14
|
+
* @param components - Array of components to schedule
|
|
15
|
+
* @param queueManager - Queue manager instance
|
|
16
|
+
* @param multiQueue - Whether to use multi-queue mode (default: true)
|
|
17
|
+
* @param logLevel - Log level for the scheduler (optional)
|
|
18
|
+
* @returns Promise that resolves to array of created workers
|
|
19
|
+
*
|
|
20
|
+
* @example
|
|
21
|
+
* ```typescript
|
|
22
|
+
* const workers = await scheduleComponents(
|
|
23
|
+
* [collector1, harvester1],
|
|
24
|
+
* queueManager,
|
|
25
|
+
* true
|
|
26
|
+
* )
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
29
|
+
export declare function scheduleComponents(components: Array<Collector | Harvester>, queueManager: QueueManager, multiQueue?: boolean, logLevel?: LogLevel): Promise<Worker[]>;
|
|
30
|
+
//# sourceMappingURL=scheduler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scheduler.d.ts","sourceRoot":"","sources":["../../src/engine/scheduler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjD,OAAO,EAAU,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAibrD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,kBAAkB,CACpC,UAAU,EAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,EACxC,YAAY,EAAE,YAAY,EAC1B,UAAU,GAAE,OAAc,EAC1B,QAAQ,CAAC,EAAE,QAAQ,GACpB,OAAO,CAAC,MAAM,EAAE,CAAC,CAGnB"}
|