digitaltwin-core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +125 -0
  3. package/dist/components/assets_manager.d.ts +384 -0
  4. package/dist/components/assets_manager.d.ts.map +1 -0
  5. package/dist/components/assets_manager.js +637 -0
  6. package/dist/components/assets_manager.js.map +1 -0
  7. package/dist/components/collector.d.ts +62 -0
  8. package/dist/components/collector.d.ts.map +1 -0
  9. package/dist/components/collector.js +80 -0
  10. package/dist/components/collector.js.map +1 -0
  11. package/dist/components/global_assets_handler.d.ts +63 -0
  12. package/dist/components/global_assets_handler.d.ts.map +1 -0
  13. package/dist/components/global_assets_handler.js +127 -0
  14. package/dist/components/global_assets_handler.js.map +1 -0
  15. package/dist/components/handler.d.ts +33 -0
  16. package/dist/components/handler.d.ts.map +1 -0
  17. package/dist/components/handler.js +27 -0
  18. package/dist/components/handler.js.map +1 -0
  19. package/dist/components/harvester.d.ts +54 -0
  20. package/dist/components/harvester.d.ts.map +1 -0
  21. package/dist/components/harvester.js +253 -0
  22. package/dist/components/harvester.js.map +1 -0
  23. package/dist/components/index.d.ts +8 -0
  24. package/dist/components/index.d.ts.map +1 -0
  25. package/dist/components/index.js +6 -0
  26. package/dist/components/index.js.map +1 -0
  27. package/dist/components/interfaces.d.ts +18 -0
  28. package/dist/components/interfaces.d.ts.map +1 -0
  29. package/dist/components/interfaces.js +2 -0
  30. package/dist/components/interfaces.js.map +1 -0
  31. package/dist/components/types.d.ts +47 -0
  32. package/dist/components/types.d.ts.map +1 -0
  33. package/dist/components/types.js +2 -0
  34. package/dist/components/types.js.map +1 -0
  35. package/dist/database/adapters/knex_database_adapter.d.ts +47 -0
  36. package/dist/database/adapters/knex_database_adapter.d.ts.map +1 -0
  37. package/dist/database/adapters/knex_database_adapter.js +180 -0
  38. package/dist/database/adapters/knex_database_adapter.js.map +1 -0
  39. package/dist/database/database_adapter.d.ts +92 -0
  40. package/dist/database/database_adapter.d.ts.map +1 -0
  41. package/dist/database/database_adapter.js +6 -0
  42. package/dist/database/database_adapter.js.map +1 -0
  43. package/dist/engine/digital_twin_engine.d.ts +237 -0
  44. package/dist/engine/digital_twin_engine.d.ts.map +1 -0
  45. package/dist/engine/digital_twin_engine.js +574 -0
  46. package/dist/engine/digital_twin_engine.js.map +1 -0
  47. package/dist/engine/endpoints.d.ts +7 -0
  48. package/dist/engine/endpoints.d.ts.map +1 -0
  49. package/dist/engine/endpoints.js +18 -0
  50. package/dist/engine/endpoints.js.map +1 -0
  51. package/dist/engine/events.d.ts +13 -0
  52. package/dist/engine/events.d.ts.map +1 -0
  53. package/dist/engine/events.js +11 -0
  54. package/dist/engine/events.js.map +1 -0
  55. package/dist/engine/initializer.d.ts +8 -0
  56. package/dist/engine/initializer.d.ts.map +1 -0
  57. package/dist/engine/initializer.js +23 -0
  58. package/dist/engine/initializer.js.map +1 -0
  59. package/dist/engine/queue_manager.d.ts +78 -0
  60. package/dist/engine/queue_manager.d.ts.map +1 -0
  61. package/dist/engine/queue_manager.js +167 -0
  62. package/dist/engine/queue_manager.js.map +1 -0
  63. package/dist/engine/scheduler.d.ts +30 -0
  64. package/dist/engine/scheduler.d.ts.map +1 -0
  65. package/dist/engine/scheduler.js +378 -0
  66. package/dist/engine/scheduler.js.map +1 -0
  67. package/dist/env/env.d.ts +25 -0
  68. package/dist/env/env.d.ts.map +1 -0
  69. package/dist/env/env.js +55 -0
  70. package/dist/env/env.js.map +1 -0
  71. package/dist/index.d.ts +23 -0
  72. package/dist/index.d.ts.map +1 -0
  73. package/dist/index.js +31 -0
  74. package/dist/index.js.map +1 -0
  75. package/dist/storage/adapters/local_storage_service.d.ts +29 -0
  76. package/dist/storage/adapters/local_storage_service.d.ts.map +1 -0
  77. package/dist/storage/adapters/local_storage_service.js +50 -0
  78. package/dist/storage/adapters/local_storage_service.js.map +1 -0
  79. package/dist/storage/adapters/ovh_storage_service.d.ts +32 -0
  80. package/dist/storage/adapters/ovh_storage_service.d.ts.map +1 -0
  81. package/dist/storage/adapters/ovh_storage_service.js +70 -0
  82. package/dist/storage/adapters/ovh_storage_service.js.map +1 -0
  83. package/dist/storage/storage_factory.d.ts +14 -0
  84. package/dist/storage/storage_factory.d.ts.map +1 -0
  85. package/dist/storage/storage_factory.js +36 -0
  86. package/dist/storage/storage_factory.js.map +1 -0
  87. package/dist/storage/storage_service.d.ts +26 -0
  88. package/dist/storage/storage_service.d.ts.map +1 -0
  89. package/dist/storage/storage_service.js +7 -0
  90. package/dist/storage/storage_service.js.map +1 -0
  91. package/dist/types/data_record.d.ts +17 -0
  92. package/dist/types/data_record.d.ts.map +1 -0
  93. package/dist/types/data_record.js +2 -0
  94. package/dist/types/data_record.js.map +1 -0
  95. package/dist/utils/logger.d.ts +17 -0
  96. package/dist/utils/logger.d.ts.map +1 -0
  97. package/dist/utils/logger.js +35 -0
  98. package/dist/utils/logger.js.map +1 -0
  99. package/dist/utils/map_to_data_record.d.ts +10 -0
  100. package/dist/utils/map_to_data_record.d.ts.map +1 -0
  101. package/dist/utils/map_to_data_record.js +21 -0
  102. package/dist/utils/map_to_data_record.js.map +1 -0
  103. package/dist/utils/servable_endpoint.d.ts +6 -0
  104. package/dist/utils/servable_endpoint.d.ts.map +1 -0
  105. package/dist/utils/servable_endpoint.js +15 -0
  106. package/dist/utils/servable_endpoint.js.map +1 -0
  107. package/package.json +95 -0
@@ -0,0 +1,8 @@
1
+ import { Collector } from '../components/collector.js';
2
+ import { Harvester } from '../components/harvester.js';
3
+ import { AssetsManager } from '../components/assets_manager.js';
4
+ import { DatabaseAdapter } from '../database/database_adapter.js';
5
+ import { StorageService } from '../storage/storage_service.js';
6
+ export declare function initializeComponents(components: Array<Collector | Harvester>, database: DatabaseAdapter, storage: StorageService): Promise<void>;
7
+ export declare function initializeAssetsManagers(assetsManagers: AssetsManager[], database: DatabaseAdapter, storage: StorageService): Promise<void>;
8
+ //# sourceMappingURL=initializer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"initializer.d.ts","sourceRoot":"","sources":["../../src/engine/initializer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAA;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAA;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAA;AAE9D,wBAAsB,oBAAoB,CACtC,UAAU,EAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,EACxC,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,cAAc,GACxB,OAAO,CAAC,IAAI,CAAC,CAMf;AAED,wBAAsB,wBAAwB,CAC1C,cAAc,EAAE,aAAa,EAAE,EAC/B,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,cAAc,GACxB,OAAO,CAAC,IAAI,CAAC,CAMf"}
@@ -0,0 +1,23 @@
1
+ export async function initializeComponents(components, database, storage) {
2
+ for (const comp of components) {
3
+ const config = comp.getConfiguration();
4
+ await ensureTableExists(database, config.name);
5
+ comp.setDependencies(database, storage);
6
+ }
7
+ }
8
+ export async function initializeAssetsManagers(assetsManagers, database, storage) {
9
+ for (const manager of assetsManagers) {
10
+ const config = manager.getConfiguration();
11
+ await ensureTableExists(database, config.name);
12
+ manager.setDependencies(database, storage);
13
+ }
14
+ }
15
+ async function ensureTableExists(database, tableName) {
16
+ const exists = await database.doesTableExists(tableName);
17
+ if (!exists) {
18
+ console.log(`Creating table for component "${tableName}"...`);
19
+ await database.createTable(tableName);
20
+ console.log(`✅ Table "${tableName}" created successfully`);
21
+ }
22
+ }
23
+ //# sourceMappingURL=initializer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"initializer.js","sourceRoot":"","sources":["../../src/engine/initializer.ts"],"names":[],"mappings":"AAMA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,UAAwC,EACxC,QAAyB,EACzB,OAAuB;IAEvB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAA;QACtC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC3C,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,cAA+B,EAC/B,QAAyB,EACzB,OAAuB;IAEvB,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;QACzC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;QAC9C,OAAO,CAAC,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC9C,CAAC;AACL,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAyB,EAAE,SAAiB;IACzE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;IACxD,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,iCAAiC,SAAS,MAAM,CAAC,CAAA;QAC7D,MAAM,QAAQ,CAAC,WAAW,CAAC,SAAS,CAAC,CAAA;QACrC,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,wBAAwB,CAAC,CAAA;IAC9D,CAAC;AACL,CAAC"}
@@ -0,0 +1,78 @@
1
+ import { Queue, QueueOptions, ConnectionOptions } from 'bullmq';
2
+ /**
3
+ * Configuration options for the Queue Manager
4
+ */
5
+ export interface QueueConfig {
6
+ /** Number of workers for collectors (data collection) */
7
+ collectorWorkers?: number;
8
+ /** Number of workers for harvesters (data processing) */
9
+ harvesterWorkers?: number;
10
+ /** Redis connection configuration */
11
+ redis?: ConnectionOptions;
12
+ /** Advanced options for each queue type */
13
+ queueOptions?: {
14
+ collectors?: Partial<QueueOptions>;
15
+ harvesters?: Partial<QueueOptions>;
16
+ priority?: Partial<QueueOptions>;
17
+ };
18
+ }
19
+ /**
20
+ * Queue Manager - Manages BullMQ queues for different component types
21
+ *
22
+ * Handles three types of queues:
23
+ * - Collector queue: High priority, fast execution for data collection
24
+ * - Harvester queue: Medium priority, slower processing for data transformation
25
+ * - Priority queue: Urgent/manual jobs with highest priority
26
+ *
27
+ * @example
28
+ * ```typescript
29
+ * const queueManager = new QueueManager({
30
+ * redis: { host: 'localhost', port: 6379 },
31
+ * collectorWorkers: 3,
32
+ * harvesterWorkers: 2
33
+ * })
34
+ *
35
+ * await queueManager.close()
36
+ * ```
37
+ */
38
+ export declare class QueueManager {
39
+ #private;
40
+ readonly collectorQueue: Queue;
41
+ readonly harvesterQueue: Queue;
42
+ readonly priorityQueue: Queue;
43
+ /**
44
+ * Creates a new Queue Manager instance
45
+ * @param config - Queue configuration options
46
+ */
47
+ constructor(config?: QueueConfig);
48
+ /**
49
+ * Closes all queues gracefully
50
+ * @returns Promise that resolves when all queues are closed
51
+ */
52
+ close(): Promise<void>;
53
+ /**
54
+ * Gets statistics for all queues
55
+ * @returns Object containing stats for each queue type
56
+ */
57
+ getQueueStats(): Promise<{
58
+ collectors: {
59
+ waiting: number;
60
+ active: number;
61
+ completed: number;
62
+ failed: number;
63
+ };
64
+ harvesters: {
65
+ waiting: number;
66
+ active: number;
67
+ completed: number;
68
+ failed: number;
69
+ };
70
+ priority: {
71
+ waiting: number;
72
+ active: number;
73
+ completed: number;
74
+ failed: number;
75
+ };
76
+ }>;
77
+ }
78
+ //# sourceMappingURL=queue_manager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"queue_manager.d.ts","sourceRoot":"","sources":["../../src/engine/queue_manager.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAA;AAE/D;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB,yDAAyD;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,yDAAyD;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,qCAAqC;IACrC,KAAK,CAAC,EAAE,iBAAiB,CAAA;IACzB,2CAA2C;IAC3C,YAAY,CAAC,EAAE;QACX,UAAU,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;QAClC,UAAU,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;QAClC,QAAQ,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;KACnC,CAAA;CACJ;AAkCD;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,YAAY;;IACrB,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAA;IAC9B,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAA;IAC9B,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAA;IAE7B;;;OAGG;gBACS,MAAM,GAAE,WAAgB;IA2DpC;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAuB5B;;;OAGG;IACG,aAAa;;;;;;;;;;;;;;;;;;;;CA4BtB"}
@@ -0,0 +1,167 @@
1
+ // src/engine/queue_manager.ts
2
+ import { Queue } from 'bullmq';
3
+ /**
4
+ * Queue configuration constants
5
+ */
6
+ const QUEUE_DEFAULTS = {
7
+ REDIS: {
8
+ host: 'localhost',
9
+ port: 6379,
10
+ maxRetriesPerRequest: null,
11
+ enableReadyCheck: true,
12
+ retryStrategy: (times) => Math.min(times * 50, 2000)
13
+ },
14
+ COLLECTORS: {
15
+ name: 'dt-collectors',
16
+ attempts: 3,
17
+ backoffDelay: 2000,
18
+ keepCompleted: 100,
19
+ keepFailed: 50
20
+ },
21
+ HARVESTERS: {
22
+ name: 'dt-harvesters',
23
+ attempts: 5,
24
+ backoffDelay: 5000,
25
+ keepCompleted: 50,
26
+ keepFailed: 100
27
+ },
28
+ PRIORITY: {
29
+ name: 'dt-priority',
30
+ attempts: 2,
31
+ priority: 1
32
+ }
33
+ };
34
+ /**
35
+ * Queue Manager - Manages BullMQ queues for different component types
36
+ *
37
+ * Handles three types of queues:
38
+ * - Collector queue: High priority, fast execution for data collection
39
+ * - Harvester queue: Medium priority, slower processing for data transformation
40
+ * - Priority queue: Urgent/manual jobs with highest priority
41
+ *
42
+ * @example
43
+ * ```typescript
44
+ * const queueManager = new QueueManager({
45
+ * redis: { host: 'localhost', port: 6379 },
46
+ * collectorWorkers: 3,
47
+ * harvesterWorkers: 2
48
+ * })
49
+ *
50
+ * await queueManager.close()
51
+ * ```
52
+ */
53
+ export class QueueManager {
54
+ /**
55
+ * Creates a new Queue Manager instance
56
+ * @param config - Queue configuration options
57
+ */
58
+ constructor(config = {}) {
59
+ const baseConnection = config.redis || QUEUE_DEFAULTS.REDIS;
60
+ this.collectorQueue = this.#createCollectorQueue(baseConnection, config.queueOptions?.collectors);
61
+ this.harvesterQueue = this.#createHarvesterQueue(baseConnection, config.queueOptions?.harvesters);
62
+ this.priorityQueue = this.#createPriorityQueue(baseConnection, config.queueOptions?.priority);
63
+ }
64
+ /**
65
+ * Creates collector queue with optimized settings for data collection
66
+ * @private
67
+ */
68
+ #createCollectorQueue(connection, options) {
69
+ return new Queue(QUEUE_DEFAULTS.COLLECTORS.name, {
70
+ connection,
71
+ defaultJobOptions: {
72
+ attempts: QUEUE_DEFAULTS.COLLECTORS.attempts,
73
+ backoff: { type: 'exponential', delay: QUEUE_DEFAULTS.COLLECTORS.backoffDelay },
74
+ removeOnComplete: { count: QUEUE_DEFAULTS.COLLECTORS.keepCompleted },
75
+ removeOnFail: { count: QUEUE_DEFAULTS.COLLECTORS.keepFailed }
76
+ },
77
+ ...options
78
+ });
79
+ }
80
+ /**
81
+ * Creates harvester queue with settings optimized for data processing
82
+ * @private
83
+ */
84
+ #createHarvesterQueue(connection, options) {
85
+ return new Queue(QUEUE_DEFAULTS.HARVESTERS.name, {
86
+ connection,
87
+ defaultJobOptions: {
88
+ attempts: QUEUE_DEFAULTS.HARVESTERS.attempts,
89
+ backoff: { type: 'exponential', delay: QUEUE_DEFAULTS.HARVESTERS.backoffDelay },
90
+ removeOnComplete: { count: QUEUE_DEFAULTS.HARVESTERS.keepCompleted },
91
+ removeOnFail: { count: QUEUE_DEFAULTS.HARVESTERS.keepFailed }
92
+ },
93
+ ...options
94
+ });
95
+ }
96
+ /**
97
+ * Creates priority queue for urgent/manual jobs
98
+ * @private
99
+ */
100
+ #createPriorityQueue(connection, options) {
101
+ return new Queue(QUEUE_DEFAULTS.PRIORITY.name, {
102
+ connection,
103
+ defaultJobOptions: {
104
+ priority: QUEUE_DEFAULTS.PRIORITY.priority,
105
+ attempts: QUEUE_DEFAULTS.PRIORITY.attempts,
106
+ removeOnComplete: true,
107
+ removeOnFail: false
108
+ },
109
+ ...options
110
+ });
111
+ }
112
+ /**
113
+ * Closes all queues gracefully
114
+ * @returns Promise that resolves when all queues are closed
115
+ */
116
+ async close() {
117
+ // Close all queues and their Redis connections
118
+ await Promise.all([
119
+ this.collectorQueue.close(),
120
+ this.harvesterQueue.close(),
121
+ this.priorityQueue.close()
122
+ ]);
123
+ // Force close any lingering Redis connections
124
+ try {
125
+ // Get the Redis client from the queue and close it
126
+ const redisClient = this.collectorQueue.redisConnection;
127
+ if (redisClient && typeof redisClient.disconnect === 'function') {
128
+ await redisClient.disconnect();
129
+ }
130
+ }
131
+ catch (error) {
132
+ // Ignore errors during forced cleanup
133
+ }
134
+ // Wait for Redis connections to fully close
135
+ await new Promise(resolve => setTimeout(resolve, 200));
136
+ }
137
+ /**
138
+ * Gets statistics for all queues
139
+ * @returns Object containing stats for each queue type
140
+ */
141
+ async getQueueStats() {
142
+ const [collectorStats, harvesterStats, priorityStats] = await Promise.all([
143
+ this.#getStats(this.collectorQueue),
144
+ this.#getStats(this.harvesterQueue),
145
+ this.#getStats(this.priorityQueue)
146
+ ]);
147
+ return {
148
+ collectors: collectorStats,
149
+ harvesters: harvesterStats,
150
+ priority: priorityStats
151
+ };
152
+ }
153
+ /**
154
+ * Gets statistics for a specific queue
155
+ * @private
156
+ */
157
+ async #getStats(queue) {
158
+ const [waiting, active, completed, failed] = await Promise.all([
159
+ queue.getWaitingCount(),
160
+ queue.getActiveCount(),
161
+ queue.getCompletedCount(),
162
+ queue.getFailedCount()
163
+ ]);
164
+ return { waiting, active, completed, failed };
165
+ }
166
+ }
167
+ //# sourceMappingURL=queue_manager.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"queue_manager.js","sourceRoot":"","sources":["../../src/engine/queue_manager.ts"],"names":[],"mappings":"AAAA,8BAA8B;AAC9B,OAAO,EAAE,KAAK,EAAmC,MAAM,QAAQ,CAAA;AAoB/D;;GAEG;AACH,MAAM,cAAc,GAAG;IACnB,KAAK,EAAE;QACH,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,IAAI;QACV,oBAAoB,EAAE,IAAI;QAC1B,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,EAAE,IAAI,CAAC;KAC/D;IACD,UAAU,EAAE;QACR,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,CAAC;QACX,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,GAAG;QAClB,UAAU,EAAE,EAAE;KACjB;IACD,UAAU,EAAE;QACR,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,CAAC;QACX,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,EAAE;QACjB,UAAU,EAAE,GAAG;KAClB;IACD,QAAQ,EAAE;QACN,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;KACd;CACK,CAAA;AAEV;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,YAAY;IAKrB;;;OAGG;IACH,YAAY,SAAsB,EAAE;QAChC,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,IAAI,cAAc,CAAC,KAAK,CAAA;QAE3D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;QACjG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;QACjG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IACjG,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,UAA6B,EAAE,OAA+B;QAChF,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,EAAE;YAC7C,UAAU;YACV,iBAAiB,EAAE;gBACf,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,QAAQ;gBAC5C,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,YAAY,EAAE;gBAC/E,gBAAgB,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,aAAa,EAAE;gBACpE,YAAY,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,UAAU,EAAE;aAChE;YACD,GAAG,OAAO;SACb,CAAC,CAAA;IACN,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,UAA6B,EAAE,OAA+B;QAChF,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,EAAE;YAC7C,UAAU;YACV,iBAAiB,EAAE;gBACf,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,QAAQ;gBAC5C,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,YAAY,EAAE;gBAC/E,gBAAgB,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,aAAa,EAAE;gBACpE,YAAY,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,UAAU,EAAE;aAChE;YACD,GAAG,OAAO;SACb,CAAC,CAAA;IACN,CAAC;IAED;;;OAGG;IACH,oBAAoB,CAAC,UAA6B,EAAE,OAA+B;QAC/E,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE;YAC3C,UAAU;YACV,iBAAiB,EAAE;gBACf,QAAQ,EAAE,cAAc,CAAC,QAAQ,CAAC,QAAQ;gBAC1C,QAAQ,EAAE,cAAc,CAAC,QAAQ,CAAC,QAAQ;gBAC1C,gBAAgB,EAAE,IAAI;gBACtB,YAAY,EAAE,KAAK;aACtB;YACD,GAAG,OAAO;SACb,CAAC,CAAA;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,+CAA+C;QAC/C,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE;YAC3B,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE;YAC3B,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE;SAC7B,CAAC,CAAA;QAEF,8CAA8C;QAC9C,IAAI,CAAC;YACD,mDAAmD;YACnD,MAAM,WAAW,GAAI,IAAI,CAAC,cAAsB,CAAC,eAAe,CAAA;YAChE,IAAI,WAAW,IAAI,OAAO,WAAW,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBAC9D,MAAM,WAAW,CAAC,UAAU,EAAE,CAAA;YAClC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,sCAAsC;QAC1C,CAAC;QAED,4CAA4C;QAC5C,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;IAC1D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,CAAC,cAAc,EAAE,cAAc,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACtE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC;SACrC,CAAC,CAAA;QAEF,OAAO;YACH,UAAU,EAAE,cAAc;YAC1B,UAAU,EAAE,cAAc;YAC1B,QAAQ,EAAE,aAAa;SAC1B,CAAA;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,KAAY;QACxB,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,KAAK,CAAC,eAAe,EAAE;YACvB,KAAK,CAAC,cAAc,EAAE;YACtB,KAAK,CAAC,iBAAiB,EAAE;YACzB,KAAK,CAAC,cAAc,EAAE;SACzB,CAAC,CAAA;QAEF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAA;IACjD,CAAC;CACJ"}
@@ -0,0 +1,30 @@
1
+ import { Collector } from '../components/collector.js';
2
+ import { Harvester } from '../components/harvester.js';
3
+ import { Worker } from 'bullmq';
4
+ import { QueueManager } from './queue_manager.js';
5
+ import { LogLevel } from '../utils/logger.js';
6
+ /**
7
+ * Schedules components for execution using the queue manager
8
+ *
9
+ * This function creates a scheduler instance and sets up:
10
+ * - Job scheduling based on component schedules
11
+ * - Event-driven harvester triggers
12
+ * - Workers for processing jobs
13
+ *
14
+ * @param components - Array of components to schedule
15
+ * @param queueManager - Queue manager instance
16
+ * @param multiQueue - Whether to use multi-queue mode (default: true)
17
+ * @param logLevel - Log level for the scheduler (optional)
18
+ * @returns Promise that resolves to array of created workers
19
+ *
20
+ * @example
21
+ * ```typescript
22
+ * const workers = await scheduleComponents(
23
+ * [collector1, harvester1],
24
+ * queueManager,
25
+ * true
26
+ * )
27
+ * ```
28
+ */
29
+ export declare function scheduleComponents(components: Array<Collector | Harvester>, queueManager: QueueManager, multiQueue?: boolean, logLevel?: LogLevel): Promise<Worker[]>;
30
+ //# sourceMappingURL=scheduler.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scheduler.d.ts","sourceRoot":"","sources":["../../src/engine/scheduler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AACjD,OAAO,EAAU,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAibrD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,kBAAkB,CACpC,UAAU,EAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,EACxC,YAAY,EAAE,YAAY,EAC1B,UAAU,GAAE,OAAc,EAC1B,QAAQ,CAAC,EAAE,QAAQ,GACpB,OAAO,CAAC,MAAM,EAAE,CAAC,CAGnB"}