digitaltwin-core 0.14.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/LICENSE +20 -20
  2. package/README.md +494 -359
  3. package/dist/auth/apisix_parser.d.ts +141 -0
  4. package/dist/auth/apisix_parser.d.ts.map +1 -0
  5. package/dist/auth/apisix_parser.js +161 -0
  6. package/dist/auth/apisix_parser.js.map +1 -0
  7. package/dist/auth/auth_config.d.ts +126 -0
  8. package/dist/auth/auth_config.d.ts.map +1 -0
  9. package/dist/auth/auth_config.js +169 -0
  10. package/dist/auth/auth_config.js.map +1 -0
  11. package/dist/auth/auth_provider.d.ts +118 -0
  12. package/dist/auth/auth_provider.d.ts.map +1 -0
  13. package/dist/auth/auth_provider.js +8 -0
  14. package/dist/auth/auth_provider.js.map +1 -0
  15. package/dist/auth/auth_provider_factory.d.ts +91 -0
  16. package/dist/auth/auth_provider_factory.d.ts.map +1 -0
  17. package/dist/auth/auth_provider_factory.js +146 -0
  18. package/dist/auth/auth_provider_factory.js.map +1 -0
  19. package/dist/auth/index.d.ts +8 -0
  20. package/dist/auth/index.d.ts.map +1 -0
  21. package/dist/auth/index.js +7 -0
  22. package/dist/auth/index.js.map +1 -0
  23. package/dist/auth/providers/gateway_auth_provider.d.ts +78 -0
  24. package/dist/auth/providers/gateway_auth_provider.d.ts.map +1 -0
  25. package/dist/auth/providers/gateway_auth_provider.js +109 -0
  26. package/dist/auth/providers/gateway_auth_provider.js.map +1 -0
  27. package/dist/auth/providers/index.d.ts +4 -0
  28. package/dist/auth/providers/index.d.ts.map +1 -0
  29. package/dist/auth/providers/index.js +4 -0
  30. package/dist/auth/providers/index.js.map +1 -0
  31. package/dist/auth/providers/jwt_auth_provider.d.ts +91 -0
  32. package/dist/auth/providers/jwt_auth_provider.d.ts.map +1 -0
  33. package/dist/auth/providers/jwt_auth_provider.js +204 -0
  34. package/dist/auth/providers/jwt_auth_provider.js.map +1 -0
  35. package/dist/auth/providers/no_auth_provider.d.ts +61 -0
  36. package/dist/auth/providers/no_auth_provider.d.ts.map +1 -0
  37. package/dist/auth/providers/no_auth_provider.js +76 -0
  38. package/dist/auth/providers/no_auth_provider.js.map +1 -0
  39. package/dist/auth/types.d.ts +100 -0
  40. package/dist/auth/types.d.ts.map +1 -0
  41. package/dist/auth/types.js +2 -0
  42. package/dist/auth/types.js.map +1 -0
  43. package/dist/auth/user_service.d.ts +86 -0
  44. package/dist/auth/user_service.d.ts.map +1 -0
  45. package/dist/auth/user_service.js +237 -0
  46. package/dist/auth/user_service.js.map +1 -0
  47. package/dist/components/assets_manager.d.ts +662 -0
  48. package/dist/components/assets_manager.d.ts.map +1 -0
  49. package/dist/components/assets_manager.js +1537 -0
  50. package/dist/components/assets_manager.js.map +1 -0
  51. package/dist/components/async_upload.d.ts +20 -0
  52. package/dist/components/async_upload.d.ts.map +1 -0
  53. package/dist/components/async_upload.js +10 -0
  54. package/dist/components/async_upload.js.map +1 -0
  55. package/dist/components/collector.d.ts +203 -0
  56. package/dist/components/collector.d.ts.map +1 -0
  57. package/dist/components/collector.js +214 -0
  58. package/dist/components/collector.js.map +1 -0
  59. package/dist/components/custom_table_manager.d.ts +503 -0
  60. package/dist/components/custom_table_manager.d.ts.map +1 -0
  61. package/dist/components/custom_table_manager.js +1023 -0
  62. package/dist/components/custom_table_manager.js.map +1 -0
  63. package/dist/components/global_assets_handler.d.ts +63 -0
  64. package/dist/components/global_assets_handler.d.ts.map +1 -0
  65. package/dist/components/global_assets_handler.js +127 -0
  66. package/dist/components/global_assets_handler.js.map +1 -0
  67. package/dist/components/handler.d.ts +104 -0
  68. package/dist/components/handler.d.ts.map +1 -0
  69. package/dist/components/handler.js +110 -0
  70. package/dist/components/handler.js.map +1 -0
  71. package/dist/components/harvester.d.ts +182 -0
  72. package/dist/components/harvester.d.ts.map +1 -0
  73. package/dist/components/harvester.js +406 -0
  74. package/dist/components/harvester.js.map +1 -0
  75. package/dist/components/index.d.ts +11 -0
  76. package/dist/components/index.d.ts.map +1 -0
  77. package/dist/components/index.js +9 -0
  78. package/dist/components/index.js.map +1 -0
  79. package/dist/components/interfaces.d.ts +126 -0
  80. package/dist/components/interfaces.d.ts.map +1 -0
  81. package/dist/components/interfaces.js +8 -0
  82. package/dist/components/interfaces.js.map +1 -0
  83. package/dist/components/map_manager.d.ts +61 -0
  84. package/dist/components/map_manager.d.ts.map +1 -0
  85. package/dist/components/map_manager.js +242 -0
  86. package/dist/components/map_manager.js.map +1 -0
  87. package/dist/components/tileset_manager.d.ts +125 -0
  88. package/dist/components/tileset_manager.d.ts.map +1 -0
  89. package/dist/components/tileset_manager.js +623 -0
  90. package/dist/components/tileset_manager.js.map +1 -0
  91. package/dist/components/types.d.ts +226 -0
  92. package/dist/components/types.d.ts.map +1 -0
  93. package/dist/components/types.js +8 -0
  94. package/dist/components/types.js.map +1 -0
  95. package/dist/database/adapters/knex_database_adapter.d.ts +97 -0
  96. package/dist/database/adapters/knex_database_adapter.d.ts.map +1 -0
  97. package/dist/database/adapters/knex_database_adapter.js +729 -0
  98. package/dist/database/adapters/knex_database_adapter.js.map +1 -0
  99. package/dist/database/database_adapter.d.ts +262 -0
  100. package/dist/database/database_adapter.d.ts.map +1 -0
  101. package/dist/database/database_adapter.js +46 -0
  102. package/dist/database/database_adapter.js.map +1 -0
  103. package/dist/engine/digital_twin_engine.d.ts +295 -0
  104. package/dist/engine/digital_twin_engine.d.ts.map +1 -0
  105. package/dist/engine/digital_twin_engine.js +907 -0
  106. package/dist/engine/digital_twin_engine.js.map +1 -0
  107. package/dist/engine/endpoints.d.ts +47 -0
  108. package/dist/engine/endpoints.d.ts.map +1 -0
  109. package/dist/engine/endpoints.js +88 -0
  110. package/dist/engine/endpoints.js.map +1 -0
  111. package/dist/engine/error_handler.d.ts +20 -0
  112. package/dist/engine/error_handler.d.ts.map +1 -0
  113. package/dist/engine/error_handler.js +69 -0
  114. package/dist/engine/error_handler.js.map +1 -0
  115. package/dist/engine/events.d.ts +93 -0
  116. package/dist/engine/events.d.ts.map +1 -0
  117. package/dist/engine/events.js +71 -0
  118. package/dist/engine/events.js.map +1 -0
  119. package/dist/engine/health.d.ts +112 -0
  120. package/dist/engine/health.d.ts.map +1 -0
  121. package/dist/engine/health.js +190 -0
  122. package/dist/engine/health.js.map +1 -0
  123. package/dist/engine/initializer.d.ts +62 -0
  124. package/dist/engine/initializer.d.ts.map +1 -0
  125. package/dist/engine/initializer.js +108 -0
  126. package/dist/engine/initializer.js.map +1 -0
  127. package/dist/engine/queue_manager.d.ts +87 -0
  128. package/dist/engine/queue_manager.d.ts.map +1 -0
  129. package/dist/engine/queue_manager.js +196 -0
  130. package/dist/engine/queue_manager.js.map +1 -0
  131. package/dist/engine/scheduler.d.ts +30 -0
  132. package/dist/engine/scheduler.d.ts.map +1 -0
  133. package/dist/engine/scheduler.js +378 -0
  134. package/dist/engine/scheduler.js.map +1 -0
  135. package/dist/engine/upload_processor.d.ts +36 -0
  136. package/dist/engine/upload_processor.d.ts.map +1 -0
  137. package/dist/engine/upload_processor.js +113 -0
  138. package/dist/engine/upload_processor.js.map +1 -0
  139. package/dist/env/env.d.ts +134 -0
  140. package/dist/env/env.d.ts.map +1 -0
  141. package/dist/env/env.js +177 -0
  142. package/dist/env/env.js.map +1 -0
  143. package/dist/errors/index.d.ts +94 -0
  144. package/dist/errors/index.d.ts.map +1 -0
  145. package/dist/errors/index.js +149 -0
  146. package/dist/errors/index.js.map +1 -0
  147. package/dist/index.d.ts +55 -0
  148. package/dist/index.d.ts.map +1 -0
  149. package/dist/index.js +65 -0
  150. package/dist/index.js.map +1 -0
  151. package/dist/openapi/generator.d.ts +93 -0
  152. package/dist/openapi/generator.d.ts.map +1 -0
  153. package/dist/openapi/generator.js +293 -0
  154. package/dist/openapi/generator.js.map +1 -0
  155. package/dist/openapi/index.d.ts +9 -0
  156. package/dist/openapi/index.d.ts.map +1 -0
  157. package/dist/openapi/index.js +9 -0
  158. package/dist/openapi/index.js.map +1 -0
  159. package/dist/openapi/types.d.ts +182 -0
  160. package/dist/openapi/types.d.ts.map +1 -0
  161. package/dist/openapi/types.js +16 -0
  162. package/dist/openapi/types.js.map +1 -0
  163. package/dist/storage/adapters/local_storage_service.d.ts +57 -0
  164. package/dist/storage/adapters/local_storage_service.d.ts.map +1 -0
  165. package/dist/storage/adapters/local_storage_service.js +132 -0
  166. package/dist/storage/adapters/local_storage_service.js.map +1 -0
  167. package/dist/storage/adapters/ovh_storage_service.d.ts +72 -0
  168. package/dist/storage/adapters/ovh_storage_service.d.ts.map +1 -0
  169. package/dist/storage/adapters/ovh_storage_service.js +205 -0
  170. package/dist/storage/adapters/ovh_storage_service.js.map +1 -0
  171. package/dist/storage/storage_factory.d.ts +14 -0
  172. package/dist/storage/storage_factory.d.ts.map +1 -0
  173. package/dist/storage/storage_factory.js +43 -0
  174. package/dist/storage/storage_factory.js.map +1 -0
  175. package/dist/storage/storage_service.d.ts +163 -0
  176. package/dist/storage/storage_service.d.ts.map +1 -0
  177. package/dist/storage/storage_service.js +58 -0
  178. package/dist/storage/storage_service.js.map +1 -0
  179. package/dist/types/data_record.d.ts +123 -0
  180. package/dist/types/data_record.d.ts.map +1 -0
  181. package/dist/types/data_record.js +8 -0
  182. package/dist/types/data_record.js.map +1 -0
  183. package/dist/utils/graceful_shutdown.d.ts +44 -0
  184. package/dist/utils/graceful_shutdown.d.ts.map +1 -0
  185. package/dist/utils/graceful_shutdown.js +79 -0
  186. package/dist/utils/graceful_shutdown.js.map +1 -0
  187. package/dist/utils/http_responses.d.ts +175 -0
  188. package/dist/utils/http_responses.d.ts.map +1 -0
  189. package/dist/utils/http_responses.js +216 -0
  190. package/dist/utils/http_responses.js.map +1 -0
  191. package/dist/utils/index.d.ts +8 -0
  192. package/dist/utils/index.d.ts.map +1 -0
  193. package/dist/utils/index.js +6 -0
  194. package/dist/utils/index.js.map +1 -0
  195. package/dist/utils/logger.d.ts +74 -0
  196. package/dist/utils/logger.d.ts.map +1 -0
  197. package/dist/utils/logger.js +92 -0
  198. package/dist/utils/logger.js.map +1 -0
  199. package/dist/utils/map_to_data_record.d.ts +10 -0
  200. package/dist/utils/map_to_data_record.d.ts.map +1 -0
  201. package/dist/utils/map_to_data_record.js +36 -0
  202. package/dist/utils/map_to_data_record.js.map +1 -0
  203. package/dist/utils/safe_async.d.ts +50 -0
  204. package/dist/utils/safe_async.d.ts.map +1 -0
  205. package/dist/utils/safe_async.js +90 -0
  206. package/dist/utils/safe_async.js.map +1 -0
  207. package/dist/utils/servable_endpoint.d.ts +63 -0
  208. package/dist/utils/servable_endpoint.d.ts.map +1 -0
  209. package/dist/utils/servable_endpoint.js +67 -0
  210. package/dist/utils/servable_endpoint.js.map +1 -0
  211. package/dist/utils/zip_utils.d.ts +66 -0
  212. package/dist/utils/zip_utils.d.ts.map +1 -0
  213. package/dist/utils/zip_utils.js +169 -0
  214. package/dist/utils/zip_utils.js.map +1 -0
  215. package/dist/validation/index.d.ts +3 -0
  216. package/dist/validation/index.d.ts.map +1 -0
  217. package/dist/validation/index.js +7 -0
  218. package/dist/validation/index.js.map +1 -0
  219. package/dist/validation/schemas.d.ts +273 -0
  220. package/dist/validation/schemas.d.ts.map +1 -0
  221. package/dist/validation/schemas.js +82 -0
  222. package/dist/validation/schemas.js.map +1 -0
  223. package/dist/validation/validate.d.ts +49 -0
  224. package/dist/validation/validate.d.ts.map +1 -0
  225. package/dist/validation/validate.js +110 -0
  226. package/dist/validation/validate.js.map +1 -0
  227. package/package.json +23 -13
@@ -0,0 +1,190 @@
1
+ /**
2
+ * Health checker with support for custom checks
3
+ */
4
+ export class HealthChecker {
5
+ #checks = new Map();
6
+ #componentCounts;
7
+ #version;
8
+ /**
9
+ * Register a custom health check
10
+ * @param name Unique name for the check
11
+ * @param checkFn Function that performs the check
12
+ *
13
+ * @example
14
+ * ```typescript
15
+ * healthChecker.registerCheck('external-api', async () => {
16
+ * try {
17
+ * await fetch('https://api.example.com/health')
18
+ * return { status: 'up' }
19
+ * } catch (error) {
20
+ * return { status: 'down', error: error.message }
21
+ * }
22
+ * })
23
+ * ```
24
+ */
25
+ registerCheck(name, checkFn) {
26
+ this.#checks.set(name, checkFn);
27
+ }
28
+ /**
29
+ * Remove a health check
30
+ * @param name Name of the check to remove
31
+ */
32
+ removeCheck(name) {
33
+ return this.#checks.delete(name);
34
+ }
35
+ /**
36
+ * Get list of registered check names
37
+ */
38
+ getCheckNames() {
39
+ return Array.from(this.#checks.keys());
40
+ }
41
+ /**
42
+ * Set component counts for health status
43
+ */
44
+ setComponentCounts(counts) {
45
+ this.#componentCounts = counts;
46
+ }
47
+ /**
48
+ * Set version for health status
49
+ */
50
+ setVersion(version) {
51
+ this.#version = version;
52
+ }
53
+ /**
54
+ * Perform all registered health checks
55
+ */
56
+ async performCheck() {
57
+ const checks = {};
58
+ // Run all checks in parallel
59
+ const entries = Array.from(this.#checks.entries());
60
+ const results = await Promise.all(entries.map(async ([name, checkFn]) => {
61
+ const start = Date.now();
62
+ try {
63
+ const result = await checkFn();
64
+ return [name, { ...result, latency: result.latency ?? Date.now() - start }];
65
+ }
66
+ catch (error) {
67
+ return [
68
+ name,
69
+ {
70
+ status: 'down',
71
+ latency: Date.now() - start,
72
+ error: error instanceof Error ? error.message : 'Unknown error'
73
+ }
74
+ ];
75
+ }
76
+ }));
77
+ results.forEach(([name, result]) => {
78
+ checks[name] = result;
79
+ });
80
+ // Determine overall status
81
+ const allChecks = Object.values(checks);
82
+ const anyDown = allChecks.some(c => c.status === 'down');
83
+ const databaseDown = checks['database']?.status === 'down';
84
+ let status;
85
+ if (databaseDown) {
86
+ status = 'unhealthy';
87
+ }
88
+ else if (anyDown) {
89
+ status = 'degraded';
90
+ }
91
+ else {
92
+ status = 'healthy';
93
+ }
94
+ return {
95
+ status,
96
+ timestamp: new Date().toISOString(),
97
+ uptime: process.uptime(),
98
+ ...(this.#version && { version: this.#version }),
99
+ checks,
100
+ ...(this.#componentCounts && { components: this.#componentCounts })
101
+ };
102
+ }
103
+ }
104
+ /**
105
+ * Create a database health check function
106
+ */
107
+ export function createDatabaseCheck(db) {
108
+ return async () => {
109
+ const start = Date.now();
110
+ try {
111
+ // Use doesTableExists as a ping - it will fail if DB is unreachable
112
+ await db.doesTableExists('_health_check_ping');
113
+ return { status: 'up', latency: Date.now() - start };
114
+ }
115
+ catch (error) {
116
+ return {
117
+ status: 'down',
118
+ latency: Date.now() - start,
119
+ error: error instanceof Error ? error.message : 'Unknown error'
120
+ };
121
+ }
122
+ };
123
+ }
124
+ /**
125
+ * Create a Redis health check function via QueueManager
126
+ */
127
+ export function createRedisCheck(qm) {
128
+ return async () => {
129
+ const start = Date.now();
130
+ try {
131
+ await qm.getQueueStats();
132
+ return { status: 'up', latency: Date.now() - start };
133
+ }
134
+ catch (error) {
135
+ return {
136
+ status: 'down',
137
+ latency: Date.now() - start,
138
+ error: error instanceof Error ? error.message : 'Unknown error'
139
+ };
140
+ }
141
+ };
142
+ }
143
+ /**
144
+ * Create a storage health check function
145
+ */
146
+ export function createStorageCheck(storage) {
147
+ return async () => {
148
+ const start = Date.now();
149
+ try {
150
+ if ('checkConnection' in storage && typeof storage.checkConnection === 'function') {
151
+ await storage.checkConnection();
152
+ }
153
+ return { status: 'up', latency: Date.now() - start };
154
+ }
155
+ catch (error) {
156
+ return {
157
+ status: 'down',
158
+ latency: Date.now() - start,
159
+ error: error instanceof Error ? error.message : 'Unknown error'
160
+ };
161
+ }
162
+ };
163
+ }
164
+ /**
165
+ * Perform deep health check on all services (convenience function)
166
+ */
167
+ export async function performHealthCheck(database, queueManager, storage, componentCounts, version) {
168
+ const checker = new HealthChecker();
169
+ checker.registerCheck('database', createDatabaseCheck(database));
170
+ if (queueManager) {
171
+ checker.registerCheck('redis', createRedisCheck(queueManager));
172
+ }
173
+ if (storage) {
174
+ checker.registerCheck('storage', createStorageCheck(storage));
175
+ }
176
+ if (componentCounts) {
177
+ checker.setComponentCounts(componentCounts);
178
+ }
179
+ if (version) {
180
+ checker.setVersion(version);
181
+ }
182
+ return checker.performCheck();
183
+ }
184
+ /**
185
+ * Simple liveness check - always returns ok if process is running
186
+ */
187
+ export function livenessCheck() {
188
+ return { status: 'ok' };
189
+ }
190
+ //# sourceMappingURL=health.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"health.js","sourceRoot":"","sources":["../../src/engine/health.ts"],"names":[],"mappings":"AAiDA;;GAEG;AACH,MAAM,OAAO,aAAa;IACb,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAA;IACnD,gBAAgB,CAAkB;IAClC,QAAQ,CAAS;IAEjB;;;;;;;;;;;;;;;;OAgBG;IACH,aAAa,CAAC,IAAY,EAAE,OAAsB;QAC9C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IACnC,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC;IAED;;OAEG;IACH,aAAa;QACT,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;IAC1C,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,MAAuB;QACtC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAA;IAClC,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,OAAe;QACtB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,MAAM,MAAM,GAAgC,EAAE,CAAA;QAE9C,6BAA6B;QAC7B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;QAClD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YACxB,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,OAAO,EAAE,CAAA;gBAC9B,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAU,CAAA;YACxF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO;oBACH,IAAI;oBACJ;wBACI,MAAM,EAAE,MAAe;wBACvB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;wBAC3B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;qBAClE;iBACK,CAAA;YACd,CAAC;QACL,CAAC,CAAC,CACL,CAAA;QAED,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAA;QACzB,CAAC,CAAC,CAAA;QAEF,2BAA2B;QAC3B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QACvC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;QACxD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,EAAE,MAAM,KAAK,MAAM,CAAA;QAE1D,IAAI,MAA8B,CAAA;QAClC,IAAI,YAAY,EAAE,CAAC;YACf,MAAM,GAAG,WAAW,CAAA;QACxB,CAAC;aAAM,IAAI,OAAO,EAAE,CAAC;YACjB,MAAM,GAAG,UAAU,CAAA;QACvB,CAAC;aAAM,CAAC;YACJ,MAAM,GAAG,SAAS,CAAA;QACtB,CAAC;QAED,OAAO;YACH,MAAM;YACN,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;YACxB,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChD,MAAM;YACN,GAAG,CAAC,IAAI,CAAC,gBAAgB,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;SACtE,CAAA;IACL,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,EAAmB;IACnD,OAAO,KAAK,IAAI,EAAE;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,IAAI,CAAC;YACD,oEAAoE;YACpE,MAAM,EAAE,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA;YAC9C,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAA;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC3B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;aAClE,CAAA;QACL,CAAC;IACL,CAAC,CAAA;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,EAAgB;IAC7C,OAAO,KAAK,IAAI,EAAE;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,IAAI,CAAC;YACD,MAAM,EAAE,CAAC,aAAa,EAAE,CAAA;YACxB,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAA;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC3B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;aAClE,CAAA;QACL,CAAC;IACL,CAAC,CAAA;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAuB;IACtD,OAAO,KAAK,IAAI,EAAE;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,IAAI,CAAC;YACD,IAAI,iBAAiB,IAAI,OAAO,IAAI,OAAO,OAAO,CAAC,eAAe,KAAK,UAAU,EAAE,CAAC;gBAChF,MAAO,OAAgD,CAAC,eAAe,EAAE,CAAA;YAC7E,CAAC;YACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAA;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO;gBACH,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC3B,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;aAClE,CAAA;QACL,CAAC;IACL,CAAC,CAAA;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACpC,QAAyB,EACzB,YAAkC,EAClC,OAAwB,EACxB,eAAiC,EACjC,OAAgB;IAEhB,MAAM,OAAO,GAAG,IAAI,aAAa,EAAE,CAAA;IAEnC,OAAO,CAAC,aAAa,CAAC,UAAU,EAAE,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAA;IAEhE,IAAI,YAAY,EAAE,CAAC;QACf,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAA;IAClE,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACV,OAAO,CAAC,aAAa,CAAC,SAAS,EAAE,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAA;IAC/C,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACV,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,OAAO,CAAC,YAAY,EAAE,CAAA;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa;IACzB,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;AAC3B,CAAC"}
@@ -0,0 +1,62 @@
1
+ /**
2
+ * @fileoverview Component initialization utilities for the digital twin engine
3
+ *
4
+ * This module handles the setup and initialization of digital twin components,
5
+ * including database table creation and dependency injection.
6
+ */
7
+ import type { Collector } from '../components/collector.js';
8
+ import type { Harvester } from '../components/harvester.js';
9
+ import type { AssetsManager } from '../components/assets_manager.js';
10
+ import type { DatabaseAdapter } from '../database/database_adapter.js';
11
+ import type { StorageService } from '../storage/storage_service.js';
12
+ /**
13
+ * Initializes data collection and processing components with required dependencies.
14
+ *
15
+ * This function sets up collectors and harvesters by:
16
+ * 1. Creating necessary database tables for each component
17
+ * 2. Running automatic schema migrations if enabled
18
+ * 3. Injecting database and storage service dependencies
19
+ *
20
+ * @param components - Array of collectors and harvesters to initialize
21
+ * @param database - Database adapter instance for data storage
22
+ * @param storage - Storage service instance for file operations
23
+ * @param autoMigration - Enable automatic schema migration (default: true)
24
+ *
25
+ * @throws {Error} When database table creation fails
26
+ *
27
+ * @example
28
+ * ```typescript
29
+ * const components = [weatherCollector, trafficHarvester];
30
+ * const database = new KnexDatabaseAdapter(config);
31
+ * const storage = new LocalStorageService();
32
+ *
33
+ * await initializeComponents(components, database, storage, true);
34
+ * // Components are now ready to process data
35
+ * ```
36
+ */
37
+ export declare function initializeComponents(components: Array<Collector | Harvester>, database: DatabaseAdapter, storage: StorageService, autoMigration?: boolean): Promise<void>;
38
+ /**
39
+ * Initializes asset management components with required dependencies.
40
+ *
41
+ * Asset managers handle file-based resources (like 3D models, tilesets, maps)
42
+ * and require both database access for metadata and storage for file operations.
43
+ *
44
+ * @param assetsManagers - Array of asset managers to initialize
45
+ * @param database - Database adapter instance for metadata storage
46
+ * @param storage - Storage service instance for asset file operations
47
+ * @param autoMigration - Enable automatic schema migration (default: true)
48
+ *
49
+ * @throws {Error} When database table creation fails
50
+ *
51
+ * @example
52
+ * ```typescript
53
+ * const managers = [tilesetManager, pointCloudManager];
54
+ * const database = new KnexDatabaseAdapter(config);
55
+ * const storage = new OvhS3StorageService(credentials);
56
+ *
57
+ * await initializeAssetsManagers(managers, database, storage, true);
58
+ * // Asset managers are now ready to handle file operations
59
+ * ```
60
+ */
61
+ export declare function initializeAssetsManagers(assetsManagers: AssetsManager[], database: DatabaseAdapter, storage: StorageService, autoMigration?: boolean): Promise<void>;
62
+ //# sourceMappingURL=initializer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"initializer.d.ts","sourceRoot":"","sources":["../../src/engine/initializer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAA;AACpE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAA;AACtE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAA;AAEnE;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,oBAAoB,CACtC,UAAU,EAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC,EACxC,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,cAAc,EACvB,aAAa,GAAE,OAAc,GAC9B,OAAO,CAAC,IAAI,CAAC,CAMf;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAsB,wBAAwB,CAC1C,cAAc,EAAE,aAAa,EAAE,EAC/B,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,cAAc,EACvB,aAAa,GAAE,OAAc,GAC9B,OAAO,CAAC,IAAI,CAAC,CAMf"}
@@ -0,0 +1,108 @@
1
+ /**
2
+ * @fileoverview Component initialization utilities for the digital twin engine
3
+ *
4
+ * This module handles the setup and initialization of digital twin components,
5
+ * including database table creation and dependency injection.
6
+ */
7
+ /**
8
+ * Initializes data collection and processing components with required dependencies.
9
+ *
10
+ * This function sets up collectors and harvesters by:
11
+ * 1. Creating necessary database tables for each component
12
+ * 2. Running automatic schema migrations if enabled
13
+ * 3. Injecting database and storage service dependencies
14
+ *
15
+ * @param components - Array of collectors and harvesters to initialize
16
+ * @param database - Database adapter instance for data storage
17
+ * @param storage - Storage service instance for file operations
18
+ * @param autoMigration - Enable automatic schema migration (default: true)
19
+ *
20
+ * @throws {Error} When database table creation fails
21
+ *
22
+ * @example
23
+ * ```typescript
24
+ * const components = [weatherCollector, trafficHarvester];
25
+ * const database = new KnexDatabaseAdapter(config);
26
+ * const storage = new LocalStorageService();
27
+ *
28
+ * await initializeComponents(components, database, storage, true);
29
+ * // Components are now ready to process data
30
+ * ```
31
+ */
32
+ export async function initializeComponents(components, database, storage, autoMigration = true) {
33
+ for (const comp of components) {
34
+ const config = comp.getConfiguration();
35
+ await ensureTableExists(database, config.name, autoMigration);
36
+ comp.setDependencies(database, storage);
37
+ }
38
+ }
39
+ /**
40
+ * Initializes asset management components with required dependencies.
41
+ *
42
+ * Asset managers handle file-based resources (like 3D models, tilesets, maps)
43
+ * and require both database access for metadata and storage for file operations.
44
+ *
45
+ * @param assetsManagers - Array of asset managers to initialize
46
+ * @param database - Database adapter instance for metadata storage
47
+ * @param storage - Storage service instance for asset file operations
48
+ * @param autoMigration - Enable automatic schema migration (default: true)
49
+ *
50
+ * @throws {Error} When database table creation fails
51
+ *
52
+ * @example
53
+ * ```typescript
54
+ * const managers = [tilesetManager, pointCloudManager];
55
+ * const database = new KnexDatabaseAdapter(config);
56
+ * const storage = new OvhS3StorageService(credentials);
57
+ *
58
+ * await initializeAssetsManagers(managers, database, storage, true);
59
+ * // Asset managers are now ready to handle file operations
60
+ * ```
61
+ */
62
+ export async function initializeAssetsManagers(assetsManagers, database, storage, autoMigration = true) {
63
+ for (const manager of assetsManagers) {
64
+ const config = manager.getConfiguration();
65
+ await ensureTableExists(database, config.name, autoMigration);
66
+ manager.setDependencies(database, storage);
67
+ }
68
+ }
69
+ /**
70
+ * Ensures a database table exists for the specified component.
71
+ *
72
+ * Checks if a table exists in the database and creates it if missing.
73
+ * If autoMigration is enabled and table exists, runs schema migration.
74
+ * This function is called during component initialization to ensure
75
+ * each component has its required storage table available.
76
+ *
77
+ * @param database - Database adapter to check/create table with
78
+ * @param tableName - Name of the table to ensure exists
79
+ * @param autoMigration - Enable automatic schema migration for existing tables
80
+ *
81
+ * @throws {Error} When table creation or migration fails
82
+ *
83
+ * @example
84
+ * ```typescript
85
+ * // Internal usage during component initialization
86
+ * await ensureTableExists(database, 'weather_data_collector', true);
87
+ * // Table now exists and schema is up-to-date
88
+ * ```
89
+ *
90
+ * @internal This function is used internally by initialization functions
91
+ */
92
+ async function ensureTableExists(database, tableName, autoMigration = true) {
93
+ const exists = await database.doesTableExists(tableName);
94
+ if (!exists) {
95
+ await database.createTable(tableName);
96
+ if (process.env.NODE_ENV !== 'test') {
97
+ console.log(`[DigitalTwin] Created table "${tableName}"`);
98
+ }
99
+ }
100
+ else if (autoMigration) {
101
+ // Table exists, run migration to add missing columns/indexes
102
+ const migrations = await database.migrateTableSchema(tableName);
103
+ if (migrations.length > 0 && process.env.NODE_ENV !== 'test') {
104
+ console.log(`[DigitalTwin] Migrated "${tableName}": ${migrations.join(', ')}`);
105
+ }
106
+ }
107
+ }
108
+ //# sourceMappingURL=initializer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"initializer.js","sourceRoot":"","sources":["../../src/engine/initializer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,UAAwC,EACxC,QAAyB,EACzB,OAAuB,EACvB,gBAAyB,IAAI;IAE7B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAA;QACtC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;QAC7D,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC3C,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,cAA+B,EAC/B,QAAyB,EACzB,OAAuB,EACvB,gBAAyB,IAAI;IAE7B,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;QACzC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;QAC7D,OAAO,CAAC,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC9C,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,KAAK,UAAU,iBAAiB,CAC5B,QAAyB,EACzB,SAAiB,EACjB,gBAAyB,IAAI;IAE7B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;IACxD,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,QAAQ,CAAC,WAAW,CAAC,SAAS,CAAC,CAAA;QACrC,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,gCAAgC,SAAS,GAAG,CAAC,CAAA;QAC7D,CAAC;IACL,CAAC;SAAM,IAAI,aAAa,EAAE,CAAC;QACvB,6DAA6D;QAC7D,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAC/D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,2BAA2B,SAAS,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAClF,CAAC;IACL,CAAC;AACL,CAAC"}
@@ -0,0 +1,87 @@
1
+ import type { QueueOptions, ConnectionOptions } from 'bullmq';
2
+ import { Queue } from 'bullmq';
3
+ /**
4
+ * Configuration options for the Queue Manager
5
+ */
6
+ export interface QueueConfig {
7
+ /** Number of workers for collectors (data collection) */
8
+ collectorWorkers?: number;
9
+ /** Number of workers for harvesters (data processing) */
10
+ harvesterWorkers?: number;
11
+ /** Redis connection configuration */
12
+ redis?: ConnectionOptions;
13
+ /** Advanced options for each queue type */
14
+ queueOptions?: {
15
+ collectors?: Partial<QueueOptions>;
16
+ harvesters?: Partial<QueueOptions>;
17
+ priority?: Partial<QueueOptions>;
18
+ uploads?: Partial<QueueOptions>;
19
+ };
20
+ }
21
+ /**
22
+ * Queue Manager - Manages BullMQ queues for different component types
23
+ *
24
+ * Handles three types of queues:
25
+ * - Collector queue: High priority, fast execution for data collection
26
+ * - Harvester queue: Medium priority, slower processing for data transformation
27
+ * - Priority queue: Urgent/manual jobs with highest priority
28
+ *
29
+ * @example
30
+ * ```typescript
31
+ * const queueManager = new QueueManager({
32
+ * redis: { host: 'localhost', port: 6379 },
33
+ * collectorWorkers: 3,
34
+ * harvesterWorkers: 2
35
+ * })
36
+ *
37
+ * await queueManager.close()
38
+ * ```
39
+ */
40
+ export declare class QueueManager {
41
+ #private;
42
+ readonly collectorQueue: Queue;
43
+ readonly harvesterQueue: Queue;
44
+ readonly priorityQueue: Queue;
45
+ readonly uploadQueue: Queue;
46
+ /**
47
+ * Creates a new Queue Manager instance
48
+ * @param config - Queue configuration options
49
+ */
50
+ constructor(config?: QueueConfig);
51
+ /**
52
+ * Closes all queues gracefully
53
+ * @returns Promise that resolves when all queues are closed
54
+ */
55
+ close(): Promise<void>;
56
+ /**
57
+ * Gets statistics for all queues
58
+ * @returns Object containing stats for each queue type
59
+ */
60
+ getQueueStats(): Promise<{
61
+ collectors: {
62
+ waiting: number;
63
+ active: number;
64
+ completed: number;
65
+ failed: number;
66
+ };
67
+ harvesters: {
68
+ waiting: number;
69
+ active: number;
70
+ completed: number;
71
+ failed: number;
72
+ };
73
+ priority: {
74
+ waiting: number;
75
+ active: number;
76
+ completed: number;
77
+ failed: number;
78
+ };
79
+ uploads: {
80
+ waiting: number;
81
+ active: number;
82
+ completed: number;
83
+ failed: number;
84
+ };
85
+ }>;
86
+ }
87
+ //# sourceMappingURL=queue_manager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"queue_manager.d.ts","sourceRoot":"","sources":["../../src/engine/queue_manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAA;AAE9B;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB,yDAAyD;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,yDAAyD;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,qCAAqC;IACrC,KAAK,CAAC,EAAE,iBAAiB,CAAA;IACzB,2CAA2C;IAC3C,YAAY,CAAC,EAAE;QACX,UAAU,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;QAClC,UAAU,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;QAClC,QAAQ,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;QAChC,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;KAClC,CAAA;CACJ;AAyCD;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,YAAY;;IACrB,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAA;IAC9B,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAA;IAC9B,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAA;IAC7B,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAA;IAE3B;;;OAGG;gBACS,MAAM,GAAE,WAAgB;IA6EpC;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA+B5B;;;OAGG;IACG,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;CA8BtB"}
@@ -0,0 +1,196 @@
1
+ import { Queue } from 'bullmq';
2
+ /**
3
+ * Queue configuration constants
4
+ */
5
+ const QUEUE_DEFAULTS = {
6
+ REDIS: {
7
+ host: 'localhost',
8
+ port: 6379,
9
+ maxRetriesPerRequest: null,
10
+ enableReadyCheck: true,
11
+ retryStrategy: (times) => Math.min(times * 50, 2000)
12
+ },
13
+ COLLECTORS: {
14
+ name: 'dt-collectors',
15
+ attempts: 3,
16
+ backoffDelay: 2000,
17
+ keepCompleted: 100,
18
+ keepFailed: 50
19
+ },
20
+ HARVESTERS: {
21
+ name: 'dt-harvesters',
22
+ attempts: 5,
23
+ backoffDelay: 5000,
24
+ keepCompleted: 50,
25
+ keepFailed: 100
26
+ },
27
+ PRIORITY: {
28
+ name: 'dt-priority',
29
+ attempts: 2,
30
+ priority: 1
31
+ },
32
+ UPLOADS: {
33
+ name: 'dt-uploads',
34
+ attempts: 1, // No retries - temp file is deleted after first attempt
35
+ backoffDelay: 10000,
36
+ keepCompleted: 50,
37
+ keepFailed: 100
38
+ }
39
+ };
40
+ /**
41
+ * Queue Manager - Manages BullMQ queues for different component types
42
+ *
43
+ * Handles three types of queues:
44
+ * - Collector queue: High priority, fast execution for data collection
45
+ * - Harvester queue: Medium priority, slower processing for data transformation
46
+ * - Priority queue: Urgent/manual jobs with highest priority
47
+ *
48
+ * @example
49
+ * ```typescript
50
+ * const queueManager = new QueueManager({
51
+ * redis: { host: 'localhost', port: 6379 },
52
+ * collectorWorkers: 3,
53
+ * harvesterWorkers: 2
54
+ * })
55
+ *
56
+ * await queueManager.close()
57
+ * ```
58
+ */
59
+ export class QueueManager {
60
+ /**
61
+ * Creates a new Queue Manager instance
62
+ * @param config - Queue configuration options
63
+ */
64
+ constructor(config = {}) {
65
+ const baseConnection = config.redis || QUEUE_DEFAULTS.REDIS;
66
+ this.collectorQueue = this.#createCollectorQueue(baseConnection, config.queueOptions?.collectors);
67
+ this.harvesterQueue = this.#createHarvesterQueue(baseConnection, config.queueOptions?.harvesters);
68
+ this.priorityQueue = this.#createPriorityQueue(baseConnection, config.queueOptions?.priority);
69
+ this.uploadQueue = this.#createUploadQueue(baseConnection, config.queueOptions?.uploads);
70
+ }
71
+ /**
72
+ * Creates collector queue with optimized settings for data collection
73
+ * @private
74
+ */
75
+ #createCollectorQueue(connection, options) {
76
+ return new Queue(QUEUE_DEFAULTS.COLLECTORS.name, {
77
+ connection,
78
+ defaultJobOptions: {
79
+ attempts: QUEUE_DEFAULTS.COLLECTORS.attempts,
80
+ backoff: { type: 'exponential', delay: QUEUE_DEFAULTS.COLLECTORS.backoffDelay },
81
+ removeOnComplete: { count: QUEUE_DEFAULTS.COLLECTORS.keepCompleted },
82
+ removeOnFail: { count: QUEUE_DEFAULTS.COLLECTORS.keepFailed }
83
+ },
84
+ ...options
85
+ });
86
+ }
87
+ /**
88
+ * Creates harvester queue with settings optimized for data processing
89
+ * @private
90
+ */
91
+ #createHarvesterQueue(connection, options) {
92
+ return new Queue(QUEUE_DEFAULTS.HARVESTERS.name, {
93
+ connection,
94
+ defaultJobOptions: {
95
+ attempts: QUEUE_DEFAULTS.HARVESTERS.attempts,
96
+ backoff: { type: 'exponential', delay: QUEUE_DEFAULTS.HARVESTERS.backoffDelay },
97
+ removeOnComplete: { count: QUEUE_DEFAULTS.HARVESTERS.keepCompleted },
98
+ removeOnFail: { count: QUEUE_DEFAULTS.HARVESTERS.keepFailed }
99
+ },
100
+ ...options
101
+ });
102
+ }
103
+ /**
104
+ * Creates priority queue for urgent/manual jobs
105
+ * @private
106
+ */
107
+ #createPriorityQueue(connection, options) {
108
+ return new Queue(QUEUE_DEFAULTS.PRIORITY.name, {
109
+ connection,
110
+ defaultJobOptions: {
111
+ priority: QUEUE_DEFAULTS.PRIORITY.priority,
112
+ attempts: QUEUE_DEFAULTS.PRIORITY.attempts,
113
+ removeOnComplete: true,
114
+ removeOnFail: false
115
+ },
116
+ ...options
117
+ });
118
+ }
119
+ /**
120
+ * Creates upload queue for async file processing (tileset extraction, large uploads)
121
+ * @private
122
+ */
123
+ #createUploadQueue(connection, options) {
124
+ return new Queue(QUEUE_DEFAULTS.UPLOADS.name, {
125
+ connection,
126
+ defaultJobOptions: {
127
+ attempts: QUEUE_DEFAULTS.UPLOADS.attempts,
128
+ backoff: { type: 'exponential', delay: QUEUE_DEFAULTS.UPLOADS.backoffDelay },
129
+ removeOnComplete: { count: QUEUE_DEFAULTS.UPLOADS.keepCompleted },
130
+ removeOnFail: { count: QUEUE_DEFAULTS.UPLOADS.keepFailed }
131
+ },
132
+ ...options
133
+ });
134
+ }
135
+ /**
136
+ * Closes all queues gracefully
137
+ * @returns Promise that resolves when all queues are closed
138
+ */
139
+ async close() {
140
+ const closePromises = [];
141
+ // Close all queues with timeout protection
142
+ const queues = [this.collectorQueue, this.harvesterQueue, this.priorityQueue, this.uploadQueue];
143
+ for (const queue of queues) {
144
+ closePromises.push(Promise.race([
145
+ queue.close(),
146
+ new Promise((_, reject) => setTimeout(() => reject(new Error('Queue close timeout')), 3000))
147
+ ]).catch(async () => {
148
+ // Force close if timeout - try to access Redis connection directly
149
+ try {
150
+ const redisConnection = queue.redisConnection;
151
+ if (redisConnection && typeof redisConnection.disconnect === 'function') {
152
+ await redisConnection.disconnect();
153
+ }
154
+ }
155
+ catch {
156
+ // Ignore forced cleanup errors
157
+ }
158
+ }));
159
+ }
160
+ await Promise.all(closePromises);
161
+ // Wait for connections to fully close
162
+ await new Promise(resolve => setTimeout(resolve, 300));
163
+ }
164
+ /**
165
+ * Gets statistics for all queues
166
+ * @returns Object containing stats for each queue type
167
+ */
168
+ async getQueueStats() {
169
+ const [collectorStats, harvesterStats, priorityStats, uploadStats] = await Promise.all([
170
+ this.#getStats(this.collectorQueue),
171
+ this.#getStats(this.harvesterQueue),
172
+ this.#getStats(this.priorityQueue),
173
+ this.#getStats(this.uploadQueue)
174
+ ]);
175
+ return {
176
+ collectors: collectorStats,
177
+ harvesters: harvesterStats,
178
+ priority: priorityStats,
179
+ uploads: uploadStats
180
+ };
181
+ }
182
+ /**
183
+ * Gets statistics for a specific queue
184
+ * @private
185
+ */
186
+ async #getStats(queue) {
187
+ const [waiting, active, completed, failed] = await Promise.all([
188
+ queue.getWaitingCount(),
189
+ queue.getActiveCount(),
190
+ queue.getCompletedCount(),
191
+ queue.getFailedCount()
192
+ ]);
193
+ return { waiting, active, completed, failed };
194
+ }
195
+ }
196
+ //# sourceMappingURL=queue_manager.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"queue_manager.js","sourceRoot":"","sources":["../../src/engine/queue_manager.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAA;AAqB9B;;GAEG;AACH,MAAM,cAAc,GAAG;IACnB,KAAK,EAAE;QACH,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,IAAI;QACV,oBAAoB,EAAE,IAAI;QAC1B,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,EAAE,IAAI,CAAC;KAC/D;IACD,UAAU,EAAE;QACR,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,CAAC;QACX,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,GAAG;QAClB,UAAU,EAAE,EAAE;KACjB;IACD,UAAU,EAAE;QACR,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,CAAC;QACX,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,EAAE;QACjB,UAAU,EAAE,GAAG;KAClB;IACD,QAAQ,EAAE;QACN,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;KACd;IACD,OAAO,EAAE;QACL,IAAI,EAAE,YAAY;QAClB,QAAQ,EAAE,CAAC,EAAE,wDAAwD;QACrE,YAAY,EAAE,KAAK;QACnB,aAAa,EAAE,EAAE;QACjB,UAAU,EAAE,GAAG;KAClB;CACK,CAAA;AAEV;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,YAAY;IAMrB;;;OAGG;IACH,YAAY,SAAsB,EAAE;QAChC,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,IAAI,cAAc,CAAC,KAAK,CAAA;QAE3D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;QACjG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;QACjG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;QAC7F,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;IAC5F,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,UAA6B,EAAE,OAA+B;QAChF,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,EAAE;YAC7C,UAAU;YACV,iBAAiB,EAAE;gBACf,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,QAAQ;gBAC5C,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,YAAY,EAAE;gBAC/E,gBAAgB,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,aAAa,EAAE;gBACpE,YAAY,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,UAAU,EAAE;aAChE;YACD,GAAG,OAAO;SACb,CAAC,CAAA;IACN,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,UAA6B,EAAE,OAA+B;QAChF,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,EAAE;YAC7C,UAAU;YACV,iBAAiB,EAAE;gBACf,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,QAAQ;gBAC5C,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,YAAY,EAAE;gBAC/E,gBAAgB,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,aAAa,EAAE;gBACpE,YAAY,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,UAAU,EAAE;aAChE;YACD,GAAG,OAAO;SACb,CAAC,CAAA;IACN,CAAC;IAED;;;OAGG;IACH,oBAAoB,CAAC,UAA6B,EAAE,OAA+B;QAC/E,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE;YAC3C,UAAU;YACV,iBAAiB,EAAE;gBACf,QAAQ,EAAE,cAAc,CAAC,QAAQ,CAAC,QAAQ;gBAC1C,QAAQ,EAAE,cAAc,CAAC,QAAQ,CAAC,QAAQ;gBAC1C,gBAAgB,EAAE,IAAI;gBACtB,YAAY,EAAE,KAAK;aACtB;YACD,GAAG,OAAO;SACb,CAAC,CAAA;IACN,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,UAA6B,EAAE,OAA+B;QAC7E,OAAO,IAAI,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE;YAC1C,UAAU;YACV,iBAAiB,EAAE;gBACf,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,QAAQ;gBACzC,OAAO,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC,YAAY,EAAE;gBAC5E,gBAAgB,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC,aAAa,EAAE;gBACjE,YAAY,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE;aAC7D;YACD,GAAG,OAAO;SACb,CAAC,CAAA;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,aAAa,GAAoB,EAAE,CAAA;QAEzC,2CAA2C;QAC3C,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;QAE/F,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,aAAa,CAAC,IAAI,CACd,OAAO,CAAC,IAAI,CAAC;gBACT,KAAK,CAAC,KAAK,EAAE;gBACb,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;aACrG,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;gBAChB,mEAAmE;gBACnE,IAAI,CAAC;oBACD,MAAM,eAAe,GAAI,KAAa,CAAC,eAAe,CAAA;oBACtD,IAAI,eAAe,IAAI,OAAO,eAAe,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;wBACtE,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;oBACtC,CAAC;gBACL,CAAC;gBAAC,MAAM,CAAC;oBACL,+BAA+B;gBACnC,CAAC;YACL,CAAC,CAAC,CACL,CAAA;QACL,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;QAEhC,sCAAsC;QACtC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAA;IAC1D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,CAAC,cAAc,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACnF,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC;YAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC;SACnC,CAAC,CAAA;QAEF,OAAO;YACH,UAAU,EAAE,cAAc;YAC1B,UAAU,EAAE,cAAc;YAC1B,QAAQ,EAAE,aAAa;YACvB,OAAO,EAAE,WAAW;SACvB,CAAA;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,KAAY;QACxB,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,KAAK,CAAC,eAAe,EAAE;YACvB,KAAK,CAAC,cAAc,EAAE;YACtB,KAAK,CAAC,iBAAiB,EAAE;YACzB,KAAK,CAAC,cAAc,EAAE;SACzB,CAAC,CAAA;QAEF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAA;IACjD,CAAC;CACJ"}