atomic-queues 1.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +686 -0
  2. package/dist/decorators/decorators.d.ts +67 -0
  3. package/dist/decorators/decorators.d.ts.map +1 -0
  4. package/dist/decorators/decorators.js +91 -0
  5. package/dist/decorators/decorators.js.map +1 -0
  6. package/dist/decorators/index.d.ts +2 -0
  7. package/dist/decorators/index.d.ts.map +1 -0
  8. package/dist/decorators/index.js +18 -0
  9. package/dist/decorators/index.js.map +1 -0
  10. package/dist/domain/index.d.ts +5 -0
  11. package/dist/domain/index.d.ts.map +1 -0
  12. package/dist/domain/index.js +21 -0
  13. package/dist/domain/index.js.map +1 -0
  14. package/dist/domain/interfaces.d.ts +614 -0
  15. package/dist/domain/interfaces.d.ts.map +1 -0
  16. package/dist/domain/interfaces.js +19 -0
  17. package/dist/domain/interfaces.js.map +1 -0
  18. package/dist/index.d.ts +40 -0
  19. package/dist/index.d.ts.map +1 -0
  20. package/dist/index.js +61 -0
  21. package/dist/index.js.map +1 -0
  22. package/dist/module/atomic-queues.module.d.ts +97 -0
  23. package/dist/module/atomic-queues.module.d.ts.map +1 -0
  24. package/dist/module/atomic-queues.module.js +197 -0
  25. package/dist/module/atomic-queues.module.js.map +1 -0
  26. package/dist/module/index.d.ts +2 -0
  27. package/dist/module/index.d.ts.map +1 -0
  28. package/dist/module/index.js +18 -0
  29. package/dist/module/index.js.map +1 -0
  30. package/dist/services/constants.d.ts +10 -0
  31. package/dist/services/constants.d.ts.map +1 -0
  32. package/dist/services/constants.js +13 -0
  33. package/dist/services/constants.js.map +1 -0
  34. package/dist/services/cron-manager/cron-manager.service.d.ts +188 -0
  35. package/dist/services/cron-manager/cron-manager.service.d.ts.map +1 -0
  36. package/dist/services/cron-manager/cron-manager.service.js +534 -0
  37. package/dist/services/cron-manager/cron-manager.service.js.map +1 -0
  38. package/dist/services/cron-manager/index.d.ts +2 -0
  39. package/dist/services/cron-manager/index.d.ts.map +1 -0
  40. package/dist/services/cron-manager/index.js +18 -0
  41. package/dist/services/cron-manager/index.js.map +1 -0
  42. package/dist/services/index-manager/index-manager.service.d.ts +146 -0
  43. package/dist/services/index-manager/index-manager.service.d.ts.map +1 -0
  44. package/dist/services/index-manager/index-manager.service.js +337 -0
  45. package/dist/services/index-manager/index-manager.service.js.map +1 -0
  46. package/dist/services/index-manager/index.d.ts +2 -0
  47. package/dist/services/index-manager/index.d.ts.map +1 -0
  48. package/dist/services/index-manager/index.js +18 -0
  49. package/dist/services/index-manager/index.js.map +1 -0
  50. package/dist/services/index.d.ts +10 -0
  51. package/dist/services/index.d.ts.map +1 -0
  52. package/dist/services/index.js +26 -0
  53. package/dist/services/index.js.map +1 -0
  54. package/dist/services/job-processor/index.d.ts +2 -0
  55. package/dist/services/job-processor/index.d.ts.map +1 -0
  56. package/dist/services/job-processor/index.js +18 -0
  57. package/dist/services/job-processor/index.js.map +1 -0
  58. package/dist/services/job-processor/job-processor.service.d.ts +156 -0
  59. package/dist/services/job-processor/job-processor.service.d.ts.map +1 -0
  60. package/dist/services/job-processor/job-processor.service.js +331 -0
  61. package/dist/services/job-processor/job-processor.service.js.map +1 -0
  62. package/dist/services/queue-manager/index.d.ts +2 -0
  63. package/dist/services/queue-manager/index.d.ts.map +1 -0
  64. package/dist/services/queue-manager/index.js +18 -0
  65. package/dist/services/queue-manager/index.js.map +1 -0
  66. package/dist/services/queue-manager/queue-manager.service.d.ts +128 -0
  67. package/dist/services/queue-manager/queue-manager.service.d.ts.map +1 -0
  68. package/dist/services/queue-manager/queue-manager.service.js +308 -0
  69. package/dist/services/queue-manager/queue-manager.service.js.map +1 -0
  70. package/dist/services/resource-lock/index.d.ts +2 -0
  71. package/dist/services/resource-lock/index.d.ts.map +1 -0
  72. package/dist/services/resource-lock/index.js +18 -0
  73. package/dist/services/resource-lock/index.js.map +1 -0
  74. package/dist/services/resource-lock/resource-lock.service.d.ts +124 -0
  75. package/dist/services/resource-lock/resource-lock.service.d.ts.map +1 -0
  76. package/dist/services/resource-lock/resource-lock.service.js +379 -0
  77. package/dist/services/resource-lock/resource-lock.service.js.map +1 -0
  78. package/dist/services/service-queue/index.d.ts +2 -0
  79. package/dist/services/service-queue/index.d.ts.map +1 -0
  80. package/dist/services/service-queue/index.js +18 -0
  81. package/dist/services/service-queue/index.js.map +1 -0
  82. package/dist/services/service-queue/service-queue.service.d.ts +232 -0
  83. package/dist/services/service-queue/service-queue.service.d.ts.map +1 -0
  84. package/dist/services/service-queue/service-queue.service.js +647 -0
  85. package/dist/services/service-queue/service-queue.service.js.map +1 -0
  86. package/dist/services/shutdown-state/index.d.ts +2 -0
  87. package/dist/services/shutdown-state/index.d.ts.map +1 -0
  88. package/dist/services/shutdown-state/index.js +18 -0
  89. package/dist/services/shutdown-state/index.js.map +1 -0
  90. package/dist/services/shutdown-state/shutdown-state.service.d.ts +69 -0
  91. package/dist/services/shutdown-state/shutdown-state.service.d.ts.map +1 -0
  92. package/dist/services/shutdown-state/shutdown-state.service.js +127 -0
  93. package/dist/services/shutdown-state/shutdown-state.service.js.map +1 -0
  94. package/dist/services/worker-manager/index.d.ts +2 -0
  95. package/dist/services/worker-manager/index.d.ts.map +1 -0
  96. package/dist/services/worker-manager/index.js +18 -0
  97. package/dist/services/worker-manager/index.js.map +1 -0
  98. package/dist/services/worker-manager/worker-manager.service.d.ts +163 -0
  99. package/dist/services/worker-manager/worker-manager.service.d.ts.map +1 -0
  100. package/dist/services/worker-manager/worker-manager.service.js +460 -0
  101. package/dist/services/worker-manager/worker-manager.service.js.map +1 -0
  102. package/dist/utils/helpers.d.ts +124 -0
  103. package/dist/utils/helpers.d.ts.map +1 -0
  104. package/dist/utils/helpers.js +229 -0
  105. package/dist/utils/helpers.js.map +1 -0
  106. package/dist/utils/index.d.ts +2 -0
  107. package/dist/utils/index.d.ts.map +1 -0
  108. package/dist/utils/index.js +18 -0
  109. package/dist/utils/index.js.map +1 -0
  110. package/package.json +80 -0
@@ -0,0 +1,647 @@
1
+ "use strict";
2
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
3
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
4
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
5
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
6
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
7
+ };
8
+ var __metadata = (this && this.__metadata) || function (k, v) {
9
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10
+ };
11
+ var __param = (this && this.__param) || function (paramIndex, decorator) {
12
+ return function (target, key) { decorator(target, key, paramIndex); }
13
+ };
14
+ var __importDefault = (this && this.__importDefault) || function (mod) {
15
+ return (mod && mod.__esModule) ? mod : { "default": mod };
16
+ };
17
+ var ServiceQueueManager_1;
18
+ Object.defineProperty(exports, "__esModule", { value: true });
19
+ exports.ServiceQueueManager = exports.ServiceQueueJobNames = void 0;
20
+ const common_1 = require("@nestjs/common");
21
+ const bullmq_1 = require("bullmq");
22
+ const ioredis_1 = __importDefault(require("ioredis"));
23
+ const uuid_1 = require("uuid");
24
+ const constants_1 = require("../constants");
25
+ /**
26
+ * Service-level job names for global atomic operations.
27
+ * These operations MUST be processed by exactly ONE worker across the entire distributed system.
28
+ */
29
+ var ServiceQueueJobNames;
30
+ (function (ServiceQueueJobNames) {
31
+ /** Get the count of all workers across all nodes */
32
+ ServiceQueueJobNames["GET_GLOBAL_WORKER_COUNT"] = "get-global-worker-count";
33
+ /** Get workers for a specific entity across all nodes */
34
+ ServiceQueueJobNames["GET_ENTITY_WORKERS"] = "get-entity-workers";
35
+ /** Verify ownership of a resource */
36
+ ServiceQueueJobNames["VERIFY_OWNERSHIP"] = "verify-ownership";
37
+ /** Acquire global lock */
38
+ ServiceQueueJobNames["ACQUIRE_GLOBAL_LOCK"] = "acquire-global-lock";
39
+ /** Release global lock */
40
+ ServiceQueueJobNames["RELEASE_GLOBAL_LOCK"] = "release-global-lock";
41
+ /** Run scaling cycle for CronManager - triggers worker spawn/terminate decisions */
42
+ ServiceQueueJobNames["RUN_SCALING_CYCLE"] = "run-scaling-cycle";
43
+ /** Spawn a worker for a specific entity - used when opening a table/entity */
44
+ ServiceQueueJobNames["SPAWN_ENTITY_WORKER"] = "spawn-entity-worker";
45
+ /** Custom service operation */
46
+ ServiceQueueJobNames["CUSTOM"] = "custom";
47
+ })(ServiceQueueJobNames || (exports.ServiceQueueJobNames = ServiceQueueJobNames = {}));
48
+ /**
49
+ * ServiceQueueManager
50
+ *
51
+ * Manages the global service-level queue for operations that MUST be atomic
52
+ * across the entire distributed system. Unlike per-entity queues that can have
53
+ * one worker per entity, the service queue has EXACTLY ONE worker globally.
54
+ *
55
+ * Use cases:
56
+ * - Querying global worker counts (can't race with worker creation/deletion)
57
+ * - Ownership verification for resources
58
+ * - Global state mutations
59
+ * - Cross-node coordination
60
+ *
61
+ * Architecture:
62
+ * - Uses a distributed lock to ensure only ONE service worker exists globally
63
+ * - The worker can run on ANY node
64
+ * - If the worker dies, another node will acquire the lock and spawn it
65
+ * - All operations go through the single queue for serialization
66
+ *
67
+ * @example
68
+ * ```typescript
69
+ * // Execute a global atomic operation
70
+ * const workerCount = await serviceQueue.executeServiceOperation(
71
+ * ServiceQueueJobNames.GET_GLOBAL_WORKER_COUNT,
72
+ * { entityType: 'table' },
73
+ * );
74
+ * ```
75
+ */
76
+ let ServiceQueueManager = ServiceQueueManager_1 = class ServiceQueueManager {
77
+ constructor(redis, config) {
78
+ this.redis = redis;
79
+ this.config = config;
80
+ this.logger = new common_1.Logger(ServiceQueueManager_1.name);
81
+ this.lockTTL = 10; // 10 seconds lock TTL
82
+ this.lockRenewalInterval = 3000; // Renew every 3 seconds
83
+ this.serviceQueue = null;
84
+ this.serviceWorker = null;
85
+ this.lockRenewalTimer = null;
86
+ this.subscriberClient = null;
87
+ this.hasLock = false;
88
+ // Pending operation callbacks (for request-response pattern)
89
+ this.pendingOperations = new Map();
90
+ // Custom processors registered by the consuming application
91
+ this.customProcessors = new Map();
92
+ // Scaling cycle handler (registered by CronManager)
93
+ this.scalingCycleHandler = null;
94
+ this.keyPrefix = config.keyPrefix || 'aq';
95
+ this.serviceQueueName =
96
+ config.serviceQueue?.queueName || `${this.keyPrefix}-service-queue`;
97
+ this.serviceWorkerName =
98
+ config.serviceQueue?.workerName || `${this.keyPrefix}-service-worker`;
99
+ this.lockKey = `${this.keyPrefix}:service-worker-lock`;
100
+ this.nodeId = (0, uuid_1.v4)();
101
+ }
102
+ /**
103
+ * Initialize on module start.
104
+ * Attempts to acquire the global service worker lock.
105
+ */
106
+ async onModuleInit() {
107
+ if (this.config.serviceQueue?.enabled === false) {
108
+ this.logger.log('Service queue is disabled by configuration');
109
+ return;
110
+ }
111
+ // Create the queue (all nodes need access to add jobs)
112
+ this.serviceQueue = new bullmq_1.Queue(this.serviceQueueName, {
113
+ connection: this.redis.duplicate(),
114
+ defaultJobOptions: {
115
+ removeOnComplete: true,
116
+ removeOnFail: 100,
117
+ },
118
+ });
119
+ // Set up subscriber for response channel
120
+ this.subscriberClient = this.redis.duplicate();
121
+ // Try to acquire the lock and become the service worker
122
+ await this.tryAcquireServiceWorkerLock();
123
+ // Start periodic lock acquisition attempts (in case current holder dies)
124
+ this.startLockAcquisitionLoop();
125
+ this.logger.log(`ServiceQueueManager initialized on node ${this.nodeId}`);
126
+ }
127
+ /**
128
+ * Cleanup on shutdown.
129
+ */
130
+ async onApplicationShutdown() {
131
+ // Stop lock renewal
132
+ if (this.lockRenewalTimer) {
133
+ clearInterval(this.lockRenewalTimer);
134
+ this.lockRenewalTimer = null;
135
+ }
136
+ // Close worker if we own it
137
+ if (this.serviceWorker && this.hasLock) {
138
+ await this.serviceWorker.close();
139
+ this.serviceWorker = null;
140
+ }
141
+ // Release lock
142
+ if (this.hasLock) {
143
+ await this.releaseLock();
144
+ }
145
+ // Close queue
146
+ if (this.serviceQueue) {
147
+ await this.serviceQueue.close();
148
+ this.serviceQueue = null;
149
+ }
150
+ // Close subscriber
151
+ if (this.subscriberClient) {
152
+ await this.subscriberClient.quit();
153
+ this.subscriberClient = null;
154
+ }
155
+ // Reject all pending operations
156
+ for (const [uuid, pending] of this.pendingOperations) {
157
+ clearTimeout(pending.timeout);
158
+ pending.reject(new Error('Service queue shutting down'));
159
+ this.pendingOperations.delete(uuid);
160
+ }
161
+ this.logger.log('ServiceQueueManager shut down');
162
+ }
163
+ // =========================================================================
164
+ // PUBLIC API
165
+ // =========================================================================
166
+ /**
167
+ * Execute a service-level operation atomically.
168
+ * This queues the operation to the service queue and waits for the result.
169
+ *
170
+ * @param jobName The type of operation to execute
171
+ * @param payload The operation payload
172
+ * @param timeoutMs Timeout in milliseconds (default: 30000)
173
+ * @returns The operation result
174
+ */
175
+ async executeServiceOperation(jobName, payload, timeoutMs = 30000) {
176
+ if (!this.serviceQueue) {
177
+ throw new Error('Service queue not initialized');
178
+ }
179
+ const uuid = (0, uuid_1.v4)();
180
+ const responseChannel = `${this.keyPrefix}:service-response:${uuid}`;
181
+ const jobData = {
182
+ uuid,
183
+ jobName,
184
+ payload,
185
+ responseChannel,
186
+ };
187
+ // Set up response listener before adding job
188
+ const resultPromise = this.waitForResponse(uuid, responseChannel, timeoutMs);
189
+ // Add job to service queue
190
+ await this.serviceQueue.add(jobName, jobData);
191
+ return resultPromise;
192
+ }
193
+ /**
194
+ * Queue a service operation without waiting for result (fire-and-forget).
195
+ */
196
+ async queueServiceOperation(jobName, payload) {
197
+ if (!this.serviceQueue) {
198
+ throw new Error('Service queue not initialized');
199
+ }
200
+ const uuid = (0, uuid_1.v4)();
201
+ const jobData = {
202
+ uuid,
203
+ jobName,
204
+ payload,
205
+ };
206
+ await this.serviceQueue.add(jobName, jobData);
207
+ return uuid;
208
+ }
209
+ /**
210
+ * Register a custom processor for service-level operations.
211
+ * This allows the consuming application to add custom atomic operations.
212
+ */
213
+ registerCustomProcessor(name, processor) {
214
+ this.customProcessors.set(name, processor);
215
+ this.logger.debug(`Registered custom service processor: ${name}`);
216
+ }
217
+ /**
218
+ * Check if this node is the service worker owner.
219
+ */
220
+ isServiceWorkerOwner() {
221
+ return this.hasLock;
222
+ }
223
+ /**
224
+ * Register the scaling cycle handler (called by CronManager).
225
+ * This allows CronManager to register its internal scaling logic to be
226
+ * executed atomically by the service worker.
227
+ */
228
+ registerScalingCycleHandler(handler) {
229
+ this.scalingCycleHandler = handler;
230
+ this.logger.debug('Scaling cycle handler registered');
231
+ }
232
+ /**
233
+ * Register the spawn worker handler (called by CronManager or TableWorkerScalingService).
234
+ * This allows directly spawning a worker for a specific entity without waiting
235
+ * for the next scaling cycle.
236
+ */
237
+ registerSpawnWorkerHandler(handler) {
238
+ this.spawnWorkerHandler = handler;
239
+ this.logger.debug('Spawn worker handler registered');
240
+ }
241
+ /**
242
+ * Trigger a scaling cycle for an entity type.
243
+ * This queues the job to the service queue - only the service worker will execute it.
244
+ */
245
+ async triggerScalingCycle(entityType) {
246
+ if (!this.serviceQueue) {
247
+ this.logger.warn('Service queue not initialized, cannot trigger scaling cycle');
248
+ return;
249
+ }
250
+ // Fire-and-forget - we don't need to wait for the result
251
+ await this.queueServiceOperation(ServiceQueueJobNames.RUN_SCALING_CYCLE, {
252
+ entityType,
253
+ });
254
+ }
255
+ /**
256
+ * Request spawning a worker for a specific entity.
257
+ * This is used when an entity (e.g., table) is opened and needs a worker immediately,
258
+ * without waiting for the next scaling cycle.
259
+ *
260
+ * The job is processed by the service worker to ensure atomic operation.
261
+ *
262
+ * @param entityType The type of entity (e.g., 'table')
263
+ * @param entityId The ID of the entity (e.g., tableId)
264
+ */
265
+ async requestSpawnEntityWorker(entityType, entityId) {
266
+ if (!this.serviceQueue) {
267
+ this.logger.warn('Service queue not initialized, cannot spawn entity worker');
268
+ return;
269
+ }
270
+ this.logger.log(`Requesting worker spawn for ${entityType}/${entityId}`);
271
+ // Fire-and-forget - we don't need to wait for the result
272
+ await this.queueServiceOperation(ServiceQueueJobNames.SPAWN_ENTITY_WORKER, {
273
+ entityType,
274
+ entityId,
275
+ });
276
+ }
277
+ /**
278
+ * Get the service queue name.
279
+ */
280
+ getQueueName() {
281
+ return this.serviceQueueName;
282
+ }
283
+ /**
284
+ * Get the service worker name.
285
+ */
286
+ getWorkerName() {
287
+ return this.serviceWorkerName;
288
+ }
289
+ /**
290
+ * Get pending job count in the service queue.
291
+ */
292
+ async getQueueDepth() {
293
+ if (!this.serviceQueue)
294
+ return 0;
295
+ const counts = await this.serviceQueue.getJobCounts();
296
+ return counts.waiting + counts.active + counts.delayed;
297
+ }
298
+ // =========================================================================
299
+ // PRIVATE METHODS
300
+ // =========================================================================
301
+ /**
302
+ * Try to acquire the global service worker lock.
303
+ */
304
+ async tryAcquireServiceWorkerLock() {
305
+ // Use SET NX EX for atomic lock acquisition
306
+ const result = await this.redis.set(this.lockKey, this.nodeId, 'EX', this.lockTTL, 'NX');
307
+ if (result === 'OK') {
308
+ this.hasLock = true;
309
+ this.logger.log(`Node ${this.nodeId} acquired service worker lock`);
310
+ // Start the service worker
311
+ await this.startServiceWorker();
312
+ // Start lock renewal
313
+ this.startLockRenewal();
314
+ return true;
315
+ }
316
+ // Log who holds the lock and its TTL
317
+ const currentOwner = await this.redis.get(this.lockKey);
318
+ const ttl = await this.redis.ttl(this.lockKey);
319
+ this.logger.debug(`Service worker lock held by node ${currentOwner} (TTL: ${ttl}s). ` +
320
+ `This node ${this.nodeId} will retry after lock expires.`);
321
+ return false;
322
+ }
323
+ /**
324
+ * Release the service worker lock.
325
+ */
326
+ async releaseLock() {
327
+ // Only release if we own the lock
328
+ const currentOwner = await this.redis.get(this.lockKey);
329
+ if (currentOwner === this.nodeId) {
330
+ await this.redis.del(this.lockKey);
331
+ this.hasLock = false;
332
+ this.logger.log(`Node ${this.nodeId} released service worker lock`);
333
+ }
334
+ }
335
+ /**
336
+ * Start periodic lock renewal.
337
+ */
338
+ startLockRenewal() {
339
+ this.lockRenewalTimer = setInterval(async () => {
340
+ if (!this.hasLock)
341
+ return;
342
+ try {
343
+ // Only extend if we still own the lock
344
+ const currentOwner = await this.redis.get(this.lockKey);
345
+ if (currentOwner === this.nodeId) {
346
+ await this.redis.expire(this.lockKey, this.lockTTL);
347
+ }
348
+ else {
349
+ // We lost the lock somehow
350
+ this.hasLock = false;
351
+ this.logger.warn(`Node ${this.nodeId} lost service worker lock unexpectedly`);
352
+ await this.stopServiceWorker();
353
+ }
354
+ }
355
+ catch (error) {
356
+ this.logger.error(`Error renewing service worker lock: ${error}`);
357
+ }
358
+ }, this.lockRenewalInterval);
359
+ }
360
+ /**
361
+ * Start periodic lock acquisition attempts.
362
+ * Retries more frequently initially, then backs off to lockTTL interval.
363
+ */
364
+ startLockAcquisitionLoop() {
365
+ let retryCount = 0;
366
+ const maxFastRetries = 5;
367
+ const fastRetryInterval = 2000; // 2 seconds initially
368
+ const normalRetryInterval = (this.lockTTL + 1) * 1000;
369
+ const tryAcquire = async () => {
370
+ if (this.hasLock)
371
+ return;
372
+ // Check if lock exists
373
+ const lockExists = await this.redis.exists(this.lockKey);
374
+ if (!lockExists) {
375
+ const acquired = await this.tryAcquireServiceWorkerLock();
376
+ if (acquired)
377
+ return;
378
+ }
379
+ // Schedule next attempt
380
+ retryCount++;
381
+ const interval = retryCount <= maxFastRetries ? fastRetryInterval : normalRetryInterval;
382
+ setTimeout(tryAcquire, interval);
383
+ };
384
+ // Start the acquisition loop after a short delay
385
+ setTimeout(tryAcquire, fastRetryInterval);
386
+ }
387
+ /**
388
+ * Start the service worker.
389
+ */
390
+ async startServiceWorker() {
391
+ if (this.serviceWorker) {
392
+ await this.serviceWorker.close();
393
+ }
394
+ this.serviceWorker = new bullmq_1.Worker(this.serviceQueueName, async (job) => {
395
+ return this.processServiceJob(job);
396
+ }, {
397
+ connection: this.redis.duplicate(),
398
+ concurrency: 1, // MUST be 1 for atomic operations
399
+ });
400
+ this.serviceWorker.on('ready', () => {
401
+ this.logger.log(`Service worker ready on node ${this.nodeId}`);
402
+ });
403
+ this.serviceWorker.on('error', (error) => {
404
+ this.logger.error(`Service worker error: ${error.message}`);
405
+ });
406
+ }
407
+ /**
408
+ * Stop the service worker.
409
+ */
410
+ async stopServiceWorker() {
411
+ if (this.serviceWorker) {
412
+ await this.serviceWorker.close();
413
+ this.serviceWorker = null;
414
+ }
415
+ }
416
+ /**
417
+ * Process a service queue job.
418
+ */
419
+ async processServiceJob(job) {
420
+ const { uuid, jobName, payload, responseChannel } = job.data;
421
+ this.logger.debug(`Processing service job ${uuid}: ${jobName}`);
422
+ let result;
423
+ let error = null;
424
+ try {
425
+ switch (jobName) {
426
+ case ServiceQueueJobNames.GET_GLOBAL_WORKER_COUNT:
427
+ result = await this.handleGetGlobalWorkerCount(payload);
428
+ break;
429
+ case ServiceQueueJobNames.GET_ENTITY_WORKERS:
430
+ result = await this.handleGetEntityWorkers(payload);
431
+ break;
432
+ case ServiceQueueJobNames.VERIFY_OWNERSHIP:
433
+ result = await this.handleVerifyOwnership(payload);
434
+ break;
435
+ case ServiceQueueJobNames.ACQUIRE_GLOBAL_LOCK:
436
+ result = await this.handleAcquireGlobalLock(payload);
437
+ break;
438
+ case ServiceQueueJobNames.RELEASE_GLOBAL_LOCK:
439
+ result = await this.handleReleaseGlobalLock(payload);
440
+ break;
441
+ case ServiceQueueJobNames.RUN_SCALING_CYCLE:
442
+ result = await this.handleRunScalingCycle(payload);
443
+ break;
444
+ case ServiceQueueJobNames.SPAWN_ENTITY_WORKER:
445
+ result = await this.handleSpawnEntityWorker(payload);
446
+ break;
447
+ case ServiceQueueJobNames.CUSTOM:
448
+ result = await this.handleCustomOperation(payload);
449
+ break;
450
+ default:
451
+ throw new Error(`Unknown service job name: ${jobName}`);
452
+ }
453
+ }
454
+ catch (err) {
455
+ error = err;
456
+ this.logger.error(`Service job ${uuid} failed: ${error.message}`);
457
+ }
458
+ // Send response if channel specified
459
+ if (responseChannel) {
460
+ await this.redis.publish(responseChannel, JSON.stringify({
461
+ uuid,
462
+ success: !error,
463
+ result,
464
+ error: error?.message,
465
+ }));
466
+ }
467
+ if (error)
468
+ throw error;
469
+ return result;
470
+ }
471
+ /**
472
+ * Wait for a response on a channel.
473
+ */
474
+ async waitForResponse(uuid, channel, timeoutMs) {
475
+ return new Promise((resolve, reject) => {
476
+ const timeout = setTimeout(() => {
477
+ this.pendingOperations.delete(uuid);
478
+ this.subscriberClient?.unsubscribe(channel).catch(() => { });
479
+ reject(new Error(`Service operation timed out after ${timeoutMs}ms`));
480
+ }, timeoutMs);
481
+ this.pendingOperations.set(uuid, {
482
+ resolve: resolve,
483
+ reject,
484
+ timeout
485
+ });
486
+ // Subscribe to response channel
487
+ this.subscriberClient?.subscribe(channel).then(() => {
488
+ const handler = (ch, message) => {
489
+ if (ch === channel) {
490
+ const response = JSON.parse(message);
491
+ if (response.uuid === uuid) {
492
+ clearTimeout(timeout);
493
+ this.pendingOperations.delete(uuid);
494
+ this.subscriberClient?.unsubscribe(channel).catch(() => { });
495
+ this.subscriberClient?.off('message', handler);
496
+ if (response.success) {
497
+ resolve(response.result);
498
+ }
499
+ else {
500
+ reject(new Error(response.error || 'Service operation failed'));
501
+ }
502
+ }
503
+ }
504
+ };
505
+ this.subscriberClient?.on('message', handler);
506
+ });
507
+ });
508
+ }
509
+ // =========================================================================
510
+ // BUILT-IN SERVICE HANDLERS
511
+ // =========================================================================
512
+ /**
513
+ * Get global worker count across all nodes.
514
+ */
515
+ async handleGetGlobalWorkerCount(payload) {
516
+ const { entityType } = payload;
517
+ const pattern = entityType
518
+ ? `${this.keyPrefix}:worker:*:${entityType}-*`
519
+ : `${this.keyPrefix}:worker:*:*`;
520
+ const keys = await this.scanKeys(pattern);
521
+ return keys.length;
522
+ }
523
+ /**
524
+ * Get workers for a specific entity.
525
+ * Uses the worker heartbeat TTL keys as the single source of truth.
526
+ */
527
+ async handleGetEntityWorkers(payload) {
528
+ const { entityType, entityId } = payload;
529
+ // Worker heartbeat keys follow pattern: {prefix}:worker:{nodeId}:{workerName}
530
+ // Worker names follow pattern: {entityId}-worker
531
+ const workerName = `${entityId}-worker`;
532
+ const pattern = `${this.keyPrefix}:worker:*:${workerName}`;
533
+ const keys = await this.scanKeys(pattern);
534
+ return keys.map((key) => key.split(':').pop());
535
+ }
536
+ /**
537
+ * Verify ownership of a resource.
538
+ */
539
+ async handleVerifyOwnership(payload) {
540
+ const { resourceType, resourceId, expectedOwner } = payload;
541
+ const key = `${this.keyPrefix}:lock:${resourceType}:${resourceId}`;
542
+ const owner = await this.redis.get(key);
543
+ return {
544
+ owned: owner === expectedOwner,
545
+ owner: owner || undefined,
546
+ };
547
+ }
548
+ /**
549
+ * Acquire a global lock atomically.
550
+ */
551
+ async handleAcquireGlobalLock(payload) {
552
+ const { lockName, ownerId, ttlSeconds = 30 } = payload;
553
+ const key = `${this.keyPrefix}:global-lock:${lockName}`;
554
+ const result = await this.redis.set(key, ownerId, 'EX', ttlSeconds, 'NX');
555
+ return { acquired: result === 'OK' };
556
+ }
557
+ /**
558
+ * Release a global lock atomically.
559
+ */
560
+ async handleReleaseGlobalLock(payload) {
561
+ const { lockName, ownerId } = payload;
562
+ const key = `${this.keyPrefix}:global-lock:${lockName}`;
563
+ // Only release if we own it (using Lua for atomicity)
564
+ const script = `
565
+ if redis.call("get", KEYS[1]) == ARGV[1] then
566
+ return redis.call("del", KEYS[1])
567
+ else
568
+ return 0
569
+ end
570
+ `;
571
+ const result = await this.redis.eval(script, 1, key, ownerId);
572
+ return { released: result === 1 };
573
+ }
574
+ /**
575
+ * Handle custom operation by delegating to registered processor.
576
+ */
577
+ async handleCustomOperation(payload) {
578
+ const { processorName, data } = payload;
579
+ const processor = this.customProcessors.get(processorName);
580
+ if (!processor) {
581
+ throw new Error(`No custom processor registered for: ${processorName}`);
582
+ }
583
+ return processor(data);
584
+ }
585
+ /**
586
+ * Handle scaling cycle request by delegating to registered CronManager handler.
587
+ */
588
+ async handleRunScalingCycle(payload) {
589
+ const { entityType } = payload;
590
+ if (!this.scalingCycleHandler) {
591
+ this.logger.warn(`No scaling cycle handler registered for entity type: ${entityType}`);
592
+ return { processed: false, reason: 'no_handler' };
593
+ }
594
+ try {
595
+ const result = await this.scalingCycleHandler(entityType);
596
+ return { processed: true, result };
597
+ }
598
+ catch (error) {
599
+ this.logger.error(`Scaling cycle failed for ${entityType}: ${error.message}`);
600
+ return { processed: false, error: error.message };
601
+ }
602
+ }
603
+ /**
604
+ * Handle spawn entity worker request.
605
+ * This directly spawns a worker for the specific entity, bypassing the scaling cycle.
606
+ * Used when opening a table/entity that needs a worker immediately.
607
+ */
608
+ async handleSpawnEntityWorker(payload) {
609
+ const { entityType, entityId } = payload;
610
+ this.logger.log(`Processing spawn worker request for ${entityType}/${entityId}`);
611
+ if (!this.spawnWorkerHandler) {
612
+ this.logger.warn(`No spawn worker handler registered, cannot spawn worker for ${entityType}/${entityId}`);
613
+ return { spawned: false, reason: 'no_spawn_handler' };
614
+ }
615
+ try {
616
+ // Directly spawn the worker for this specific entity
617
+ await this.spawnWorkerHandler(entityType, entityId);
618
+ this.logger.log(`Worker spawned for ${entityType}/${entityId}`);
619
+ return { spawned: true, entityType, entityId };
620
+ }
621
+ catch (error) {
622
+ this.logger.error(`Failed to spawn worker for ${entityType}/${entityId}: ${error.message}`);
623
+ return { spawned: false, error: error.message };
624
+ }
625
+ }
626
+ /**
627
+ * Scan Redis keys matching a pattern.
628
+ */
629
+ async scanKeys(pattern) {
630
+ let cursor = '0';
631
+ const keys = [];
632
+ do {
633
+ const [nextCursor, scanKeys] = await this.redis.scan(cursor, 'MATCH', pattern, 'COUNT', 100);
634
+ cursor = nextCursor;
635
+ keys.push(...scanKeys);
636
+ } while (cursor !== '0');
637
+ return keys;
638
+ }
639
+ };
640
+ exports.ServiceQueueManager = ServiceQueueManager;
641
+ exports.ServiceQueueManager = ServiceQueueManager = ServiceQueueManager_1 = __decorate([
642
+ (0, common_1.Injectable)(),
643
+ __param(0, (0, common_1.Inject)(constants_1.ATOMIC_QUEUES_REDIS)),
644
+ __param(1, (0, common_1.Inject)(constants_1.ATOMIC_QUEUES_CONFIG)),
645
+ __metadata("design:paramtypes", [ioredis_1.default, Object])
646
+ ], ServiceQueueManager);
647
+ //# sourceMappingURL=service-queue.service.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"service-queue.service.js","sourceRoot":"","sources":["../../../src/services/service-queue/service-queue.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,2CAMwB;AACxB,mCAA4C;AAC5C,sDAA4B;AAC5B,+BAAoC;AAEpC,4CAAyE;AAEzE;;;GAGG;AACH,IAAY,oBAiBX;AAjBD,WAAY,oBAAoB;IAC9B,oDAAoD;IACpD,2EAAmD,CAAA;IACnD,yDAAyD;IACzD,iEAAyC,CAAA;IACzC,qCAAqC;IACrC,6DAAqC,CAAA;IACrC,0BAA0B;IAC1B,mEAA2C,CAAA;IAC3C,0BAA0B;IAC1B,mEAA2C,CAAA;IAC3C,oFAAoF;IACpF,+DAAuC,CAAA;IACvC,8EAA8E;IAC9E,mEAA2C,CAAA;IAC3C,+BAA+B;IAC/B,yCAAiB,CAAA;AACnB,CAAC,EAjBW,oBAAoB,oCAApB,oBAAoB,QAiB/B;AAYD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEI,IAAM,mBAAmB,2BAAzB,MAAM,mBAAmB;IAmC9B,YAC+B,KAA6B,EAE1D,MAAkD;QAFJ,UAAK,GAAL,KAAK,CAAO;QAEzC,WAAM,GAAN,MAAM,CAA2B;QArCnC,WAAM,GAAG,IAAI,eAAM,CAAC,qBAAmB,CAAC,IAAI,CAAC,CAAC;QAK9C,YAAO,GAAG,EAAE,CAAC,CAAC,sBAAsB;QACpC,wBAAmB,GAAG,IAAI,CAAC,CAAC,wBAAwB;QAE7D,iBAAY,GAAiB,IAAI,CAAC;QAClC,kBAAa,GAAkB,IAAI,CAAC;QACpC,qBAAgB,GAA0B,IAAI,CAAC;QAC/C,qBAAgB,GAAiB,IAAI,CAAC;QACtC,YAAO,GAAG,KAAK,CAAC;QAGxB,6DAA6D;QAC5C,sBAAiB,GAO9B,IAAI,GAAG,EAAE,CAAC;QAEd,4DAA4D;QAC3C,qBAAgB,GAG7B,IAAI,GAAG,EAAE,CAAC;QAEd,oDAAoD;QAC5C,wBAAmB,GAAsD,IAAI,CAAC;QAOpF,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC;QAC1C,IAAI,CAAC,gBAAgB;YACnB,MAAM,CAAC,YAAY,EAAE,SAAS,IAAI,GAAG,IAAI,CAAC,SAAS,gBAAgB,CAAC;QACtE,IAAI,CAAC,iBAAiB;YACpB,MAAM,CAAC,YAAY,EAAE,UAAU,IAAI,GAAG,IAAI,CAAC,SAAS,iBAAiB,CAAC;QACxE,IAAI,CAAC,OAAO,GAAG,GAAG,IAAI,CAAC,SAAS,sBAAsB,CAAC;QACvD,IAAI,CAAC,MAAM,GAAG,IAAA,SAAM,GAAE,CAAC;IACzB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QAChB,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,OAAO,KAAK,KAAK,EAAE,CAAC;YAChD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YAC9D,OAAO;QACT,CAAC;QAED,uDAAuD;QACvD,IAAI,CAAC,YAAY,GAAG,IAAI,cAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE;YACnD,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;YAClC,iBAAiB,EAAE;gBACjB,gBAAgB,EAAE,IAAI;gBACtB,YAAY,EAAE,GAAG;aAClB;SACF,CAAC,CAAC;QAEH,yCAAyC;QACzC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAE/C,wDAAwD;QACxD,MAAM,IAAI,CAAC,2BAA2B,EAAE,CAAC;QAEzC,yEAAyE;QACzE,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEhC,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,2CAA2C,IAAI,CAAC,MAAM,EAAE,CACzD,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACzB,oBAAoB;QACpB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC/B,CAAC;QAED,4BAA4B;QAC5B,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACvC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,CAAC;QAED,eAAe;QACf,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3B,CAAC;QAED,cAAc;QACd,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;QAED,mBAAmB;QACnB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC/B,CAAC;QAED,gCAAgC;QAChC,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACrD,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAC;YACzD,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IAED,4EAA4E;IAC5E,aAAa;IACb,4EAA4E;IAE5E;;;;;;;;OAQG;IACH,KAAK,CAAC,uBAAuB,CAC3B,OAA6B,EAC7B,OAAU,EACV,SAAS,GAAG,KAAK;QAEjB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,IAAI,GAAG,IAAA,SAAM,GAAE,CAAC;QACtB,MAAM,eAAe,GAAG,GAAG,IAAI,CAAC,SAAS,qBAAqB,IAAI,EAAE,CAAC;QAErE,MAAM,OAAO,GAA4B;YACvC,IAAI;YACJ,OAAO;YACP,OAAO;YACP,eAAe;SAChB,CAAC;QAEF,6CAA6C;QAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAI,IAAI,EAAE,eAAe,EAAE,SAAS,CAAC,CAAC;QAEhF,2BAA2B;QAC3B,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,OAAO,aAAa,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB,CACzB,OAA6B,EAC7B,OAAU;QAEV,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,IAAI,GAAG,IAAA,SAAM,GAAE,CAAC;QAEtB,MAAM,OAAO,GAA4B;YACvC,IAAI;YACJ,OAAO;YACP,OAAO;SACR,CAAC;QAEF,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,uBAAuB,CACrB,IAAY,EACZ,SAAiD;QAEjD,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,oBAAoB;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;;;OAIG;IACH,2BAA2B,CACzB,OAAiD;QAEjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACxD,CAAC;IAOD;;;;OAIG;IACH,0BAA0B,CACxB,OAAgE;QAEhE,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,mBAAmB,CAAC,UAAkB;QAC1C,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;YAChF,OAAO;QACT,CAAC;QAED,yDAAyD;QACzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,oBAAoB,CAAC,iBAAiB,EAAE;YACvE,UAAU;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,wBAAwB,CAAC,UAAkB,EAAE,QAAgB;QACjE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;YAC9E,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,+BAA+B,UAAU,IAAI,QAAQ,EAAE,CAAC,CAAC;QAEzE,yDAAyD;QACzD,MAAM,IAAI,CAAC,qBAAqB,CAAC,oBAAoB,CAAC,mBAAmB,EAAE;YACzE,UAAU;YACV,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACjB,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;QACtD,OAAO,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC;IACzD,CAAC;IAED,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E;;OAEG;IACK,KAAK,CAAC,2BAA2B;QACvC,4CAA4C;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CACjC,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,MAAM,EACX,IAAI,EACJ,IAAI,CAAC,OAAO,EACZ,IAAI,CACL,CAAC;QAEF,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACpB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,QAAQ,IAAI,CAAC,MAAM,+BAA+B,CACnD,CAAC;YAEF,2BAA2B;YAC3B,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAEhC,qBAAqB;YACrB,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAExB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qCAAqC;QACrC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,oCAAoC,YAAY,UAAU,GAAG,MAAM;YACnE,aAAa,IAAI,CAAC,MAAM,iCAAiC,CAC1D,CAAC;QAEF,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW;QACvB,kCAAkC;QAClC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxD,IAAI,YAAY,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACnC,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,MAAM,+BAA+B,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,gBAAgB;QACtB,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YAC7C,IAAI,CAAC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAE1B,IAAI,CAAC;gBACH,uCAAuC;gBACvC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACxD,IAAI,YAAY,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;oBACjC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBACtD,CAAC;qBAAM,CAAC;oBACN,2BAA2B;oBAC3B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;oBACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,QAAQ,IAAI,CAAC,MAAM,wCAAwC,CAC5D,CAAC;oBACF,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACjC,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,KAAK,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACK,wBAAwB;QAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,cAAc,GAAG,CAAC,CAAC;QACzB,MAAM,iBAAiB,GAAG,IAAI,CAAC,CAAC,sBAAsB;QACtD,MAAM,mBAAmB,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;QAEtD,MAAM,UAAU,GAAG,KAAK,IAAI,EAAE;YAC5B,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO;YAEzB,uBAAuB;YACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzD,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,2BAA2B,EAAE,CAAC;gBAC1D,IAAI,QAAQ;oBAAE,OAAO;YACvB,CAAC;YAED,wBAAwB;YACxB,UAAU,EAAE,CAAC;YACb,MAAM,QAAQ,GAAG,UAAU,IAAI,cAAc,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,mBAAmB,CAAC;YACxF,UAAU,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACnC,CAAC,CAAC;QAEF,iDAAiD;QACjD,UAAU,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB;QAC9B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QACnC,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,IAAI,eAAM,CAC7B,IAAI,CAAC,gBAAgB,EACrB,KAAK,EAAE,GAA8B,EAAE,EAAE;YACvC,OAAO,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC,EACD;YACE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;YAClC,WAAW,EAAE,CAAC,EAAE,kCAAkC;SACnD,CACF,CAAC;QAEF,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAClC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gCAAgC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YACvC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB;QAC7B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC7B,GAA8B;QAE9B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;QAE7D,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,0BAA0B,IAAI,KAAK,OAAO,EAAE,CAC7C,CAAC;QAEF,IAAI,MAAe,CAAC;QACpB,IAAI,KAAK,GAAiB,IAAI,CAAC;QAE/B,IAAI,CAAC;YACH,QAAQ,OAAO,EAAE,CAAC;gBAChB,KAAK,oBAAoB,CAAC,uBAAuB;oBAC/C,MAAM,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC;oBACxD,MAAM;gBAER,KAAK,oBAAoB,CAAC,kBAAkB;oBAC1C,MAAM,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;oBACpD,MAAM;gBAER,KAAK,oBAAoB,CAAC,gBAAgB;oBACxC,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;oBACnD,MAAM;gBAER,KAAK,oBAAoB,CAAC,mBAAmB;oBAC3C,MAAM,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;oBACrD,MAAM;gBAER,KAAK,oBAAoB,CAAC,mBAAmB;oBAC3C,MAAM,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;oBACrD,MAAM;gBAER,KAAK,oBAAoB,CAAC,iBAAiB;oBACzC,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;oBACnD,MAAM;gBAER,KAAK,oBAAoB,CAAC,mBAAmB;oBAC3C,MAAM,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;oBACrD,MAAM;gBAER,KAAK,oBAAoB,CAAC,MAAM;oBAC9B,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;oBACnD,MAAM;gBAER;oBACE,MAAM,IAAI,KAAK,CAAC,6BAA6B,OAAO,EAAE,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,GAAG,GAAY,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,eAAe,IAAI,YAAY,KAAK,CAAC,OAAO,EAAE,CAC/C,CAAC;QACJ,CAAC;QAED,qCAAqC;QACrC,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CACtB,eAAe,EACf,IAAI,CAAC,SAAS,CAAC;gBACb,IAAI;gBACJ,OAAO,EAAE,CAAC,KAAK;gBACf,MAAM;gBACN,KAAK,EAAE,KAAK,EAAE,OAAO;aACtB,CAAC,CACH,CAAC;QACJ,CAAC;QAED,IAAI,KAAK;YAAE,MAAM,KAAK,CAAC;QACvB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAC3B,IAAY,EACZ,OAAe,EACf,SAAiB;QAEjB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC9B,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACpC,IAAI,CAAC,gBAAgB,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC5D,MAAM,CAAC,IAAI,KAAK,CAAC,qCAAqC,SAAS,IAAI,CAAC,CAAC,CAAC;YACxE,CAAC,EAAE,SAAS,CAAC,CAAC;YAEd,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE;gBAC/B,OAAO,EAAE,OAAoC;gBAC7C,MAAM;gBACN,OAAO;aACR,CAAC,CAAC;YAEH,gCAAgC;YAChC,IAAI,CAAC,gBAAgB,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;gBAClD,MAAM,OAAO,GAAG,CAAC,EAAU,EAAE,OAAe,EAAE,EAAE;oBAC9C,IAAI,EAAE,KAAK,OAAO,EAAE,CAAC;wBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrC,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;4BAC3B,YAAY,CAAC,OAAO,CAAC,CAAC;4BACtB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;4BACpC,IAAI,CAAC,gBAAgB,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;4BAC5D,IAAI,CAAC,gBAAgB,EAAE,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;4BAE/C,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;gCACrB,OAAO,CAAC,QAAQ,CAAC,MAAW,CAAC,CAAC;4BAChC,CAAC;iCAAM,CAAC;gCACN,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,IAAI,0BAA0B,CAAC,CAAC,CAAC;4BAClE,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC,CAAC;gBAEF,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,4BAA4B;IAC5B,4EAA4E;IAE5E;;OAEG;IACK,KAAK,CAAC,0BAA0B,CACtC,OAAgB;QAEhB,MAAM,EAAE,UAAU,EAAE,GAAG,OAAkC,CAAC;QAC1D,MAAM,OAAO,GAAG,UAAU;YACxB,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,aAAa,UAAU,IAAI;YAC9C,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,aAAa,CAAC;QAEnC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,sBAAsB,CAClC,OAAgB;QAEhB,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,OAGhC,CAAC;QACF,8EAA8E;QAC9E,iDAAiD;QACjD,MAAM,UAAU,GAAG,GAAG,QAAQ,SAAS,CAAC;QACxC,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,SAAS,aAAa,UAAU,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CACjC,OAAgB;QAEhB,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,GAAG,OAInD,CAAC;QAEF,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,SAAS,SAAS,YAAY,IAAI,UAAU,EAAE,CAAC;QACnE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAExC,OAAO;YACL,KAAK,EAAE,KAAK,KAAK,aAAa;YAC9B,KAAK,EAAE,KAAK,IAAI,SAAS;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,uBAAuB,CACnC,OAAgB;QAEhB,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,GAAG,EAAE,EAAE,GAAG,OAI9C,CAAC;QAEF,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,SAAS,gBAAgB,QAAQ,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QAE1E,OAAO,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,uBAAuB,CACnC,OAAgB;QAEhB,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,OAG7B,CAAC;QAEF,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,SAAS,gBAAgB,QAAQ,EAAE,CAAC;QAExD,sDAAsD;QACtD,MAAM,MAAM,GAAG;;;;;;KAMd,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,KAAK,CAAC,EAAE,CAAC;IACpC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CACjC,OAAgB;QAEhB,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,OAG/B,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAC3D,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,uCAAuC,aAAa,EAAE,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CACjC,OAAgB;QAEhB,MAAM,EAAE,UAAU,EAAE,GAAG,OAAiC,CAAC;QAEzD,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wDAAwD,UAAU,EAAE,CAAC,CAAC;YACvF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;QACpD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YAC1D,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,UAAU,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;YACzF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC;QAC/D,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,uBAAuB,CACnC,OAAgB;QAEhB,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,OAAmD,CAAC;QAErF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uCAAuC,UAAU,IAAI,QAAQ,EAAE,CAAC,CAAC;QAEjF,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+DAA+D,UAAU,IAAI,QAAQ,EAAE,CAAC,CAAC;YAC1G,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;QACxD,CAAC;QAED,IAAI,CAAC;YACH,qDAAqD;YACrD,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;YACpD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,UAAU,IAAI,QAAQ,EAAE,CAAC,CAAC;YAChE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,UAAU,IAAI,QAAQ,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;YACvG,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC;QAC7D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,QAAQ,CAAC,OAAe;QACpC,IAAI,MAAM,GAAG,GAAG,CAAC;QACjB,MAAM,IAAI,GAAa,EAAE,CAAC;QAE1B,GAAG,CAAC;YACF,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAClD,MAAM,EACN,OAAO,EACP,OAAO,EACP,OAAO,EACP,GAAG,CACJ,CAAC;YACF,MAAM,GAAG,UAAU,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QACzB,CAAC,QAAQ,MAAM,KAAK,GAAG,EAAE;QAEzB,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAA;AA9vBY,kDAAmB;8BAAnB,mBAAmB;IAD/B,IAAA,mBAAU,GAAE;IAqCR,WAAA,IAAA,eAAM,EAAC,+BAAmB,CAAC,CAAA;IAC3B,WAAA,IAAA,eAAM,EAAC,gCAAoB,CAAC,CAAA;qCADwB,iBAAK;GApCjD,mBAAmB,CA8vB/B"}
@@ -0,0 +1,2 @@
1
+ export * from './shutdown-state.service';
2
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/shutdown-state/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAC"}