@zintrust/workers 0.1.31 → 0.1.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/ClusterLock.js +3 -2
  2. package/dist/DeadLetterQueue.js +3 -2
  3. package/dist/HealthMonitor.js +24 -13
  4. package/dist/Observability.js +8 -0
  5. package/dist/WorkerFactory.d.ts +4 -0
  6. package/dist/WorkerFactory.js +384 -42
  7. package/dist/WorkerInit.js +122 -43
  8. package/dist/WorkerMetrics.js +5 -1
  9. package/dist/WorkerRegistry.js +8 -0
  10. package/dist/WorkerShutdown.d.ts +0 -13
  11. package/dist/WorkerShutdown.js +1 -44
  12. package/dist/build-manifest.json +99 -83
  13. package/dist/config/workerConfig.d.ts +1 -0
  14. package/dist/config/workerConfig.js +7 -1
  15. package/dist/createQueueWorker.js +281 -42
  16. package/dist/dashboard/workers-api.js +8 -1
  17. package/dist/http/WorkerController.js +90 -35
  18. package/dist/http/WorkerMonitoringService.js +29 -2
  19. package/dist/index.d.ts +1 -2
  20. package/dist/index.js +0 -1
  21. package/dist/routes/workers.js +10 -7
  22. package/dist/storage/WorkerStore.d.ts +6 -3
  23. package/dist/storage/WorkerStore.js +16 -0
  24. package/dist/telemetry/api/TelemetryMonitoringService.js +29 -2
  25. package/dist/ui/router/ui.js +58 -29
  26. package/dist/ui/workers/index.html +202 -0
  27. package/dist/ui/workers/main.js +1952 -0
  28. package/dist/ui/workers/styles.css +1350 -0
  29. package/dist/ui/workers/zintrust.svg +30 -0
  30. package/package.json +5 -5
  31. package/src/ClusterLock.ts +13 -7
  32. package/src/ComplianceManager.ts +3 -2
  33. package/src/DeadLetterQueue.ts +6 -4
  34. package/src/HealthMonitor.ts +33 -17
  35. package/src/Observability.ts +11 -0
  36. package/src/WorkerFactory.ts +446 -43
  37. package/src/WorkerInit.ts +167 -48
  38. package/src/WorkerMetrics.ts +14 -8
  39. package/src/WorkerRegistry.ts +11 -0
  40. package/src/WorkerShutdown.ts +1 -69
  41. package/src/config/workerConfig.ts +9 -1
  42. package/src/createQueueWorker.ts +428 -43
  43. package/src/dashboard/workers-api.ts +8 -1
  44. package/src/http/WorkerController.ts +111 -36
  45. package/src/http/WorkerMonitoringService.ts +35 -2
  46. package/src/index.ts +2 -3
  47. package/src/routes/workers.ts +10 -8
  48. package/src/storage/WorkerStore.ts +21 -3
  49. package/src/telemetry/api/TelemetryMonitoringService.ts +35 -2
  50. package/src/types/queue-monitor.d.ts +2 -1
  51. package/src/ui/router/EmbeddedAssets.ts +3 -0
  52. package/src/ui/router/ui.ts +57 -39
  53. package/src/WorkerShutdownDurableObject.ts +0 -64
@@ -7,10 +7,11 @@
7
7
  * - Sets up auto-scaling and health checks
8
8
  * - Ensures graceful startup and shutdown
9
9
  */
10
- import { Env, Logger, workersConfig } from '@zintrust/core';
10
+ import { Env, Logger } from '@zintrust/core';
11
11
  import { ResourceMonitor } from './ResourceMonitor';
12
12
  import { WorkerFactory } from './WorkerFactory';
13
13
  import { WorkerShutdown } from './WorkerShutdown';
14
+ import { keyPrefix } from './config/workerConfig';
14
15
  // ============================================================================
15
16
  // State
16
17
  // ============================================================================
@@ -50,6 +51,19 @@ function initializeResourceMonitoring(enableResourceMonitoring, resourceMonitori
50
51
  }
51
52
  return false;
52
53
  }
54
+ const getPersistenceOverride = (driver) => {
55
+ if (driver === 'redis') {
56
+ return { driver: 'redis', keyPrefix: keyPrefix() };
57
+ }
58
+ if (driver === 'memory') {
59
+ return { driver: 'memory' };
60
+ }
61
+ return {
62
+ driver: 'database',
63
+ connection: Env.get('WORKER_PERSISTENCE_DB_CONNECTION', 'default') ?? 'default',
64
+ table: Env.get('WORKER_PERSISTENCE_TABLE', 'zintrust_workers') ?? 'zintrust_workers',
65
+ };
66
+ };
53
67
  /**
54
68
  * Check if any workers have resource monitoring enabled
55
69
  */
@@ -65,6 +79,106 @@ function shouldStartResourceMonitoring() {
65
79
  return false;
66
80
  }
67
81
  }
82
+ const resolveAutoStartCandidates = (records) => {
83
+ return records.filter((record) => record.activeStatus !== false && record.autoStart === true);
84
+ };
85
+ const resolvePersistenceTargets = () => {
86
+ const configuredDriver = (Env.get('WORKER_PERSISTENCE_DRIVER', 'memory') || '')
87
+ .toLowerCase()
88
+ .trim();
89
+ const targets = configuredDriver === 'database'
90
+ ? [
91
+ { source: 'database', persistenceOverride: getPersistenceOverride('database') },
92
+ { source: 'redis', persistenceOverride: getPersistenceOverride('redis') },
93
+ { source: 'memory', persistenceOverride: { driver: 'memory' } },
94
+ ]
95
+ : [
96
+ { source: 'redis', persistenceOverride: getPersistenceOverride('redis') },
97
+ { source: 'memory', persistenceOverride: { driver: 'memory' } },
98
+ ];
99
+ // Sort so the configured driver comes first (priority)
100
+ return targets.sort((a, b) => {
101
+ const aIsConfigured = a.persistenceOverride.driver === configuredDriver;
102
+ const bIsConfigured = b.persistenceOverride.driver === configuredDriver;
103
+ if (aIsConfigured && !bIsConfigured)
104
+ return -1;
105
+ if (!aIsConfigured && bIsConfigured)
106
+ return 1;
107
+ return 0;
108
+ });
109
+ };
110
+ const collectAutoStartTasks = async () => {
111
+ const targets = resolvePersistenceTargets();
112
+ const tasks = [];
113
+ const seenWorkerNames = new Set();
114
+ for (const target of targets) {
115
+ try {
116
+ // eslint-disable-next-line no-await-in-loop
117
+ const records = await WorkerFactory.listPersistedRecords(target.persistenceOverride);
118
+ const candidates = resolveAutoStartCandidates(records);
119
+ Logger.debug('Auto-start discovery', {
120
+ source: target.source,
121
+ totalRecords: records.length,
122
+ candidateCount: candidates.length,
123
+ });
124
+ for (const record of candidates) {
125
+ if (seenWorkerNames.has(record.name)) {
126
+ Logger.warn(`Worker ${record.name} appears in multiple persistence stores; keeping first discovered source and skipping duplicate from ${target.source}.`);
127
+ continue;
128
+ }
129
+ seenWorkerNames.add(record.name);
130
+ tasks.push({
131
+ ...record,
132
+ persistenceOverride: target.persistenceOverride,
133
+ source: target.source,
134
+ });
135
+ }
136
+ }
137
+ catch (error) {
138
+ const message = error instanceof Error ? error.message : String(error);
139
+ Logger.warn(`Auto-start discovery failed for ${target.source} persistence: ${message}`);
140
+ }
141
+ }
142
+ return tasks;
143
+ };
144
+ const isWorkerTrulyRunning = async (name) => {
145
+ const existing = WorkerFactory.get(name);
146
+ if (!existing)
147
+ return false;
148
+ const workerLike = existing.worker;
149
+ const isRunning = typeof workerLike.isRunning === 'function'
150
+ ? await Promise.resolve(workerLike.isRunning())
151
+ : false;
152
+ const isPaused = typeof workerLike.isPaused === 'function' ? workerLike.isPaused() : false;
153
+ return isRunning && !isPaused;
154
+ };
155
+ const autoStartOneWorker = async (record) => {
156
+ const existing = WorkerFactory.get(record.name);
157
+ if (existing) {
158
+ try {
159
+ if (await isWorkerTrulyRunning(record.name)) {
160
+ return { name: record.name, started: false, skipped: true };
161
+ }
162
+ Logger.warn(`Worker ${record.name} was registered but not truly running. Restarting to recover from stale state.`);
163
+ await WorkerFactory.restart(record.name, record.persistenceOverride);
164
+ return { name: record.name, started: true, skipped: false };
165
+ }
166
+ catch (error) {
167
+ const message = error instanceof Error ? error.message : String(error);
168
+ Logger.warn(`Auto-start recovery failed for worker ${record.name}: ${message}`);
169
+ return { name: record.name, started: false, skipped: false };
170
+ }
171
+ }
172
+ try {
173
+ await WorkerFactory.startFromPersisted(record.name, record.persistenceOverride);
174
+ return { name: record.name, started: true, skipped: false };
175
+ }
176
+ catch (error) {
177
+ const message = error instanceof Error ? error.message : String(error);
178
+ Logger.warn(`Auto-start failed for worker ${record.name}: ${message}`);
179
+ return { name: record.name, started: false, skipped: false };
180
+ }
181
+ };
68
182
  /**
69
183
  * Initialize the worker management system
70
184
  */
@@ -113,54 +227,19 @@ async function initialize(options = {}) {
113
227
  }
114
228
  }
115
229
  async function autoStartPersistedWorkers() {
116
- // Check if auto-start is enabled globally via environment variable
230
+ const envAutoStart = Env.getBool('WORKER_AUTO_START', false);
231
+ const shouldAutoStart = envAutoStart;
117
232
  Logger.debug('Auto-start check', {
118
- envAutoStart: Env.getBool('WORKER_AUTO_START', false),
119
- configAutoStart: workersConfig.defaultWorker?.autoStart,
233
+ envAutoStart,
234
+ shouldAutoStart,
120
235
  });
121
- if (workersConfig.defaultWorker?.autoStart !== true) {
236
+ if (!shouldAutoStart) {
122
237
  Logger.debug('Auto-start disabled - WORKER_AUTO_START is not true');
123
238
  return;
124
239
  }
125
240
  try {
126
- const records = await WorkerFactory.listPersistedRecords();
127
- Logger.debug('Found persisted records', {
128
- count: records.length,
129
- records: records.map((r) => ({ name: r.name, autoStart: r.autoStart })),
130
- });
131
- const candidates = records.filter((record) => {
132
- if (record.activeStatus === false) {
133
- return false;
134
- }
135
- // If autoStart is explicitly true, always include
136
- if (record.autoStart === true) {
137
- return true;
138
- }
139
- // If autoStart is null or undefined and global auto-start is enabled, include
140
- if ((record.autoStart === null || record.autoStart === undefined) &&
141
- workersConfig.defaultWorker?.autoStart === true) {
142
- return true;
143
- }
144
- return false;
145
- });
146
- Logger.debug('Auto-start candidates', {
147
- count: candidates.length,
148
- candidates: candidates.map((c) => c.name),
149
- });
150
- const results = await Promise.all(candidates.map(async (record) => {
151
- if (WorkerFactory.get(record.name)) {
152
- return { name: record.name, started: false, skipped: true };
153
- }
154
- try {
155
- await WorkerFactory.startFromPersisted(record.name);
156
- return { name: record.name, started: true, skipped: false };
157
- }
158
- catch (error) {
159
- const message = error instanceof Error ? error.message : String(error);
160
- Logger.warn(`Auto-start failed for worker ${record.name}: ${message}`);
161
- return { name: record.name, started: false, skipped: false };
162
- }
163
- }));
241
+ const candidates = await collectAutoStartTasks();
242
+ const results = await Promise.all(candidates.map(async (record) => autoStartOneWorker(record)));
164
243
  const startedCount = results.filter((item) => item.started).length;
165
244
  const skippedCount = results.filter((item) => item.skipped).length;
166
245
  Logger.info('Auto-started persisted workers', {
@@ -33,7 +33,11 @@ const getValidClient = async () => {
33
33
  if (!redisClient) {
34
34
  redisClient = createRedisConnection(cachedConfig);
35
35
  }
36
- return redisClient;
36
+ const client = redisClient;
37
+ if (!client) {
38
+ throw ErrorFactory.createConnectionError('Failed to initialize Redis client');
39
+ }
40
+ return client;
37
41
  };
38
42
  /**
39
43
  * Helper: Get Redis key for metrics
@@ -11,6 +11,11 @@ const registrations = new Map();
11
11
  // Cleanup configuration
12
12
  const STOPPED_WORKER_CLEANUP_DELAY = 5 * 60 * 1000; // 5 minutes
13
13
  const cleanupTimers = new Map();
14
+ const isUnrefableTimer = (value) => {
15
+ if (typeof value !== 'object' || value === null)
16
+ return false;
17
+ return 'unref' in value && typeof value.unref === 'function';
18
+ };
14
19
  /**
15
20
  * Helper: Schedule cleanup of stopped worker
16
21
  */
@@ -38,6 +43,9 @@ const scheduleStoppedWorkerCleanup = (name) => {
38
43
  cleanupTimers.delete(name);
39
44
  }
40
45
  }, STOPPED_WORKER_CLEANUP_DELAY);
46
+ if (isUnrefableTimer(timer)) {
47
+ timer.unref();
48
+ }
41
49
  cleanupTimers.set(name, timer);
42
50
  };
43
51
  /**
@@ -24,11 +24,6 @@ interface IShutdownState {
24
24
  startedAt: Date | null;
25
25
  reason: string | null;
26
26
  }
27
- type DurableShutdownState = {
28
- shuttingDown: boolean;
29
- startedAt?: string;
30
- reason?: string;
31
- };
32
27
  /**
33
28
  * Perform graceful shutdown of all worker modules
34
29
  */
@@ -70,13 +65,5 @@ export declare const WorkerShutdown: Readonly<{
70
65
  * Get current shutdown state
71
66
  */
72
67
  getShutdownState: typeof getShutdownState;
73
- /**
74
- * Request shutdown via Durable Object (Workers)
75
- */
76
- requestDurableShutdown: (reason?: string) => Promise<boolean>;
77
- /**
78
- * Read shutdown state from Durable Object (Workers)
79
- */
80
- getDurableShutdownState: () => Promise<DurableShutdownState | null>;
81
68
  }>;
82
69
  export {};
@@ -4,7 +4,7 @@
4
4
  * Centralized graceful shutdown handling for the worker management system.
5
5
  * Coordinates orderly shutdown of all worker modules and the WorkerFactory.
6
6
  */
7
- import { Cloudflare, Logger } from '@zintrust/core';
7
+ import { Logger } from '@zintrust/core';
8
8
  import { WorkerFactory } from './WorkerFactory';
9
9
  // ============================================================================
10
10
  // Implementation
@@ -15,41 +15,6 @@ const state = {
15
15
  startedAt: null,
16
16
  reason: null,
17
17
  };
18
- const getDurableShutdownStub = () => {
19
- const env = Cloudflare.getWorkersEnv();
20
- if (env === null)
21
- return null;
22
- const namespace = env['WORKER_SHUTDOWN'];
23
- if (!namespace ||
24
- typeof namespace.idFromName !== 'function' ||
25
- typeof namespace.get !== 'function') {
26
- return null;
27
- }
28
- const id = namespace.idFromName('zintrust-shutdown');
29
- return namespace.get(id) ?? null;
30
- };
31
- const requestDurableShutdown = async (reason = 'manual') => {
32
- const stub = getDurableShutdownStub();
33
- if (!stub) {
34
- Logger.warn('Worker shutdown Durable Object binding not configured');
35
- return false;
36
- }
37
- const res = await stub.fetch('https://worker-shutdown/shutdown', {
38
- method: 'POST',
39
- headers: { 'content-type': 'application/json' },
40
- body: JSON.stringify({ reason }),
41
- });
42
- return res.ok;
43
- };
44
- const getDurableShutdownState = async () => {
45
- const stub = getDurableShutdownStub();
46
- if (!stub)
47
- return null;
48
- const res = await stub.fetch('https://worker-shutdown/status');
49
- if (!res.ok)
50
- return null;
51
- return (await res.json());
52
- };
53
18
  let shutdownHandlersRegistered = false;
54
19
  const signalHandlers = {};
55
20
  /**
@@ -217,12 +182,4 @@ export const WorkerShutdown = Object.freeze({
217
182
  * Get current shutdown state
218
183
  */
219
184
  getShutdownState,
220
- /**
221
- * Request shutdown via Durable Object (Workers)
222
- */
223
- requestDurableShutdown,
224
- /**
225
- * Read shutdown state from Durable Object (Workers)
226
- */
227
- getDurableShutdownState,
228
185
  });