@zintrust/workers 0.1.31 → 0.1.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ClusterLock.js +3 -2
- package/dist/DeadLetterQueue.js +3 -2
- package/dist/HealthMonitor.js +24 -13
- package/dist/Observability.js +8 -0
- package/dist/WorkerFactory.d.ts +4 -0
- package/dist/WorkerFactory.js +409 -42
- package/dist/WorkerInit.js +122 -43
- package/dist/WorkerMetrics.js +5 -1
- package/dist/WorkerRegistry.js +8 -0
- package/dist/WorkerShutdown.d.ts +0 -13
- package/dist/WorkerShutdown.js +1 -44
- package/dist/build-manifest.json +101 -85
- package/dist/config/workerConfig.d.ts +1 -0
- package/dist/config/workerConfig.js +7 -1
- package/dist/createQueueWorker.js +281 -42
- package/dist/dashboard/workers-api.js +8 -1
- package/dist/http/WorkerController.js +90 -35
- package/dist/http/WorkerMonitoringService.js +29 -2
- package/dist/http/middleware/FeaturesValidator.js +5 -4
- package/dist/index.d.ts +1 -2
- package/dist/index.js +0 -1
- package/dist/routes/workers.js +10 -7
- package/dist/storage/WorkerStore.d.ts +6 -3
- package/dist/storage/WorkerStore.js +16 -0
- package/dist/telemetry/api/TelemetryMonitoringService.js +29 -2
- package/dist/ui/router/ui.js +58 -29
- package/dist/ui/workers/index.html +202 -0
- package/dist/ui/workers/main.js +1952 -0
- package/dist/ui/workers/styles.css +1350 -0
- package/dist/ui/workers/zintrust.svg +30 -0
- package/package.json +5 -5
- package/src/ClusterLock.ts +13 -7
- package/src/ComplianceManager.ts +3 -2
- package/src/DeadLetterQueue.ts +6 -4
- package/src/HealthMonitor.ts +33 -17
- package/src/Observability.ts +11 -0
- package/src/WorkerFactory.ts +480 -43
- package/src/WorkerInit.ts +167 -48
- package/src/WorkerMetrics.ts +14 -8
- package/src/WorkerRegistry.ts +11 -0
- package/src/WorkerShutdown.ts +1 -69
- package/src/config/workerConfig.ts +9 -1
- package/src/createQueueWorker.ts +428 -43
- package/src/dashboard/workers-api.ts +8 -1
- package/src/http/WorkerController.ts +111 -36
- package/src/http/WorkerMonitoringService.ts +35 -2
- package/src/http/middleware/FeaturesValidator.ts +8 -19
- package/src/index.ts +2 -3
- package/src/routes/workers.ts +10 -8
- package/src/storage/WorkerStore.ts +21 -3
- package/src/telemetry/api/TelemetryMonitoringService.ts +35 -2
- package/src/types/queue-monitor.d.ts +2 -1
- package/src/ui/components/WorkerExpandPanel.js +0 -8
- package/src/ui/router/EmbeddedAssets.ts +3 -0
- package/src/ui/router/ui.ts +57 -39
- package/src/WorkerShutdownDurableObject.ts +0 -64
package/dist/WorkerInit.js
CHANGED
|
@@ -7,10 +7,11 @@
|
|
|
7
7
|
* - Sets up auto-scaling and health checks
|
|
8
8
|
* - Ensures graceful startup and shutdown
|
|
9
9
|
*/
|
|
10
|
-
import { Env, Logger
|
|
10
|
+
import { Env, Logger } from '@zintrust/core';
|
|
11
11
|
import { ResourceMonitor } from './ResourceMonitor';
|
|
12
12
|
import { WorkerFactory } from './WorkerFactory';
|
|
13
13
|
import { WorkerShutdown } from './WorkerShutdown';
|
|
14
|
+
import { keyPrefix } from './config/workerConfig';
|
|
14
15
|
// ============================================================================
|
|
15
16
|
// State
|
|
16
17
|
// ============================================================================
|
|
@@ -50,6 +51,19 @@ function initializeResourceMonitoring(enableResourceMonitoring, resourceMonitori
|
|
|
50
51
|
}
|
|
51
52
|
return false;
|
|
52
53
|
}
|
|
54
|
+
const getPersistenceOverride = (driver) => {
|
|
55
|
+
if (driver === 'redis') {
|
|
56
|
+
return { driver: 'redis', keyPrefix: keyPrefix() };
|
|
57
|
+
}
|
|
58
|
+
if (driver === 'memory') {
|
|
59
|
+
return { driver: 'memory' };
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
driver: 'database',
|
|
63
|
+
connection: Env.get('WORKER_PERSISTENCE_DB_CONNECTION', 'default') ?? 'default',
|
|
64
|
+
table: Env.get('WORKER_PERSISTENCE_TABLE', 'zintrust_workers') ?? 'zintrust_workers',
|
|
65
|
+
};
|
|
66
|
+
};
|
|
53
67
|
/**
|
|
54
68
|
* Check if any workers have resource monitoring enabled
|
|
55
69
|
*/
|
|
@@ -65,6 +79,106 @@ function shouldStartResourceMonitoring() {
|
|
|
65
79
|
return false;
|
|
66
80
|
}
|
|
67
81
|
}
|
|
82
|
+
const resolveAutoStartCandidates = (records) => {
|
|
83
|
+
return records.filter((record) => record.activeStatus !== false && record.autoStart === true);
|
|
84
|
+
};
|
|
85
|
+
const resolvePersistenceTargets = () => {
|
|
86
|
+
const configuredDriver = (Env.get('WORKER_PERSISTENCE_DRIVER', 'memory') || '')
|
|
87
|
+
.toLowerCase()
|
|
88
|
+
.trim();
|
|
89
|
+
const targets = configuredDriver === 'database'
|
|
90
|
+
? [
|
|
91
|
+
{ source: 'database', persistenceOverride: getPersistenceOverride('database') },
|
|
92
|
+
{ source: 'redis', persistenceOverride: getPersistenceOverride('redis') },
|
|
93
|
+
{ source: 'memory', persistenceOverride: { driver: 'memory' } },
|
|
94
|
+
]
|
|
95
|
+
: [
|
|
96
|
+
{ source: 'redis', persistenceOverride: getPersistenceOverride('redis') },
|
|
97
|
+
{ source: 'memory', persistenceOverride: { driver: 'memory' } },
|
|
98
|
+
];
|
|
99
|
+
// Sort so the configured driver comes first (priority)
|
|
100
|
+
return targets.sort((a, b) => {
|
|
101
|
+
const aIsConfigured = a.persistenceOverride.driver === configuredDriver;
|
|
102
|
+
const bIsConfigured = b.persistenceOverride.driver === configuredDriver;
|
|
103
|
+
if (aIsConfigured && !bIsConfigured)
|
|
104
|
+
return -1;
|
|
105
|
+
if (!aIsConfigured && bIsConfigured)
|
|
106
|
+
return 1;
|
|
107
|
+
return 0;
|
|
108
|
+
});
|
|
109
|
+
};
|
|
110
|
+
const collectAutoStartTasks = async () => {
|
|
111
|
+
const targets = resolvePersistenceTargets();
|
|
112
|
+
const tasks = [];
|
|
113
|
+
const seenWorkerNames = new Set();
|
|
114
|
+
for (const target of targets) {
|
|
115
|
+
try {
|
|
116
|
+
// eslint-disable-next-line no-await-in-loop
|
|
117
|
+
const records = await WorkerFactory.listPersistedRecords(target.persistenceOverride);
|
|
118
|
+
const candidates = resolveAutoStartCandidates(records);
|
|
119
|
+
Logger.debug('Auto-start discovery', {
|
|
120
|
+
source: target.source,
|
|
121
|
+
totalRecords: records.length,
|
|
122
|
+
candidateCount: candidates.length,
|
|
123
|
+
});
|
|
124
|
+
for (const record of candidates) {
|
|
125
|
+
if (seenWorkerNames.has(record.name)) {
|
|
126
|
+
Logger.warn(`Worker ${record.name} appears in multiple persistence stores; keeping first discovered source and skipping duplicate from ${target.source}.`);
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
seenWorkerNames.add(record.name);
|
|
130
|
+
tasks.push({
|
|
131
|
+
...record,
|
|
132
|
+
persistenceOverride: target.persistenceOverride,
|
|
133
|
+
source: target.source,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch (error) {
|
|
138
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
139
|
+
Logger.warn(`Auto-start discovery failed for ${target.source} persistence: ${message}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return tasks;
|
|
143
|
+
};
|
|
144
|
+
const isWorkerTrulyRunning = async (name) => {
|
|
145
|
+
const existing = WorkerFactory.get(name);
|
|
146
|
+
if (!existing)
|
|
147
|
+
return false;
|
|
148
|
+
const workerLike = existing.worker;
|
|
149
|
+
const isRunning = typeof workerLike.isRunning === 'function'
|
|
150
|
+
? await Promise.resolve(workerLike.isRunning())
|
|
151
|
+
: false;
|
|
152
|
+
const isPaused = typeof workerLike.isPaused === 'function' ? workerLike.isPaused() : false;
|
|
153
|
+
return isRunning && !isPaused;
|
|
154
|
+
};
|
|
155
|
+
const autoStartOneWorker = async (record) => {
|
|
156
|
+
const existing = WorkerFactory.get(record.name);
|
|
157
|
+
if (existing) {
|
|
158
|
+
try {
|
|
159
|
+
if (await isWorkerTrulyRunning(record.name)) {
|
|
160
|
+
return { name: record.name, started: false, skipped: true };
|
|
161
|
+
}
|
|
162
|
+
Logger.warn(`Worker ${record.name} was registered but not truly running. Restarting to recover from stale state.`);
|
|
163
|
+
await WorkerFactory.restart(record.name, record.persistenceOverride);
|
|
164
|
+
return { name: record.name, started: true, skipped: false };
|
|
165
|
+
}
|
|
166
|
+
catch (error) {
|
|
167
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
168
|
+
Logger.warn(`Auto-start recovery failed for worker ${record.name}: ${message}`);
|
|
169
|
+
return { name: record.name, started: false, skipped: false };
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
await WorkerFactory.startFromPersisted(record.name, record.persistenceOverride);
|
|
174
|
+
return { name: record.name, started: true, skipped: false };
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
178
|
+
Logger.warn(`Auto-start failed for worker ${record.name}: ${message}`);
|
|
179
|
+
return { name: record.name, started: false, skipped: false };
|
|
180
|
+
}
|
|
181
|
+
};
|
|
68
182
|
/**
|
|
69
183
|
* Initialize the worker management system
|
|
70
184
|
*/
|
|
@@ -113,54 +227,19 @@ async function initialize(options = {}) {
|
|
|
113
227
|
}
|
|
114
228
|
}
|
|
115
229
|
async function autoStartPersistedWorkers() {
|
|
116
|
-
|
|
230
|
+
const envAutoStart = Env.getBool('WORKER_AUTO_START', false);
|
|
231
|
+
const shouldAutoStart = envAutoStart;
|
|
117
232
|
Logger.debug('Auto-start check', {
|
|
118
|
-
envAutoStart
|
|
119
|
-
|
|
233
|
+
envAutoStart,
|
|
234
|
+
shouldAutoStart,
|
|
120
235
|
});
|
|
121
|
-
if (
|
|
236
|
+
if (!shouldAutoStart) {
|
|
122
237
|
Logger.debug('Auto-start disabled - WORKER_AUTO_START is not true');
|
|
123
238
|
return;
|
|
124
239
|
}
|
|
125
240
|
try {
|
|
126
|
-
const
|
|
127
|
-
|
|
128
|
-
count: records.length,
|
|
129
|
-
records: records.map((r) => ({ name: r.name, autoStart: r.autoStart })),
|
|
130
|
-
});
|
|
131
|
-
const candidates = records.filter((record) => {
|
|
132
|
-
if (record.activeStatus === false) {
|
|
133
|
-
return false;
|
|
134
|
-
}
|
|
135
|
-
// If autoStart is explicitly true, always include
|
|
136
|
-
if (record.autoStart === true) {
|
|
137
|
-
return true;
|
|
138
|
-
}
|
|
139
|
-
// If autoStart is null or undefined and global auto-start is enabled, include
|
|
140
|
-
if ((record.autoStart === null || record.autoStart === undefined) &&
|
|
141
|
-
workersConfig.defaultWorker?.autoStart === true) {
|
|
142
|
-
return true;
|
|
143
|
-
}
|
|
144
|
-
return false;
|
|
145
|
-
});
|
|
146
|
-
Logger.debug('Auto-start candidates', {
|
|
147
|
-
count: candidates.length,
|
|
148
|
-
candidates: candidates.map((c) => c.name),
|
|
149
|
-
});
|
|
150
|
-
const results = await Promise.all(candidates.map(async (record) => {
|
|
151
|
-
if (WorkerFactory.get(record.name)) {
|
|
152
|
-
return { name: record.name, started: false, skipped: true };
|
|
153
|
-
}
|
|
154
|
-
try {
|
|
155
|
-
await WorkerFactory.startFromPersisted(record.name);
|
|
156
|
-
return { name: record.name, started: true, skipped: false };
|
|
157
|
-
}
|
|
158
|
-
catch (error) {
|
|
159
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
160
|
-
Logger.warn(`Auto-start failed for worker ${record.name}: ${message}`);
|
|
161
|
-
return { name: record.name, started: false, skipped: false };
|
|
162
|
-
}
|
|
163
|
-
}));
|
|
241
|
+
const candidates = await collectAutoStartTasks();
|
|
242
|
+
const results = await Promise.all(candidates.map(async (record) => autoStartOneWorker(record)));
|
|
164
243
|
const startedCount = results.filter((item) => item.started).length;
|
|
165
244
|
const skippedCount = results.filter((item) => item.skipped).length;
|
|
166
245
|
Logger.info('Auto-started persisted workers', {
|
package/dist/WorkerMetrics.js
CHANGED
|
@@ -33,7 +33,11 @@ const getValidClient = async () => {
|
|
|
33
33
|
if (!redisClient) {
|
|
34
34
|
redisClient = createRedisConnection(cachedConfig);
|
|
35
35
|
}
|
|
36
|
-
|
|
36
|
+
const client = redisClient;
|
|
37
|
+
if (!client) {
|
|
38
|
+
throw ErrorFactory.createConnectionError('Failed to initialize Redis client');
|
|
39
|
+
}
|
|
40
|
+
return client;
|
|
37
41
|
};
|
|
38
42
|
/**
|
|
39
43
|
* Helper: Get Redis key for metrics
|
package/dist/WorkerRegistry.js
CHANGED
|
@@ -11,6 +11,11 @@ const registrations = new Map();
|
|
|
11
11
|
// Cleanup configuration
|
|
12
12
|
const STOPPED_WORKER_CLEANUP_DELAY = 5 * 60 * 1000; // 5 minutes
|
|
13
13
|
const cleanupTimers = new Map();
|
|
14
|
+
const isUnrefableTimer = (value) => {
|
|
15
|
+
if (typeof value !== 'object' || value === null)
|
|
16
|
+
return false;
|
|
17
|
+
return 'unref' in value && typeof value.unref === 'function';
|
|
18
|
+
};
|
|
14
19
|
/**
|
|
15
20
|
* Helper: Schedule cleanup of stopped worker
|
|
16
21
|
*/
|
|
@@ -38,6 +43,9 @@ const scheduleStoppedWorkerCleanup = (name) => {
|
|
|
38
43
|
cleanupTimers.delete(name);
|
|
39
44
|
}
|
|
40
45
|
}, STOPPED_WORKER_CLEANUP_DELAY);
|
|
46
|
+
if (isUnrefableTimer(timer)) {
|
|
47
|
+
timer.unref();
|
|
48
|
+
}
|
|
41
49
|
cleanupTimers.set(name, timer);
|
|
42
50
|
};
|
|
43
51
|
/**
|
package/dist/WorkerShutdown.d.ts
CHANGED
|
@@ -24,11 +24,6 @@ interface IShutdownState {
|
|
|
24
24
|
startedAt: Date | null;
|
|
25
25
|
reason: string | null;
|
|
26
26
|
}
|
|
27
|
-
type DurableShutdownState = {
|
|
28
|
-
shuttingDown: boolean;
|
|
29
|
-
startedAt?: string;
|
|
30
|
-
reason?: string;
|
|
31
|
-
};
|
|
32
27
|
/**
|
|
33
28
|
* Perform graceful shutdown of all worker modules
|
|
34
29
|
*/
|
|
@@ -70,13 +65,5 @@ export declare const WorkerShutdown: Readonly<{
|
|
|
70
65
|
* Get current shutdown state
|
|
71
66
|
*/
|
|
72
67
|
getShutdownState: typeof getShutdownState;
|
|
73
|
-
/**
|
|
74
|
-
* Request shutdown via Durable Object (Workers)
|
|
75
|
-
*/
|
|
76
|
-
requestDurableShutdown: (reason?: string) => Promise<boolean>;
|
|
77
|
-
/**
|
|
78
|
-
* Read shutdown state from Durable Object (Workers)
|
|
79
|
-
*/
|
|
80
|
-
getDurableShutdownState: () => Promise<DurableShutdownState | null>;
|
|
81
68
|
}>;
|
|
82
69
|
export {};
|
package/dist/WorkerShutdown.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Centralized graceful shutdown handling for the worker management system.
|
|
5
5
|
* Coordinates orderly shutdown of all worker modules and the WorkerFactory.
|
|
6
6
|
*/
|
|
7
|
-
import {
|
|
7
|
+
import { Logger } from '@zintrust/core';
|
|
8
8
|
import { WorkerFactory } from './WorkerFactory';
|
|
9
9
|
// ============================================================================
|
|
10
10
|
// Implementation
|
|
@@ -15,41 +15,6 @@ const state = {
|
|
|
15
15
|
startedAt: null,
|
|
16
16
|
reason: null,
|
|
17
17
|
};
|
|
18
|
-
const getDurableShutdownStub = () => {
|
|
19
|
-
const env = Cloudflare.getWorkersEnv();
|
|
20
|
-
if (env === null)
|
|
21
|
-
return null;
|
|
22
|
-
const namespace = env['WORKER_SHUTDOWN'];
|
|
23
|
-
if (!namespace ||
|
|
24
|
-
typeof namespace.idFromName !== 'function' ||
|
|
25
|
-
typeof namespace.get !== 'function') {
|
|
26
|
-
return null;
|
|
27
|
-
}
|
|
28
|
-
const id = namespace.idFromName('zintrust-shutdown');
|
|
29
|
-
return namespace.get(id) ?? null;
|
|
30
|
-
};
|
|
31
|
-
const requestDurableShutdown = async (reason = 'manual') => {
|
|
32
|
-
const stub = getDurableShutdownStub();
|
|
33
|
-
if (!stub) {
|
|
34
|
-
Logger.warn('Worker shutdown Durable Object binding not configured');
|
|
35
|
-
return false;
|
|
36
|
-
}
|
|
37
|
-
const res = await stub.fetch('https://worker-shutdown/shutdown', {
|
|
38
|
-
method: 'POST',
|
|
39
|
-
headers: { 'content-type': 'application/json' },
|
|
40
|
-
body: JSON.stringify({ reason }),
|
|
41
|
-
});
|
|
42
|
-
return res.ok;
|
|
43
|
-
};
|
|
44
|
-
const getDurableShutdownState = async () => {
|
|
45
|
-
const stub = getDurableShutdownStub();
|
|
46
|
-
if (!stub)
|
|
47
|
-
return null;
|
|
48
|
-
const res = await stub.fetch('https://worker-shutdown/status');
|
|
49
|
-
if (!res.ok)
|
|
50
|
-
return null;
|
|
51
|
-
return (await res.json());
|
|
52
|
-
};
|
|
53
18
|
let shutdownHandlersRegistered = false;
|
|
54
19
|
const signalHandlers = {};
|
|
55
20
|
/**
|
|
@@ -217,12 +182,4 @@ export const WorkerShutdown = Object.freeze({
|
|
|
217
182
|
* Get current shutdown state
|
|
218
183
|
*/
|
|
219
184
|
getShutdownState,
|
|
220
|
-
/**
|
|
221
|
-
* Request shutdown via Durable Object (Workers)
|
|
222
|
-
*/
|
|
223
|
-
requestDurableShutdown,
|
|
224
|
-
/**
|
|
225
|
-
* Read shutdown state from Durable Object (Workers)
|
|
226
|
-
*/
|
|
227
|
-
getDurableShutdownState,
|
|
228
185
|
});
|