@zintrust/workers 0.1.29 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -1
- package/dist/AnomalyDetection.d.ts +4 -0
- package/dist/AnomalyDetection.js +8 -0
- package/dist/BroadcastWorker.d.ts +2 -0
- package/dist/CanaryController.js +49 -5
- package/dist/ChaosEngineering.js +13 -0
- package/dist/ClusterLock.js +21 -10
- package/dist/DeadLetterQueue.js +12 -8
- package/dist/MultiQueueWorker.d.ts +1 -1
- package/dist/MultiQueueWorker.js +12 -7
- package/dist/NotificationWorker.d.ts +2 -0
- package/dist/PriorityQueue.d.ts +2 -2
- package/dist/PriorityQueue.js +20 -21
- package/dist/ResourceMonitor.js +65 -38
- package/dist/WorkerFactory.d.ts +23 -3
- package/dist/WorkerFactory.js +420 -40
- package/dist/WorkerInit.js +8 -3
- package/dist/WorkerMetrics.d.ts +2 -1
- package/dist/WorkerMetrics.js +152 -93
- package/dist/WorkerRegistry.d.ts +6 -0
- package/dist/WorkerRegistry.js +70 -1
- package/dist/WorkerShutdown.d.ts +21 -0
- package/dist/WorkerShutdown.js +82 -9
- package/dist/WorkerShutdownDurableObject.d.ts +12 -0
- package/dist/WorkerShutdownDurableObject.js +41 -0
- package/dist/build-manifest.json +171 -99
- package/dist/createQueueWorker.d.ts +2 -0
- package/dist/createQueueWorker.js +42 -27
- package/dist/dashboard/types.d.ts +5 -0
- package/dist/dashboard/workers-api.js +136 -43
- package/dist/http/WorkerApiController.js +1 -0
- package/dist/http/WorkerController.js +133 -85
- package/dist/http/WorkerMonitoringService.d.ts +11 -0
- package/dist/http/WorkerMonitoringService.js +62 -0
- package/dist/http/middleware/CustomValidation.js +1 -1
- package/dist/http/middleware/EditWorkerValidation.d.ts +1 -1
- package/dist/http/middleware/EditWorkerValidation.js +7 -6
- package/dist/http/middleware/ProcessorPathSanitizer.js +101 -35
- package/dist/http/middleware/WorkerValidationChain.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -0
- package/dist/routes/workers.js +48 -6
- package/dist/storage/WorkerStore.d.ts +4 -1
- package/dist/storage/WorkerStore.js +55 -7
- package/dist/telemetry/api/TelemetryAPI.d.ts +46 -0
- package/dist/telemetry/api/TelemetryAPI.js +219 -0
- package/dist/telemetry/api/TelemetryMonitoringService.d.ts +17 -0
- package/dist/telemetry/api/TelemetryMonitoringService.js +113 -0
- package/dist/telemetry/components/AlertPanel.d.ts +1 -0
- package/dist/telemetry/components/AlertPanel.js +13 -0
- package/dist/telemetry/components/CostTracking.d.ts +1 -0
- package/dist/telemetry/components/CostTracking.js +14 -0
- package/dist/telemetry/components/ResourceUsageChart.d.ts +1 -0
- package/dist/telemetry/components/ResourceUsageChart.js +11 -0
- package/dist/telemetry/components/WorkerHealthChart.d.ts +1 -0
- package/dist/telemetry/components/WorkerHealthChart.js +11 -0
- package/dist/telemetry/index.d.ts +15 -0
- package/dist/telemetry/index.js +60 -0
- package/dist/telemetry/routes/dashboard.d.ts +6 -0
- package/dist/telemetry/routes/dashboard.js +608 -0
- package/dist/ui/router/EmbeddedAssets.d.ts +4 -0
- package/dist/ui/router/EmbeddedAssets.js +13 -0
- package/dist/ui/router/ui.js +100 -4
- package/package.json +10 -6
- package/src/AnomalyDetection.ts +9 -0
- package/src/CanaryController.ts +41 -5
- package/src/ChaosEngineering.ts +14 -0
- package/src/ClusterLock.ts +22 -9
- package/src/DeadLetterQueue.ts +13 -8
- package/src/MultiQueueWorker.ts +15 -8
- package/src/PriorityQueue.ts +21 -22
- package/src/ResourceMonitor.ts +72 -40
- package/src/WorkerFactory.ts +545 -49
- package/src/WorkerInit.ts +8 -3
- package/src/WorkerMetrics.ts +183 -105
- package/src/WorkerRegistry.ts +80 -1
- package/src/WorkerShutdown.ts +115 -9
- package/src/WorkerShutdownDurableObject.ts +64 -0
- package/src/createQueueWorker.ts +73 -30
- package/src/dashboard/types.ts +5 -0
- package/src/dashboard/workers-api.ts +165 -52
- package/src/http/WorkerApiController.ts +1 -0
- package/src/http/WorkerController.ts +167 -90
- package/src/http/WorkerMonitoringService.ts +77 -0
- package/src/http/middleware/CustomValidation.ts +1 -1
- package/src/http/middleware/EditWorkerValidation.ts +7 -6
- package/src/http/middleware/ProcessorPathSanitizer.ts +123 -36
- package/src/http/middleware/WorkerValidationChain.ts +1 -0
- package/src/index.ts +6 -1
- package/src/routes/workers.ts +66 -9
- package/src/storage/WorkerStore.ts +59 -9
- package/src/telemetry/api/TelemetryAPI.ts +292 -0
- package/src/telemetry/api/TelemetryMonitoringService.ts +149 -0
- package/src/telemetry/components/AlertPanel.ts +13 -0
- package/src/telemetry/components/CostTracking.ts +14 -0
- package/src/telemetry/components/ResourceUsageChart.ts +11 -0
- package/src/telemetry/components/WorkerHealthChart.ts +11 -0
- package/src/telemetry/index.ts +121 -0
- package/src/telemetry/public/assets/zintrust-logo.svg +15 -0
- package/src/telemetry/routes/dashboard.ts +638 -0
- package/src/telemetry/styles/tailwind.css +1 -0
- package/src/telemetry/styles/zintrust-theme.css +8 -0
- package/src/ui/router/EmbeddedAssets.ts +13 -0
- package/src/ui/router/ui.ts +112 -5
- package/src/ui/workers/index.html +2 -2
- package/src/ui/workers/main.js +232 -61
- package/src/ui/workers/zintrust.svg +30 -0
- package/dist/dashboard/workers-dashboard-ui.d.ts +0 -3
- package/dist/dashboard/workers-dashboard-ui.js +0 -1026
- package/dist/dashboard/workers-dashboard.d.ts +0 -4
- package/dist/dashboard/workers-dashboard.js +0 -904
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { Logger, NodeSingletons, workersConfig } from '@zintrust/core';
|
|
2
|
+
import { HealthMonitor } from '../HealthMonitor';
|
|
3
|
+
import { getWorkers } from '../dashboard/workers-api';
|
|
4
|
+
// Internal state
|
|
5
|
+
const emitter = new NodeSingletons.EventEmitter();
|
|
6
|
+
emitter.setMaxListeners(Infinity);
|
|
7
|
+
let interval = null;
|
|
8
|
+
let subscribers = 0;
|
|
9
|
+
const INTERVAL_MS = workersConfig?.intervalMs || 5000;
|
|
10
|
+
const broadcastSnapshot = async () => {
|
|
11
|
+
try {
|
|
12
|
+
if (subscribers <= 0)
|
|
13
|
+
return;
|
|
14
|
+
const monitoring = await HealthMonitor.getSummary();
|
|
15
|
+
// Fetch full workers listing optimized for dashboard
|
|
16
|
+
const workersPayload = await getWorkers({ page: 1, limit: 200 });
|
|
17
|
+
const payload = {
|
|
18
|
+
type: 'snapshot',
|
|
19
|
+
ts: new Date().toISOString(),
|
|
20
|
+
monitoring,
|
|
21
|
+
workers: workersPayload,
|
|
22
|
+
};
|
|
23
|
+
emitter.emit('snapshot', payload);
|
|
24
|
+
}
|
|
25
|
+
catch (err) {
|
|
26
|
+
Logger.error('WorkerMonitoringService.broadcastSnapshot failed', err);
|
|
27
|
+
emitter.emit('error', err);
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
const startPolling = () => {
|
|
31
|
+
if (interval)
|
|
32
|
+
return;
|
|
33
|
+
Logger.debug('Starting WorkerMonitoringService polling');
|
|
34
|
+
// Initial fetch
|
|
35
|
+
void broadcastSnapshot();
|
|
36
|
+
interval = setInterval(() => {
|
|
37
|
+
void broadcastSnapshot();
|
|
38
|
+
}, INTERVAL_MS);
|
|
39
|
+
};
|
|
40
|
+
const stopPolling = () => {
|
|
41
|
+
if (interval) {
|
|
42
|
+
Logger.debug('Stopping WorkerMonitoringService polling');
|
|
43
|
+
clearInterval(interval);
|
|
44
|
+
interval = null;
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
export const WorkerMonitoringService = Object.freeze({
|
|
48
|
+
subscribe(callback) {
|
|
49
|
+
emitter.on('snapshot', callback);
|
|
50
|
+
subscribers++;
|
|
51
|
+
if (subscribers === 1) {
|
|
52
|
+
startPolling();
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
unsubscribe(callback) {
|
|
56
|
+
emitter.off('snapshot', callback);
|
|
57
|
+
subscribers--;
|
|
58
|
+
if (subscribers <= 0) {
|
|
59
|
+
stopPolling();
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
});
|
|
@@ -2,6 +2,6 @@ import { type IRequest, type IResponse } from '@zintrust/core';
|
|
|
2
2
|
export type RouteHandler = (req: IRequest, res: IResponse) => Promise<void> | void;
|
|
3
3
|
/**
|
|
4
4
|
* Composite middleware for worker edit validation
|
|
5
|
-
* Maps
|
|
5
|
+
* Maps processorSpec to processor for validation and validates all editable fields
|
|
6
6
|
*/
|
|
7
7
|
export declare const withEditWorkerValidation: (handler: RouteHandler) => RouteHandler;
|
|
@@ -10,19 +10,19 @@ import { withVersionValidation } from './VersionSanitizer';
|
|
|
10
10
|
import { withWorkerNameValidation } from './WorkerNameSanitizer';
|
|
11
11
|
/**
|
|
12
12
|
* Composite middleware for worker edit validation
|
|
13
|
-
* Maps
|
|
13
|
+
* Maps processorSpec to processor for validation and validates all editable fields
|
|
14
14
|
*/
|
|
15
15
|
export const withEditWorkerValidation = (handler) => {
|
|
16
16
|
return async (req, res) => {
|
|
17
17
|
try {
|
|
18
18
|
const data = req.data();
|
|
19
19
|
const currentBody = req.getBody();
|
|
20
|
-
// Map
|
|
20
|
+
// Map processorSpec/processorSpec to processor for validation if provided
|
|
21
21
|
let mappedBody = { ...currentBody };
|
|
22
|
-
if (data['
|
|
22
|
+
if (data['processorSpec'] && !data['processor']) {
|
|
23
23
|
mappedBody = {
|
|
24
24
|
...mappedBody,
|
|
25
|
-
processor: data['
|
|
25
|
+
processor: data['processorSpec'],
|
|
26
26
|
};
|
|
27
27
|
}
|
|
28
28
|
// Update the request body with mapped fields
|
|
@@ -31,8 +31,8 @@ export const withEditWorkerValidation = (handler) => {
|
|
|
31
31
|
return withStrictPayloadKeys([
|
|
32
32
|
'name',
|
|
33
33
|
'queueName',
|
|
34
|
-
'processor', // Validated field (mapped from
|
|
35
|
-
'
|
|
34
|
+
'processor', // Validated field (mapped from processorSpec)
|
|
35
|
+
'processorSpec',
|
|
36
36
|
'version',
|
|
37
37
|
'options', // Skip strict validation for editing
|
|
38
38
|
'infrastructure',
|
|
@@ -41,6 +41,7 @@ export const withEditWorkerValidation = (handler) => {
|
|
|
41
41
|
'concurrency', // Original field
|
|
42
42
|
'region',
|
|
43
43
|
'autoStart',
|
|
44
|
+
'activeStatus',
|
|
44
45
|
'status',
|
|
45
46
|
], withProcessorPathValidation(withWorkerNameValidation(withQueueNameValidation(withVersionValidation(withInfrastructureValidation(withFeaturesValidation(withDatacenterValidation(handler))))))))(req, res);
|
|
46
47
|
}
|
|
@@ -1,5 +1,17 @@
|
|
|
1
|
-
import { Logger, NodeSingletons } from '@zintrust/core';
|
|
2
|
-
const PROCESSOR_PATH_PATTERN = /^[a-zA-Z0-9/_.-]+\.(ts|js)$/;
|
|
1
|
+
import { Logger, NodeSingletons, workersConfig, } from '@zintrust/core';
|
|
2
|
+
const PROCESSOR_PATH_PATTERN = /^[a-zA-Z0-9/_.-]+\.(ts|js|mjs|cjs)$/;
|
|
3
|
+
const isUrlSpec = (value) => {
|
|
4
|
+
if (value.startsWith('url:'))
|
|
5
|
+
return true;
|
|
6
|
+
return value.includes('://');
|
|
7
|
+
};
|
|
8
|
+
const normalizeUrlSpec = (value) => {
|
|
9
|
+
return value.startsWith('url:') ? value.slice(4) : value;
|
|
10
|
+
};
|
|
11
|
+
const isAllowedRemoteHost = (host) => {
|
|
12
|
+
const allowlist = workersConfig.processorSpec.remoteAllowlist;
|
|
13
|
+
return allowlist.map((value) => value.toLowerCase()).includes(host.toLowerCase());
|
|
14
|
+
};
|
|
3
15
|
const decodeProcessorPath = (processor) => {
|
|
4
16
|
return processor
|
|
5
17
|
.replaceAll('/', '/') // HTML hex entity for /
|
|
@@ -11,6 +23,70 @@ const decodeProcessorPath = (processor) => {
|
|
|
11
23
|
.replaceAll('-', '-') // HTML hex entity for -
|
|
12
24
|
.replaceAll('%2D', '-'); // URL encoding for -
|
|
13
25
|
};
|
|
26
|
+
const validateUrlSpec = (processor) => {
|
|
27
|
+
const normalized = normalizeUrlSpec(processor);
|
|
28
|
+
let parsed;
|
|
29
|
+
try {
|
|
30
|
+
parsed = new URL(normalized);
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return {
|
|
34
|
+
isValid: false,
|
|
35
|
+
error: { error: 'Invalid processor url', code: 'INVALID_PROCESSOR_URL' },
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
if (parsed.protocol === 'file:') {
|
|
39
|
+
const path = NodeSingletons.path;
|
|
40
|
+
const baseDir = path.resolve(process.cwd());
|
|
41
|
+
const resolved = path.resolve(baseDir, decodeURIComponent(parsed.pathname));
|
|
42
|
+
if (!resolved.startsWith(baseDir)) {
|
|
43
|
+
return {
|
|
44
|
+
isValid: false,
|
|
45
|
+
error: { error: 'Invalid processor path', code: 'INVALID_PROCESSOR_PATH_TRAVERSAL' },
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
if (parsed.protocol !== 'https:') {
|
|
51
|
+
return {
|
|
52
|
+
isValid: false,
|
|
53
|
+
error: { error: 'Invalid processor url', code: 'INVALID_PROCESSOR_URL' },
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
if (!isAllowedRemoteHost(parsed.host)) {
|
|
57
|
+
return {
|
|
58
|
+
isValid: false,
|
|
59
|
+
error: { error: 'Invalid processor url host', code: 'INVALID_PROCESSOR_URL_HOST' },
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return { isValid: true };
|
|
64
|
+
};
|
|
65
|
+
const validateRelativePath = (processor) => {
|
|
66
|
+
if (processor.includes('..') || processor.startsWith('/')) {
|
|
67
|
+
return {
|
|
68
|
+
isValid: false,
|
|
69
|
+
error: { error: 'Invalid processor path', code: 'INVALID_PROCESSOR_PATH' },
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
if (!PROCESSOR_PATH_PATTERN.test(processor)) {
|
|
73
|
+
return {
|
|
74
|
+
isValid: false,
|
|
75
|
+
error: { error: 'Invalid processor path', code: 'INVALID_PROCESSOR_EXTENSION' },
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
return { isValid: true };
|
|
79
|
+
};
|
|
80
|
+
const sanitizeAndResolvePath = (processor) => {
|
|
81
|
+
const sanitizedProcessor = processor.replaceAll(/[^a-zA-Z0-9/_.-]/g, '');
|
|
82
|
+
const path = NodeSingletons.path;
|
|
83
|
+
const baseDir = path.resolve(process.cwd());
|
|
84
|
+
const resolved = path.resolve(baseDir, sanitizedProcessor);
|
|
85
|
+
if (!resolved.startsWith(baseDir)) {
|
|
86
|
+
return { isValid: false, sanitized: processor };
|
|
87
|
+
}
|
|
88
|
+
return { isValid: true, sanitized: sanitizedProcessor };
|
|
89
|
+
};
|
|
14
90
|
export const withProcessorPathValidation = (handler) => {
|
|
15
91
|
return async (req, res) => {
|
|
16
92
|
try {
|
|
@@ -18,49 +94,39 @@ export const withProcessorPathValidation = (handler) => {
|
|
|
18
94
|
let processor = data['processor'];
|
|
19
95
|
if (!processor) {
|
|
20
96
|
return res.setStatus(400).json({
|
|
21
|
-
error: 'Processor
|
|
22
|
-
code: '
|
|
97
|
+
error: 'Processor spec is required',
|
|
98
|
+
code: 'MISSING_PROCESSOR_SPEC',
|
|
23
99
|
});
|
|
24
100
|
}
|
|
25
101
|
// Decode URL-encoded characters
|
|
26
102
|
processor = decodeProcessorPath(processor);
|
|
27
103
|
// Trim whitespace
|
|
28
104
|
processor = processor.trim();
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
message: 'Processor path must be relative and cannot contain path traversal',
|
|
34
|
-
code: 'INVALID_PROCESSOR_PATH',
|
|
35
|
-
});
|
|
105
|
+
const isUrl = isUrlSpec(processor);
|
|
106
|
+
let validation;
|
|
107
|
+
if (isUrl) {
|
|
108
|
+
validation = validateUrlSpec(processor);
|
|
36
109
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
110
|
+
else {
|
|
111
|
+
validation = validateRelativePath(processor);
|
|
112
|
+
if (validation.isValid) {
|
|
113
|
+
const pathValidation = sanitizeAndResolvePath(processor);
|
|
114
|
+
if (pathValidation.isValid) {
|
|
115
|
+
processor = pathValidation.sanitized;
|
|
116
|
+
}
|
|
117
|
+
else {
|
|
118
|
+
validation = {
|
|
119
|
+
isValid: false,
|
|
120
|
+
error: { error: 'Invalid processor path', code: 'INVALID_PROCESSOR_PATH_TRAVERSAL' },
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
}
|
|
48
124
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
const path = NodeSingletons.path;
|
|
52
|
-
// Ensure resolved path stays within repository/app scope
|
|
53
|
-
const BASE_PROCESSOR_DIR = path.resolve(process.cwd());
|
|
54
|
-
const resolved = path.resolve(BASE_PROCESSOR_DIR, sanitizedProcessor);
|
|
55
|
-
if (!resolved.startsWith(BASE_PROCESSOR_DIR)) {
|
|
56
|
-
return res.setStatus(400).json({
|
|
57
|
-
error: 'Invalid processor path',
|
|
58
|
-
message: 'Processor path resolves outside of allowed base directory',
|
|
59
|
-
code: 'INVALID_PROCESSOR_PATH_TRAVERSAL',
|
|
60
|
-
});
|
|
125
|
+
if (!validation.isValid) {
|
|
126
|
+
return res.setStatus(400).json(validation.error);
|
|
61
127
|
}
|
|
62
128
|
const currentBody = req.getBody();
|
|
63
|
-
req.setBody({ ...currentBody, processor
|
|
129
|
+
req.setBody({ ...currentBody, processor });
|
|
64
130
|
return handler(req, res);
|
|
65
131
|
}
|
|
66
132
|
catch (error) {
|
|
@@ -23,6 +23,7 @@ export const withCreateWorkerValidation = (handler) => {
|
|
|
23
23
|
'infrastructure',
|
|
24
24
|
'features',
|
|
25
25
|
'datacenter',
|
|
26
|
+
'activeStatus',
|
|
26
27
|
], withProcessorPathValidation(withWorkerNameValidation(withQueueNameValidation(withVersionValidation(withOptionsValidation(withInfrastructureValidation(withFeaturesValidation(withDatacenterValidation(handler)))))))));
|
|
27
28
|
};
|
|
28
29
|
/**
|
package/dist/index.d.ts
CHANGED
|
@@ -23,9 +23,10 @@ export { DatacenterOrchestrator } from './DatacenterOrchestrator';
|
|
|
23
23
|
export { MultiQueueWorker } from './MultiQueueWorker';
|
|
24
24
|
export { WorkerVersioning } from './WorkerVersioning';
|
|
25
25
|
export { WorkerFactory } from './WorkerFactory';
|
|
26
|
-
export type { WorkerPersistenceConfig } from './WorkerFactory';
|
|
26
|
+
export type { ProcessorResolver, WorkerFactoryConfig, WorkerPersistenceConfig, } from './WorkerFactory';
|
|
27
27
|
export { WorkerInit } from './WorkerInit';
|
|
28
28
|
export { WorkerShutdown } from './WorkerShutdown';
|
|
29
|
+
export { ZinTrustWorkerShutdownDurableObject } from './WorkerShutdownDurableObject';
|
|
29
30
|
export { WorkerController } from './http/WorkerController';
|
|
30
31
|
export { registerWorkerRoutes } from './routes/workers';
|
|
31
32
|
export { BroadcastWorker } from './BroadcastWorker';
|
package/dist/index.js
CHANGED
|
@@ -33,6 +33,7 @@ export { WorkerVersioning } from './WorkerVersioning';
|
|
|
33
33
|
export { WorkerFactory } from './WorkerFactory';
|
|
34
34
|
export { WorkerInit } from './WorkerInit';
|
|
35
35
|
export { WorkerShutdown } from './WorkerShutdown';
|
|
36
|
+
export { ZinTrustWorkerShutdownDurableObject } from './WorkerShutdownDurableObject';
|
|
36
37
|
// HTTP Controllers & Routes
|
|
37
38
|
export { WorkerController } from './http/WorkerController';
|
|
38
39
|
export { registerWorkerRoutes } from './routes/workers';
|
package/dist/routes/workers.js
CHANGED
|
@@ -3,20 +3,24 @@
|
|
|
3
3
|
* HTTP API for managing workers with dashboard functionality
|
|
4
4
|
*/
|
|
5
5
|
import { Logger, Router } from '@zintrust/core';
|
|
6
|
-
import {
|
|
7
|
-
import { WorkerController } from '../http/WorkerController';
|
|
6
|
+
import { HealthMonitor } from '../HealthMonitor';
|
|
8
7
|
import { ValidationSchemas, withCustomValidation } from '../http/middleware/CustomValidation';
|
|
9
8
|
import { withEditWorkerValidation } from '../http/middleware/EditWorkerValidation';
|
|
10
9
|
import { withDriverValidation } from '../http/middleware/ValidateDriver';
|
|
11
10
|
import { withCreateWorkerValidation, withWorkerOperationValidation, } from '../http/middleware/WorkerValidationChain';
|
|
11
|
+
import { WorkerApiController } from '../http/WorkerApiController';
|
|
12
|
+
import { WorkerController } from '../http/WorkerController';
|
|
13
|
+
import { ResourceMonitor } from '../ResourceMonitor';
|
|
14
|
+
import { TelemetryDashboard } from '../telemetry';
|
|
12
15
|
import { registerStaticAssets } from '../ui/router/ui';
|
|
16
|
+
import { WorkerFactory } from '../WorkerFactory';
|
|
13
17
|
const controller = WorkerController.create();
|
|
14
18
|
const apiController = WorkerApiController.create();
|
|
15
19
|
function registerCoreWorkerRoutes(r) {
|
|
16
20
|
// Core worker operations
|
|
17
21
|
Router.post(r, '/create', withCreateWorkerValidation(controller.create));
|
|
18
22
|
Router.put(r, '/:name', withCreateWorkerValidation(controller.update));
|
|
19
|
-
// Worker editing with custom validation that handles
|
|
23
|
+
// Worker editing with custom validation that handles mapping
|
|
20
24
|
Router.put(r, '/:name/edit', withEditWorkerValidation(controller.update));
|
|
21
25
|
Router.post(r, '/:name/start', withDriverValidation(withWorkerOperationValidation(controller.start)));
|
|
22
26
|
Router.post(r, '/:name/auto-start', withDriverValidation(withWorkerOperationValidation(controller.setAutoStart)));
|
|
@@ -40,6 +44,8 @@ function registerWorkerQueryRoutes(r) {
|
|
|
40
44
|
Router.get(r, '/:name/driver-data', withDriverValidation(apiController.getWorkerDriverDataHandler));
|
|
41
45
|
}
|
|
42
46
|
function registerMonitoringRoutes(r) {
|
|
47
|
+
// SSE events stream for monitoring + workers snapshot
|
|
48
|
+
Router.get(r, '/events', controller.eventsStream);
|
|
43
49
|
// Health monitoring
|
|
44
50
|
Router.post(r, '/:name/monitoring/start', controller.startMonitoring);
|
|
45
51
|
Router.post(r, '/:name/monitoring/stop', controller.stopMonitoring);
|
|
@@ -48,8 +54,6 @@ function registerMonitoringRoutes(r) {
|
|
|
48
54
|
Router.put(r, '/:name/monitoring/config', controller.updateMonitoringConfig);
|
|
49
55
|
// SLA monitoring
|
|
50
56
|
Router.get(r, '/:name/sla/status', controller.getSlaStatus);
|
|
51
|
-
// SSE events stream for monitoring + workers snapshot
|
|
52
|
-
Router.get(r, '/events', controller.eventsStream);
|
|
53
57
|
}
|
|
54
58
|
function registerVersioningRoutes(r) {
|
|
55
59
|
// Versioning
|
|
@@ -66,16 +70,54 @@ function registerUtilityRoutes(r) {
|
|
|
66
70
|
function registerWorkerLifecycleRoutes(router, middleware) {
|
|
67
71
|
Router.group(router, '/api/workers', (r) => {
|
|
68
72
|
Logger.info('Registering Worker Management Routes');
|
|
73
|
+
registerMonitoringRoutes(r); // ← Move FIRST - has /events
|
|
69
74
|
registerCoreWorkerRoutes(r);
|
|
70
75
|
registerWorkerQueryRoutes(r);
|
|
71
|
-
registerMonitoringRoutes(r);
|
|
72
76
|
registerVersioningRoutes(r);
|
|
73
77
|
registerUtilityRoutes(r);
|
|
74
78
|
}, { middleware: middleware });
|
|
75
79
|
}
|
|
80
|
+
function registerWorkerTelemetryRoutes(router, middleware) {
|
|
81
|
+
const options = middleware ? { middleware } : undefined;
|
|
82
|
+
Router.group(router, '/api', (r) => {
|
|
83
|
+
Router.get(r, '/workers/system/summary', async (_req, res) => {
|
|
84
|
+
const workers = WorkerFactory.list();
|
|
85
|
+
const monitoringSummary = await HealthMonitor.getSummary();
|
|
86
|
+
const resourceUsage = ResourceMonitor.getCurrentUsage('system');
|
|
87
|
+
res.json({
|
|
88
|
+
ok: true,
|
|
89
|
+
summary: {
|
|
90
|
+
workers: workers.length,
|
|
91
|
+
monitoring: monitoringSummary,
|
|
92
|
+
resources: resourceUsage,
|
|
93
|
+
},
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
Router.get(r, '/workers/system/monitoring/summary', async (_req, res) => {
|
|
97
|
+
const summary = await HealthMonitor.getSummary();
|
|
98
|
+
res.json({ ok: true, summary });
|
|
99
|
+
});
|
|
100
|
+
Router.get(r, '/resources/current', async (_req, res) => {
|
|
101
|
+
const usage = ResourceMonitor.getCurrentUsage('system');
|
|
102
|
+
res.json({ ok: true, usage });
|
|
103
|
+
});
|
|
104
|
+
Router.get(r, '/resources/trends', async (req, res) => {
|
|
105
|
+
const period = (req.getParam('period') ?? 'day');
|
|
106
|
+
const trends = ResourceMonitor.getAllTrends('system', period);
|
|
107
|
+
res.json({ ok: true, trends });
|
|
108
|
+
});
|
|
109
|
+
}, options);
|
|
110
|
+
}
|
|
76
111
|
export function registerWorkerRoutes(router, _options, routeOptions) {
|
|
77
112
|
registerStaticAssets(router, routeOptions?.middleware ?? []);
|
|
78
113
|
registerWorkerLifecycleRoutes(router, routeOptions?.middleware);
|
|
114
|
+
registerWorkerTelemetryRoutes(router, routeOptions?.middleware);
|
|
115
|
+
// Register Telemetry Dashboard
|
|
116
|
+
const dashboard = TelemetryDashboard.create({
|
|
117
|
+
basePath: '/telemetry',
|
|
118
|
+
});
|
|
119
|
+
dashboard.registerRoutes(router);
|
|
79
120
|
Logger.info('Worker routes registered at http://127.0.0.1:7777/workers');
|
|
121
|
+
Logger.info('Telemetry dashboard registered at http://127.0.0.1:7777/telemetry');
|
|
80
122
|
}
|
|
81
123
|
export default registerWorkerRoutes;
|
|
@@ -12,7 +12,8 @@ export type WorkerRecord = {
|
|
|
12
12
|
autoStart: boolean;
|
|
13
13
|
concurrency: number;
|
|
14
14
|
region: string | null;
|
|
15
|
-
|
|
15
|
+
processorSpec?: string | null;
|
|
16
|
+
activeStatus?: boolean;
|
|
16
17
|
features?: Record<string, unknown> | null;
|
|
17
18
|
infrastructure?: Record<string, unknown> | null;
|
|
18
19
|
datacenter?: Record<string, unknown> | null;
|
|
@@ -28,10 +29,12 @@ export type WorkerStore = {
|
|
|
28
29
|
offset?: number;
|
|
29
30
|
limit?: number;
|
|
30
31
|
search?: string;
|
|
32
|
+
includeInactive?: boolean;
|
|
31
33
|
}): Promise<WorkerRecord[]>;
|
|
32
34
|
get(name: string): Promise<WorkerRecord | null>;
|
|
33
35
|
save(record: WorkerRecord): Promise<void>;
|
|
34
36
|
update(name: string, patch: Partial<WorkerRecord>): Promise<void>;
|
|
37
|
+
updateMany?: (names: string[], patch: Partial<WorkerRecord>) => Promise<void>;
|
|
35
38
|
remove(name: string): Promise<void>;
|
|
36
39
|
};
|
|
37
40
|
export declare const InMemoryWorkerStore: Readonly<{
|
|
@@ -8,6 +8,12 @@ const mergeRecord = (current, patch) => ({
|
|
|
8
8
|
...patch,
|
|
9
9
|
updatedAt: patch.updatedAt ?? now(),
|
|
10
10
|
});
|
|
11
|
+
const toSqlDateTime = (value) => {
|
|
12
|
+
if (!value)
|
|
13
|
+
return null;
|
|
14
|
+
// Use UTC and drop timezone for SQL DATETIME compatibility
|
|
15
|
+
return value.toISOString().slice(0, 19).replace('T', ' ');
|
|
16
|
+
};
|
|
11
17
|
const serializeDbWorker = (record) => ({
|
|
12
18
|
name: record.name,
|
|
13
19
|
queue_name: record.queueName,
|
|
@@ -16,13 +22,14 @@ const serializeDbWorker = (record) => ({
|
|
|
16
22
|
auto_start: record.autoStart,
|
|
17
23
|
concurrency: record.concurrency,
|
|
18
24
|
region: record.region,
|
|
19
|
-
|
|
25
|
+
processor_spec: record.processorSpec ?? null,
|
|
26
|
+
active_status: record.activeStatus ?? true,
|
|
20
27
|
features: record.features ? JSON.stringify(record.features) : null,
|
|
21
28
|
infrastructure: record.infrastructure ? JSON.stringify(record.infrastructure) : null,
|
|
22
29
|
datacenter: record.datacenter ? JSON.stringify(record.datacenter) : null,
|
|
23
|
-
created_at: record.createdAt,
|
|
24
|
-
updated_at: record.updatedAt,
|
|
25
|
-
last_health_check: record.lastHealthCheck ?? null,
|
|
30
|
+
created_at: toSqlDateTime(record.createdAt),
|
|
31
|
+
updated_at: toSqlDateTime(record.updatedAt),
|
|
32
|
+
last_health_check: toSqlDateTime(record.lastHealthCheck ?? null),
|
|
26
33
|
last_error: record.lastError ?? null,
|
|
27
34
|
connection_state: record.connectionState ?? null,
|
|
28
35
|
});
|
|
@@ -53,7 +60,8 @@ const deserializeDbWorker = (row) => {
|
|
|
53
60
|
autoStart: Boolean(row['auto_start'] ?? false),
|
|
54
61
|
concurrency: Number(row['concurrency'] ?? 0),
|
|
55
62
|
region: row['region'] ? String(row['region']) : null,
|
|
56
|
-
|
|
63
|
+
processorSpec: String(row['processor_spec']),
|
|
64
|
+
activeStatus: row['active_status'] === undefined ? true : Boolean(row['active_status']),
|
|
57
65
|
features: parseJson(row['features']),
|
|
58
66
|
infrastructure: parseJson(row['infrastructure']),
|
|
59
67
|
datacenter: parseJson(row['datacenter']),
|
|
@@ -94,6 +102,14 @@ export const InMemoryWorkerStore = Object.freeze({
|
|
|
94
102
|
return;
|
|
95
103
|
store.set(name, mergeRecord(current, patch));
|
|
96
104
|
},
|
|
105
|
+
async updateMany(names, patch) {
|
|
106
|
+
for (const name of names) {
|
|
107
|
+
const current = store.get(name);
|
|
108
|
+
if (!current)
|
|
109
|
+
continue;
|
|
110
|
+
store.set(name, mergeRecord(current, patch));
|
|
111
|
+
}
|
|
112
|
+
},
|
|
97
113
|
async remove(name) {
|
|
98
114
|
store.delete(name);
|
|
99
115
|
},
|
|
@@ -122,7 +138,7 @@ export const RedisWorkerStore = Object.freeze({
|
|
|
122
138
|
},
|
|
123
139
|
async list(options) {
|
|
124
140
|
const all = await client.hgetall(key);
|
|
125
|
-
let values = Object.values(all).map(deserialize);
|
|
141
|
+
let values = Object.values(all).map((element) => deserialize(element));
|
|
126
142
|
values.sort((a, b) => a.name.localeCompare(b.name));
|
|
127
143
|
if (options) {
|
|
128
144
|
const start = options.offset || 0;
|
|
@@ -144,6 +160,22 @@ export const RedisWorkerStore = Object.freeze({
|
|
|
144
160
|
return;
|
|
145
161
|
await client.hset(key, name, serialize(mergeRecord(current, patch)));
|
|
146
162
|
},
|
|
163
|
+
async updateMany(names, patch) {
|
|
164
|
+
if (names.length === 0)
|
|
165
|
+
return;
|
|
166
|
+
const entries = await client.hmget(key, ...names);
|
|
167
|
+
const updates = [];
|
|
168
|
+
entries.forEach((raw, index) => {
|
|
169
|
+
if (!raw)
|
|
170
|
+
return;
|
|
171
|
+
const current = deserialize(raw);
|
|
172
|
+
const updated = mergeRecord(current, patch);
|
|
173
|
+
updates.push(names[index], serialize(updated));
|
|
174
|
+
});
|
|
175
|
+
if (updates.length === 0)
|
|
176
|
+
return;
|
|
177
|
+
await client.hset(key, ...updates);
|
|
178
|
+
},
|
|
147
179
|
async remove(name) {
|
|
148
180
|
await client.hdel(key, name);
|
|
149
181
|
},
|
|
@@ -163,7 +195,7 @@ export const DbWorkerStore = Object.freeze({
|
|
|
163
195
|
if (options?.offset)
|
|
164
196
|
query.offset(options.offset);
|
|
165
197
|
const rows = await query.get();
|
|
166
|
-
return rows.map(deserializeDbWorker);
|
|
198
|
+
return rows.map((element) => deserializeDbWorker(element));
|
|
167
199
|
},
|
|
168
200
|
async get(name) {
|
|
169
201
|
const row = await db.table(table).where('name', '=', name).first();
|
|
@@ -187,6 +219,22 @@ export const DbWorkerStore = Object.freeze({
|
|
|
187
219
|
const updated = mergeRecord(current, patch);
|
|
188
220
|
await db.table(table).where('name', '=', name).update(serializeDbWorker(updated));
|
|
189
221
|
},
|
|
222
|
+
async updateMany(names, patch) {
|
|
223
|
+
if (names.length === 0)
|
|
224
|
+
return;
|
|
225
|
+
const update = {
|
|
226
|
+
updated_at: toSqlDateTime(patch.updatedAt ?? now()),
|
|
227
|
+
};
|
|
228
|
+
if (patch.status !== undefined)
|
|
229
|
+
update['status'] = patch.status;
|
|
230
|
+
if (patch.lastError !== undefined)
|
|
231
|
+
update['last_error'] = patch.lastError ?? null;
|
|
232
|
+
if (patch.lastHealthCheck !== undefined)
|
|
233
|
+
update['last_health_check'] = toSqlDateTime(patch.lastHealthCheck ?? null);
|
|
234
|
+
if (patch.connectionState !== undefined)
|
|
235
|
+
update['connection_state'] = patch.connectionState ?? null;
|
|
236
|
+
await db.table(table).whereIn('name', names).update(update);
|
|
237
|
+
},
|
|
190
238
|
async remove(name) {
|
|
191
239
|
await db.table(table).where('name', '=', name).delete();
|
|
192
240
|
},
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
export type TelemetrySettings = {
|
|
2
|
+
enabled: boolean;
|
|
3
|
+
basePath: string;
|
|
4
|
+
middleware: ReadonlyArray<string>;
|
|
5
|
+
autoRefresh: boolean;
|
|
6
|
+
refreshIntervalMs: number;
|
|
7
|
+
};
|
|
8
|
+
export type ResourceCurrentResponse = {
|
|
9
|
+
ok: boolean;
|
|
10
|
+
usage?: unknown;
|
|
11
|
+
};
|
|
12
|
+
export type SystemSummaryResponse = {
|
|
13
|
+
ok: boolean;
|
|
14
|
+
summary?: unknown;
|
|
15
|
+
};
|
|
16
|
+
export type ApiResponse<T> = {
|
|
17
|
+
ok: boolean;
|
|
18
|
+
error?: string;
|
|
19
|
+
} & T;
|
|
20
|
+
export type AlertRep = {
|
|
21
|
+
type: string;
|
|
22
|
+
severity: string;
|
|
23
|
+
message: string;
|
|
24
|
+
timestamp: string;
|
|
25
|
+
recommendation?: string;
|
|
26
|
+
};
|
|
27
|
+
export declare const TelemetryAPI: Readonly<{
|
|
28
|
+
getSystemSummary(): Promise<ApiResponse<{
|
|
29
|
+
summary: unknown;
|
|
30
|
+
}>>;
|
|
31
|
+
getMonitoringSummary(): Promise<ApiResponse<{
|
|
32
|
+
summary: unknown;
|
|
33
|
+
}>>;
|
|
34
|
+
getResourceCurrent(): Promise<ApiResponse<{
|
|
35
|
+
usage: unknown;
|
|
36
|
+
}>>;
|
|
37
|
+
getResourceTrends(): Promise<ApiResponse<{
|
|
38
|
+
trends: unknown;
|
|
39
|
+
}>>;
|
|
40
|
+
}>;
|
|
41
|
+
export declare function createSnapshotBuilder(): () => Promise<{
|
|
42
|
+
ok: boolean;
|
|
43
|
+
summary: unknown;
|
|
44
|
+
resources: unknown;
|
|
45
|
+
cost: unknown;
|
|
46
|
+
}>;
|