@gravito/zenith 1.1.3 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -10
- package/dist/bin.js +43235 -76691
- package/dist/client/index.html +13 -0
- package/dist/server/index.js +43235 -76691
- package/package.json +16 -7
- package/CHANGELOG.md +0 -62
- package/Dockerfile +0 -46
- package/Dockerfile.demo-worker +0 -29
- package/bin/flux-console.ts +0 -2
- package/doc/ECOSYSTEM_EXPANSION_RFC.md +0 -130
- package/docker-compose.yml +0 -40
- package/docs/ALERTING_GUIDE.md +0 -71
- package/docs/DEPLOYMENT.md +0 -157
- package/docs/DOCS_INTERNAL.md +0 -73
- package/docs/LARAVEL_ZENITH_ROADMAP.md +0 -109
- package/docs/QUASAR_MASTER_PLAN.md +0 -140
- package/docs/QUICK_TEST_GUIDE.md +0 -72
- package/docs/ROADMAP.md +0 -85
- package/docs/integrations/LARAVEL.md +0 -207
- package/postcss.config.js +0 -6
- package/scripts/debug_redis_keys.ts +0 -24
- package/scripts/flood-logs.ts +0 -21
- package/scripts/seed.ts +0 -213
- package/scripts/verify-throttle.ts +0 -49
- package/scripts/worker.ts +0 -124
- package/specs/PULSE_SPEC.md +0 -86
- package/src/bin.ts +0 -6
- package/src/client/App.tsx +0 -72
- package/src/client/Layout.tsx +0 -669
- package/src/client/Sidebar.tsx +0 -112
- package/src/client/ThroughputChart.tsx +0 -158
- package/src/client/WorkerStatus.tsx +0 -202
- package/src/client/components/BrandIcons.tsx +0 -168
- package/src/client/components/ConfirmDialog.tsx +0 -134
- package/src/client/components/JobInspector.tsx +0 -487
- package/src/client/components/LogArchiveModal.tsx +0 -432
- package/src/client/components/NotificationBell.tsx +0 -212
- package/src/client/components/PageHeader.tsx +0 -47
- package/src/client/components/Toaster.tsx +0 -90
- package/src/client/components/UserProfileDropdown.tsx +0 -186
- package/src/client/contexts/AuthContext.tsx +0 -105
- package/src/client/contexts/NotificationContext.tsx +0 -128
- package/src/client/index.css +0 -172
- package/src/client/main.tsx +0 -15
- package/src/client/pages/LoginPage.tsx +0 -164
- package/src/client/pages/MetricsPage.tsx +0 -445
- package/src/client/pages/OverviewPage.tsx +0 -519
- package/src/client/pages/PulsePage.tsx +0 -409
- package/src/client/pages/QueuesPage.tsx +0 -378
- package/src/client/pages/SchedulesPage.tsx +0 -535
- package/src/client/pages/SettingsPage.tsx +0 -1001
- package/src/client/pages/WorkersPage.tsx +0 -380
- package/src/client/pages/index.ts +0 -8
- package/src/client/utils.ts +0 -15
- package/src/server/config/ServerConfigManager.ts +0 -90
- package/src/server/index.ts +0 -860
- package/src/server/middleware/auth.ts +0 -127
- package/src/server/services/AlertService.ts +0 -321
- package/src/server/services/CommandService.ts +0 -136
- package/src/server/services/LogStreamProcessor.ts +0 -93
- package/src/server/services/MaintenanceScheduler.ts +0 -78
- package/src/server/services/PulseService.ts +0 -148
- package/src/server/services/QueueMetricsCollector.ts +0 -138
- package/src/server/services/QueueService.ts +0 -924
- package/src/shared/types.ts +0 -223
- package/tailwind.config.js +0 -80
- package/tests/placeholder.test.ts +0 -7
- package/tsconfig.json +0 -29
- package/tsconfig.node.json +0 -10
- package/vite.config.ts +0 -27
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
import type { Redis } from 'ioredis'
|
|
2
|
-
|
|
3
|
-
export interface MaintenanceConfig {
|
|
4
|
-
autoCleanup: boolean
|
|
5
|
-
retentionDays: number
|
|
6
|
-
lastRun?: number
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
export class MaintenanceScheduler {
|
|
10
|
-
private static readonly ONE_DAY = 24 * 60 * 60 * 1000
|
|
11
|
-
private static readonly CHECK_INTERVAL = 3600000 // 1 hour
|
|
12
|
-
|
|
13
|
-
constructor(
|
|
14
|
-
private redis: Redis,
|
|
15
|
-
private cleanupCallback: (retentionDays: number) => Promise<number>
|
|
16
|
-
) {}
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* Start the maintenance loop
|
|
20
|
-
*/
|
|
21
|
-
start(initialDelay = 30000): void {
|
|
22
|
-
setTimeout(() => {
|
|
23
|
-
const loop = async () => {
|
|
24
|
-
try {
|
|
25
|
-
await this.checkMaintenance()
|
|
26
|
-
} catch (err) {
|
|
27
|
-
console.error('[Maintenance] Task Error:', err)
|
|
28
|
-
}
|
|
29
|
-
setTimeout(loop, MaintenanceScheduler.CHECK_INTERVAL)
|
|
30
|
-
}
|
|
31
|
-
loop()
|
|
32
|
-
}, initialDelay)
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* Check and run maintenance if needed
|
|
37
|
-
*/
|
|
38
|
-
private async checkMaintenance(): Promise<void> {
|
|
39
|
-
const config = await this.getConfig()
|
|
40
|
-
if (!config.autoCleanup) {
|
|
41
|
-
return
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
const now = Date.now()
|
|
45
|
-
const lastRun = config.lastRun || 0
|
|
46
|
-
|
|
47
|
-
if (now - lastRun >= MaintenanceScheduler.ONE_DAY) {
|
|
48
|
-
console.log(
|
|
49
|
-
`[Maintenance] Starting Auto-Cleanup (Retention: ${config.retentionDays} days)...`
|
|
50
|
-
)
|
|
51
|
-
const deleted = await this.cleanupCallback(config.retentionDays)
|
|
52
|
-
console.log(`[Maintenance] Cleanup Complete. Removed ${deleted} records.`)
|
|
53
|
-
|
|
54
|
-
await this.saveConfig({
|
|
55
|
-
...config,
|
|
56
|
-
lastRun: now,
|
|
57
|
-
})
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* Get maintenance configuration
|
|
63
|
-
*/
|
|
64
|
-
async getConfig(): Promise<MaintenanceConfig> {
|
|
65
|
-
const data = await this.redis.get('gravito:zenith:maintenance:config')
|
|
66
|
-
if (data) {
|
|
67
|
-
return JSON.parse(data)
|
|
68
|
-
}
|
|
69
|
-
return { autoCleanup: false, retentionDays: 30 }
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
/**
|
|
73
|
-
* Save maintenance configuration
|
|
74
|
-
*/
|
|
75
|
-
async saveConfig(config: MaintenanceConfig): Promise<void> {
|
|
76
|
-
await this.redis.set('gravito:zenith:maintenance:config', JSON.stringify(config))
|
|
77
|
-
}
|
|
78
|
-
}
|
|
@@ -1,148 +0,0 @@
|
|
|
1
|
-
import { Redis } from 'ioredis'
|
|
2
|
-
import type { PulseNode } from '../../shared/types'
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* PulseService manages the discovery and health monitoring of system nodes.
|
|
6
|
-
*
|
|
7
|
-
* This service acts as the heartbeat of the distributed system, scanning Redis
|
|
8
|
-
* for ephemeral keys emitted by active Quasar agents. It aggregates these
|
|
9
|
-
* signals to provide a real-time view of the cluster topology, grouping nodes
|
|
10
|
-
* by their service roles.
|
|
11
|
-
*
|
|
12
|
-
* It is essential for:
|
|
13
|
-
* - Visualizing cluster health and scale.
|
|
14
|
-
* - Detecting silent node failures via heartbeat expiry.
|
|
15
|
-
* - Providing metadata for alerting systems.
|
|
16
|
-
*
|
|
17
|
-
* @public
|
|
18
|
-
* @since 3.0.0
|
|
19
|
-
*
|
|
20
|
-
* @example
|
|
21
|
-
* ```typescript
|
|
22
|
-
* const pulse = new PulseService('redis://localhost:6379');
|
|
23
|
-
* await pulse.connect();
|
|
24
|
-
* const nodes = await pulse.getNodes();
|
|
25
|
-
* ```
|
|
26
|
-
*/
|
|
27
|
-
export class PulseService {
|
|
28
|
-
private redis: Redis
|
|
29
|
-
private prefix = 'gravito:quasar:node:'
|
|
30
|
-
|
|
31
|
-
/**
|
|
32
|
-
* Creates a new instance of PulseService.
|
|
33
|
-
*
|
|
34
|
-
* @param redisUrl - Connection string for the Redis instance used for coordination.
|
|
35
|
-
*/
|
|
36
|
-
constructor(redisUrl: string) {
|
|
37
|
-
this.redis = new Redis(redisUrl, {
|
|
38
|
-
lazyConnect: true,
|
|
39
|
-
})
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* Establishes the connection to Redis.
|
|
44
|
-
*
|
|
45
|
-
* Must be called before any other operations.
|
|
46
|
-
*
|
|
47
|
-
* @returns Promise that resolves when connected.
|
|
48
|
-
* @throws {Error} If connection fails.
|
|
49
|
-
*/
|
|
50
|
-
async connect() {
|
|
51
|
-
await this.redis.connect()
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* Discovers active Pulse nodes across the cluster.
|
|
56
|
-
*
|
|
57
|
-
* Uses Redis SCAN to find all keys matching the heartbeat pattern.
|
|
58
|
-
* Nodes are grouped by service name to facilitate dashboard rendering.
|
|
59
|
-
* Stale nodes (older than 60s) are filtered out to ensure data freshness.
|
|
60
|
-
*
|
|
61
|
-
* @returns A map of service names to their active node instances.
|
|
62
|
-
* @throws {Error} If Redis operations fail.
|
|
63
|
-
*
|
|
64
|
-
* @example
|
|
65
|
-
* ```typescript
|
|
66
|
-
* const map = await pulse.getNodes();
|
|
67
|
-
* console.log(map['worker-service']); // Array of worker nodes
|
|
68
|
-
* ```
|
|
69
|
-
*/
|
|
70
|
-
async getNodes(): Promise<Record<string, PulseNode[]>> {
|
|
71
|
-
const nodes: PulseNode[] = []
|
|
72
|
-
let cursor = '0'
|
|
73
|
-
const now = Date.now()
|
|
74
|
-
|
|
75
|
-
do {
|
|
76
|
-
// Scan for pulse keys
|
|
77
|
-
const result = await this.redis.scan(cursor, 'MATCH', `${this.prefix}*`, 'COUNT', 100)
|
|
78
|
-
cursor = result[0]
|
|
79
|
-
const keys = result[1]
|
|
80
|
-
|
|
81
|
-
if (keys.length > 0) {
|
|
82
|
-
// Fetch values
|
|
83
|
-
const values = await this.redis.mget(...keys)
|
|
84
|
-
|
|
85
|
-
values.forEach((v) => {
|
|
86
|
-
if (v) {
|
|
87
|
-
try {
|
|
88
|
-
const node = JSON.parse(v) as PulseNode
|
|
89
|
-
// Filter out stale nodes if TTL didn't catch them yet (grace period 60s)
|
|
90
|
-
if (now - node.timestamp < 60000) {
|
|
91
|
-
nodes.push(node)
|
|
92
|
-
}
|
|
93
|
-
} catch (_e) {
|
|
94
|
-
// Ignore malformed
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
})
|
|
98
|
-
}
|
|
99
|
-
} while (cursor !== '0')
|
|
100
|
-
|
|
101
|
-
// Group by service
|
|
102
|
-
const grouped: Record<string, PulseNode[]> = {}
|
|
103
|
-
|
|
104
|
-
// Sort nodes by service name, then by node id for stable UI positions
|
|
105
|
-
nodes.sort((a, b) => {
|
|
106
|
-
const sComp = a.service.localeCompare(b.service)
|
|
107
|
-
if (sComp !== 0) {
|
|
108
|
-
return sComp
|
|
109
|
-
}
|
|
110
|
-
return a.id.localeCompare(b.id)
|
|
111
|
-
})
|
|
112
|
-
|
|
113
|
-
for (const node of nodes) {
|
|
114
|
-
if (!grouped[node.service]) {
|
|
115
|
-
grouped[node.service] = []
|
|
116
|
-
}
|
|
117
|
-
grouped[node.service].push(node)
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
return grouped
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/**
|
|
124
|
-
* Manually records a heartbeat for the current process.
|
|
125
|
-
*
|
|
126
|
-
* This is typically used when Zenith itself needs to appear in the node list.
|
|
127
|
-
* The heartbeat is stored with a short TTL (30s) to ensure auto-removal
|
|
128
|
-
* upon crash or shutdown.
|
|
129
|
-
*
|
|
130
|
-
* @param node - The node metadata to broadcast.
|
|
131
|
-
* @returns Promise resolving when the heartbeat is saved.
|
|
132
|
-
* @throws {Error} If Redis write fails.
|
|
133
|
-
*
|
|
134
|
-
* @example
|
|
135
|
-
* ```typescript
|
|
136
|
-
* await pulse.recordHeartbeat({
|
|
137
|
-
* id: 'zenith-1',
|
|
138
|
-
* service: 'dashboard',
|
|
139
|
-
* // ...other props
|
|
140
|
-
* });
|
|
141
|
-
* ```
|
|
142
|
-
*/
|
|
143
|
-
async recordHeartbeat(node: PulseNode): Promise<void> {
|
|
144
|
-
const key = `${this.prefix}${node.service}:${node.id}`
|
|
145
|
-
// TTL 30 seconds
|
|
146
|
-
await this.redis.set(key, JSON.stringify(node), 'EX', 30)
|
|
147
|
-
}
|
|
148
|
-
}
|
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
import type { Redis } from 'ioredis'
|
|
2
|
-
import type { GlobalStats, QueueStats, WorkerReport } from './QueueService'
|
|
3
|
-
|
|
4
|
-
export class QueueMetricsCollector {
|
|
5
|
-
constructor(
|
|
6
|
-
private redis: Redis,
|
|
7
|
-
private prefix = 'queue:'
|
|
8
|
-
) {}
|
|
9
|
-
|
|
10
|
-
/**
|
|
11
|
-
* Discover all queues using SCAN to avoid blocking Redis
|
|
12
|
-
*/
|
|
13
|
-
async listQueues(): Promise<QueueStats[]> {
|
|
14
|
-
const queues = new Set<string>()
|
|
15
|
-
let cursor = '0'
|
|
16
|
-
let iterations = 0
|
|
17
|
-
const MAX_ITERATIONS = 1000
|
|
18
|
-
|
|
19
|
-
do {
|
|
20
|
-
const result = await this.redis.scan(cursor, 'MATCH', `${this.prefix}*`, 'COUNT', 100)
|
|
21
|
-
cursor = result[0]
|
|
22
|
-
const keys = result[1]
|
|
23
|
-
|
|
24
|
-
for (const key of keys) {
|
|
25
|
-
const relative = key.slice(this.prefix.length)
|
|
26
|
-
const parts = relative.split(':')
|
|
27
|
-
const candidateName = parts[0]
|
|
28
|
-
|
|
29
|
-
if (candidateName && !this.isSystemKey(candidateName)) {
|
|
30
|
-
queues.add(candidateName)
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
iterations++
|
|
35
|
-
} while (cursor !== '0' && iterations < MAX_ITERATIONS)
|
|
36
|
-
|
|
37
|
-
return this.collectQueueStats(Array.from(queues).sort())
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* Collect statistics for a list of queues
|
|
42
|
-
*/
|
|
43
|
-
private async collectQueueStats(queueNames: string[]): Promise<QueueStats[]> {
|
|
44
|
-
const stats: QueueStats[] = []
|
|
45
|
-
const BATCH_SIZE = 10
|
|
46
|
-
|
|
47
|
-
for (let i = 0; i < queueNames.length; i += BATCH_SIZE) {
|
|
48
|
-
const batch = queueNames.slice(i, i + BATCH_SIZE)
|
|
49
|
-
const batchResults = await Promise.all(batch.map(async (name) => this.getQueueStats(name)))
|
|
50
|
-
stats.push(...batchResults)
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
return stats
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* Get statistics for a single queue
|
|
58
|
-
*/
|
|
59
|
-
private async getQueueStats(name: string): Promise<QueueStats> {
|
|
60
|
-
const waiting = await this.redis.llen(`${this.prefix}${name}`)
|
|
61
|
-
const delayed = await this.redis.zcard(`${this.prefix}${name}:delayed`)
|
|
62
|
-
const failed = await this.redis.llen(`${this.prefix}${name}:failed`)
|
|
63
|
-
const active = await this.redis.scard(`${this.prefix}${name}:active`)
|
|
64
|
-
const paused = await this.redis.get(`${this.prefix}${name}:paused`)
|
|
65
|
-
|
|
66
|
-
return {
|
|
67
|
-
name,
|
|
68
|
-
waiting,
|
|
69
|
-
delayed,
|
|
70
|
-
failed,
|
|
71
|
-
active,
|
|
72
|
-
paused: paused === '1',
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* Get global statistics including throughput
|
|
78
|
-
*/
|
|
79
|
-
async getGlobalStats(): Promise<GlobalStats> {
|
|
80
|
-
const queues = await this.listQueues()
|
|
81
|
-
const throughput = await this.getThroughputData()
|
|
82
|
-
|
|
83
|
-
return {
|
|
84
|
-
queues,
|
|
85
|
-
throughput,
|
|
86
|
-
workers: [],
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* Get throughput data for the last hour
|
|
92
|
-
*/
|
|
93
|
-
private async getThroughputData(): Promise<{ timestamp: string; count: number }[]> {
|
|
94
|
-
const data: { timestamp: string; count: number }[] = []
|
|
95
|
-
const currentMinute = Math.floor(Date.now() / 60000)
|
|
96
|
-
|
|
97
|
-
for (let i = 0; i < 60; i++) {
|
|
98
|
-
const minute = currentMinute - i
|
|
99
|
-
const count = await this.redis.get(`flux_console:throughput:${minute}`)
|
|
100
|
-
data.push({
|
|
101
|
-
timestamp: new Date(minute * 60000).toISOString(),
|
|
102
|
-
count: Number(count) || 0,
|
|
103
|
-
})
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
return data.reverse()
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/**
|
|
110
|
-
* Get reports from all active workers
|
|
111
|
-
*/
|
|
112
|
-
async listWorkers(): Promise<WorkerReport[]> {
|
|
113
|
-
const workers = await this.redis.smembers('flux_console:workers')
|
|
114
|
-
|
|
115
|
-
return Promise.all(workers.map(async (workerId) => this.getWorkerReport(workerId)))
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/**
|
|
119
|
-
* Get report from a specific worker
|
|
120
|
-
*/
|
|
121
|
-
private async getWorkerReport(workerId: string): Promise<WorkerReport> {
|
|
122
|
-
const data = await this.redis.get(`flux_console:worker:${workerId}`)
|
|
123
|
-
|
|
124
|
-
if (!data) {
|
|
125
|
-
throw new Error(`Worker ${workerId} not found`)
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
return JSON.parse(data) as WorkerReport
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
/**
|
|
132
|
-
* Check if a key is a system key
|
|
133
|
-
*/
|
|
134
|
-
private isSystemKey(key: string): boolean {
|
|
135
|
-
const systemKeys = ['active', 'schedules', 'schedule', 'lock']
|
|
136
|
-
return systemKeys.includes(key)
|
|
137
|
-
}
|
|
138
|
-
}
|