@gravito/zenith 1.1.2 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -22
- package/README.zh-TW.md +88 -0
- package/dist/bin.js +54699 -39316
- package/dist/client/assets/index-C80c1frR.css +1 -0
- package/dist/client/assets/index-CrWem9u3.js +434 -0
- package/dist/client/index.html +2 -2
- package/dist/server/index.js +54699 -39316
- package/package.json +20 -9
- package/CHANGELOG.md +0 -47
- package/Dockerfile +0 -46
- package/Dockerfile.demo-worker +0 -29
- package/ECOSYSTEM_EXPANSION_RFC.md +0 -130
- package/bin/flux-console.ts +0 -2
- package/dist/client/assets/index-BSMp8oq_.js +0 -436
- package/dist/client/assets/index-BwxlHx-_.css +0 -1
- package/docker-compose.yml +0 -40
- package/docs/ALERTING_GUIDE.md +0 -71
- package/docs/DEPLOYMENT.md +0 -157
- package/docs/DOCS_INTERNAL.md +0 -73
- package/docs/LARAVEL_ZENITH_ROADMAP.md +0 -109
- package/docs/QUASAR_MASTER_PLAN.md +0 -140
- package/docs/QUICK_TEST_GUIDE.md +0 -72
- package/docs/ROADMAP.md +0 -85
- package/docs/integrations/LARAVEL.md +0 -207
- package/postcss.config.js +0 -6
- package/scripts/debug_redis_keys.ts +0 -24
- package/scripts/flood-logs.ts +0 -21
- package/scripts/seed.ts +0 -213
- package/scripts/verify-throttle.ts +0 -49
- package/scripts/worker.ts +0 -124
- package/specs/PULSE_SPEC.md +0 -86
- package/src/bin.ts +0 -6
- package/src/client/App.tsx +0 -72
- package/src/client/Layout.tsx +0 -672
- package/src/client/Sidebar.tsx +0 -112
- package/src/client/ThroughputChart.tsx +0 -144
- package/src/client/WorkerStatus.tsx +0 -226
- package/src/client/components/BrandIcons.tsx +0 -168
- package/src/client/components/ConfirmDialog.tsx +0 -126
- package/src/client/components/JobInspector.tsx +0 -554
- package/src/client/components/LogArchiveModal.tsx +0 -432
- package/src/client/components/NotificationBell.tsx +0 -212
- package/src/client/components/PageHeader.tsx +0 -47
- package/src/client/components/Toaster.tsx +0 -90
- package/src/client/components/UserProfileDropdown.tsx +0 -186
- package/src/client/contexts/AuthContext.tsx +0 -105
- package/src/client/contexts/NotificationContext.tsx +0 -128
- package/src/client/index.css +0 -174
- package/src/client/index.html +0 -12
- package/src/client/main.tsx +0 -15
- package/src/client/pages/LoginPage.tsx +0 -162
- package/src/client/pages/MetricsPage.tsx +0 -417
- package/src/client/pages/OverviewPage.tsx +0 -517
- package/src/client/pages/PulsePage.tsx +0 -488
- package/src/client/pages/QueuesPage.tsx +0 -379
- package/src/client/pages/SchedulesPage.tsx +0 -540
- package/src/client/pages/SettingsPage.tsx +0 -1020
- package/src/client/pages/WorkersPage.tsx +0 -394
- package/src/client/pages/index.ts +0 -8
- package/src/client/utils.ts +0 -15
- package/src/server/config/ServerConfigManager.ts +0 -90
- package/src/server/index.ts +0 -860
- package/src/server/middleware/auth.ts +0 -127
- package/src/server/services/AlertService.ts +0 -321
- package/src/server/services/CommandService.ts +0 -137
- package/src/server/services/LogStreamProcessor.ts +0 -93
- package/src/server/services/MaintenanceScheduler.ts +0 -78
- package/src/server/services/PulseService.ts +0 -91
- package/src/server/services/QueueMetricsCollector.ts +0 -138
- package/src/server/services/QueueService.ts +0 -631
- package/src/shared/types.ts +0 -198
- package/tailwind.config.js +0 -73
- package/tests/placeholder.test.ts +0 -7
- package/tsconfig.json +0 -38
- package/tsconfig.node.json +0 -12
- package/vite.config.ts +0 -27
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
import type { Context, Next } from 'hono'
|
|
2
|
-
import { deleteCookie, getCookie, setCookie } from 'hono/cookie'
|
|
3
|
-
|
|
4
|
-
// Session token store (in-memory for simplicity, consider Redis for production)
|
|
5
|
-
const sessions = new Map<string, { createdAt: number; expiresAt: number }>()
|
|
6
|
-
|
|
7
|
-
// Configuration
|
|
8
|
-
const SESSION_DURATION_MS = 24 * 60 * 60 * 1000 // 24 hours
|
|
9
|
-
const SESSION_COOKIE_NAME = 'flux_session'
|
|
10
|
-
|
|
11
|
-
/**
|
|
12
|
-
* Generate a secure random session token
|
|
13
|
-
*/
|
|
14
|
-
function generateSessionToken(): string {
|
|
15
|
-
const array = new Uint8Array(32)
|
|
16
|
-
crypto.getRandomValues(array)
|
|
17
|
-
return Array.from(array, (byte) => byte.toString(16).padStart(2, '0')).join('')
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* Check if authentication is enabled
|
|
22
|
-
*/
|
|
23
|
-
export function isAuthEnabled(): boolean {
|
|
24
|
-
return !!process.env.AUTH_PASSWORD
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* Verify the provided password against the environment variable
|
|
29
|
-
*/
|
|
30
|
-
export function verifyPassword(password: string): boolean {
|
|
31
|
-
const authPassword = process.env.AUTH_PASSWORD
|
|
32
|
-
if (!authPassword) {
|
|
33
|
-
return true // No password set, allow access
|
|
34
|
-
}
|
|
35
|
-
return password === authPassword
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Create a new session and set the cookie
|
|
40
|
-
*/
|
|
41
|
-
export function createSession(c: Context): string {
|
|
42
|
-
const token = generateSessionToken()
|
|
43
|
-
const now = Date.now()
|
|
44
|
-
|
|
45
|
-
sessions.set(token, {
|
|
46
|
-
createdAt: now,
|
|
47
|
-
expiresAt: now + SESSION_DURATION_MS,
|
|
48
|
-
})
|
|
49
|
-
|
|
50
|
-
setCookie(c, SESSION_COOKIE_NAME, token, {
|
|
51
|
-
httpOnly: true,
|
|
52
|
-
secure: process.env.NODE_ENV === 'production',
|
|
53
|
-
sameSite: 'Lax',
|
|
54
|
-
maxAge: SESSION_DURATION_MS / 1000,
|
|
55
|
-
path: '/',
|
|
56
|
-
})
|
|
57
|
-
|
|
58
|
-
return token
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* Validate a session token
|
|
63
|
-
*/
|
|
64
|
-
export function validateSession(token: string): boolean {
|
|
65
|
-
const session = sessions.get(token)
|
|
66
|
-
if (!session) {
|
|
67
|
-
return false
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
if (Date.now() > session.expiresAt) {
|
|
71
|
-
sessions.delete(token)
|
|
72
|
-
return false
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
return true
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
/**
|
|
79
|
-
* Destroy a session
|
|
80
|
-
*/
|
|
81
|
-
export function destroySession(c: Context): void {
|
|
82
|
-
const token = getCookie(c, SESSION_COOKIE_NAME)
|
|
83
|
-
if (token) {
|
|
84
|
-
sessions.delete(token)
|
|
85
|
-
}
|
|
86
|
-
deleteCookie(c, SESSION_COOKIE_NAME, { path: '/' })
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Authentication middleware for API routes
|
|
91
|
-
*/
|
|
92
|
-
export async function authMiddleware(c: Context, next: Next) {
|
|
93
|
-
// If no password is set, allow all requests
|
|
94
|
-
if (!isAuthEnabled()) {
|
|
95
|
-
return next()
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
// Allow auth endpoints without authentication
|
|
99
|
-
const path = c.req.path
|
|
100
|
-
if (path === '/api/auth/login' || path === '/api/auth/status' || path === '/api/auth/logout') {
|
|
101
|
-
return next()
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
// Check for valid session
|
|
105
|
-
const token = getCookie(c, SESSION_COOKIE_NAME)
|
|
106
|
-
if (token && validateSession(token)) {
|
|
107
|
-
return next()
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
// Unauthorized
|
|
111
|
-
return c.json({ error: 'Unauthorized', message: 'Please login to access this resource' }, 401)
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
/**
|
|
115
|
-
* Clean up expired sessions periodically
|
|
116
|
-
*/
|
|
117
|
-
export function cleanupExpiredSessions(): void {
|
|
118
|
-
const now = Date.now()
|
|
119
|
-
for (const [token, session] of sessions) {
|
|
120
|
-
if (now > session.expiresAt) {
|
|
121
|
-
sessions.delete(token)
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// Run cleanup every 10 minutes
|
|
127
|
-
setInterval(cleanupExpiredSessions, 10 * 60 * 1000)
|
|
@@ -1,321 +0,0 @@
|
|
|
1
|
-
import { EventEmitter } from 'node:events'
|
|
2
|
-
import { Redis } from 'ioredis'
|
|
3
|
-
import nodemailer from 'nodemailer'
|
|
4
|
-
import type { AlertConfig, AlertEvent, AlertRule, PulseNode } from '../../shared/types'
|
|
5
|
-
import type { WorkerReport } from './QueueService'
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* AlertService monitors system telemetry and triggers notifications.
|
|
9
|
-
*
|
|
10
|
-
* It evaluates rules against real-time data from Quasar agents and
|
|
11
|
-
* dispatches alerts via Slack, Discord, or Email when thresholds are exceeded.
|
|
12
|
-
*
|
|
13
|
-
* @public
|
|
14
|
-
* @since 3.0.0
|
|
15
|
-
*/
|
|
16
|
-
export class AlertService {
|
|
17
|
-
private redis: Redis
|
|
18
|
-
private rules: AlertRule[] = []
|
|
19
|
-
private config: AlertConfig = { channels: {} }
|
|
20
|
-
private cooldowns: Map<string, number> = new Map()
|
|
21
|
-
private emitter = new EventEmitter()
|
|
22
|
-
private readonly RULES_KEY = 'gravito:zenith:alerts:rules'
|
|
23
|
-
private readonly CONFIG_KEY = 'gravito:zenith:alerts:config'
|
|
24
|
-
|
|
25
|
-
constructor(redisUrl: string) {
|
|
26
|
-
this.redis = new Redis(redisUrl, {
|
|
27
|
-
lazyConnect: true,
|
|
28
|
-
})
|
|
29
|
-
|
|
30
|
-
// Initial default rules
|
|
31
|
-
this.rules = [
|
|
32
|
-
{
|
|
33
|
-
id: 'global_failure_spike',
|
|
34
|
-
name: 'High Failure Rate',
|
|
35
|
-
type: 'failure',
|
|
36
|
-
threshold: 50,
|
|
37
|
-
cooldownMinutes: 30,
|
|
38
|
-
},
|
|
39
|
-
{
|
|
40
|
-
id: 'global_backlog_critical',
|
|
41
|
-
name: 'Queue Backlog Warning',
|
|
42
|
-
type: 'backlog',
|
|
43
|
-
threshold: 1000,
|
|
44
|
-
cooldownMinutes: 60,
|
|
45
|
-
},
|
|
46
|
-
{
|
|
47
|
-
id: 'no_workers_online',
|
|
48
|
-
name: 'All Workers Offline',
|
|
49
|
-
type: 'worker_lost',
|
|
50
|
-
threshold: 1,
|
|
51
|
-
cooldownMinutes: 15,
|
|
52
|
-
},
|
|
53
|
-
]
|
|
54
|
-
|
|
55
|
-
// Default configuration (with env fallback for Slack)
|
|
56
|
-
if (process.env.SLACK_WEBHOOK_URL) {
|
|
57
|
-
this.config.channels.slack = {
|
|
58
|
-
enabled: true,
|
|
59
|
-
webhookUrl: process.env.SLACK_WEBHOOK_URL,
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
this.init().catch((err) => console.error('[AlertService] Failed to initialize:', err))
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
async connect() {
|
|
67
|
-
if (this.redis.status === 'wait') {
|
|
68
|
-
await this.redis.connect()
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
private async init() {
|
|
73
|
-
await this.loadRules()
|
|
74
|
-
await this.loadConfig()
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
async loadRules() {
|
|
78
|
-
try {
|
|
79
|
-
const data = await this.redis.get(this.RULES_KEY)
|
|
80
|
-
if (data) {
|
|
81
|
-
this.rules = JSON.parse(data)
|
|
82
|
-
}
|
|
83
|
-
} catch (err) {
|
|
84
|
-
console.error('[AlertService] Error loading rules from Redis:', err)
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
async loadConfig() {
|
|
89
|
-
try {
|
|
90
|
-
const data = await this.redis.get(this.CONFIG_KEY)
|
|
91
|
-
if (data) {
|
|
92
|
-
this.config = JSON.parse(data)
|
|
93
|
-
}
|
|
94
|
-
} catch (err) {
|
|
95
|
-
console.error('[AlertService] Error loading config from Redis:', err)
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
async saveRules(rules: AlertRule[]) {
|
|
100
|
-
this.rules = rules
|
|
101
|
-
await this.redis.set(this.RULES_KEY, JSON.stringify(rules))
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
async saveConfig(config: AlertConfig) {
|
|
105
|
-
this.config = config
|
|
106
|
-
await this.redis.set(this.CONFIG_KEY, JSON.stringify(config))
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
async addRule(rule: AlertRule) {
|
|
110
|
-
this.rules.push(rule)
|
|
111
|
-
await this.saveRules(this.rules)
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
async deleteRule(id: string) {
|
|
115
|
-
this.rules = this.rules.filter((r) => r.id !== id)
|
|
116
|
-
await this.saveRules(this.rules)
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
onAlert(callback: (event: AlertEvent) => void) {
|
|
120
|
-
this.emitter.on('alert', callback)
|
|
121
|
-
return () => this.emitter.off('alert', callback)
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
/**
|
|
125
|
-
* Evaluates rules against provided data.
|
|
126
|
-
*/
|
|
127
|
-
async check(data: {
|
|
128
|
-
queues: any[]
|
|
129
|
-
nodes: Record<string, PulseNode[]>
|
|
130
|
-
workers: WorkerReport[]
|
|
131
|
-
totals: { waiting: number; delayed: number; failed: number }
|
|
132
|
-
}) {
|
|
133
|
-
const now = Date.now()
|
|
134
|
-
|
|
135
|
-
for (const rule of this.rules) {
|
|
136
|
-
// 1. Check Cool-down
|
|
137
|
-
const lastFire = this.cooldowns.get(rule.id) || 0
|
|
138
|
-
if (now - lastFire < rule.cooldownMinutes * 60 * 1000) {
|
|
139
|
-
continue
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
let fired = false
|
|
143
|
-
let message = ''
|
|
144
|
-
let severity: 'warning' | 'critical' = 'warning'
|
|
145
|
-
|
|
146
|
-
// 2. Evaluate Rule
|
|
147
|
-
switch (rule.type) {
|
|
148
|
-
case 'backlog': {
|
|
149
|
-
const targetValue = rule.queue
|
|
150
|
-
? data.queues.find((q) => q.name === rule.queue)?.waiting || 0
|
|
151
|
-
: data.totals.waiting
|
|
152
|
-
|
|
153
|
-
if (targetValue >= rule.threshold) {
|
|
154
|
-
fired = true
|
|
155
|
-
severity = 'critical'
|
|
156
|
-
message = rule.queue
|
|
157
|
-
? `Queue backlog on ${rule.queue}: ${targetValue} jobs waiting.`
|
|
158
|
-
: `Queue backlog detected: ${targetValue} jobs waiting across all queues.`
|
|
159
|
-
}
|
|
160
|
-
break
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
case 'failure': {
|
|
164
|
-
const targetValue = rule.queue
|
|
165
|
-
? data.queues.find((q) => q.name === rule.queue)?.failed || 0
|
|
166
|
-
: data.totals.failed
|
|
167
|
-
|
|
168
|
-
if (targetValue >= rule.threshold) {
|
|
169
|
-
fired = true
|
|
170
|
-
severity = 'warning'
|
|
171
|
-
message = rule.queue
|
|
172
|
-
? `High failure count on ${rule.queue}: ${targetValue} jobs failed.`
|
|
173
|
-
: `High failure count: ${targetValue} jobs are currently in failed state.`
|
|
174
|
-
}
|
|
175
|
-
break
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
case 'worker_lost':
|
|
179
|
-
if (data.workers.length < rule.threshold) {
|
|
180
|
-
fired = true
|
|
181
|
-
severity = 'critical'
|
|
182
|
-
message = `System Incident: Zero worker nodes detected! Jobs will not be processed.`
|
|
183
|
-
}
|
|
184
|
-
break
|
|
185
|
-
|
|
186
|
-
case 'node_cpu':
|
|
187
|
-
// Check all pulse nodes
|
|
188
|
-
for (const serviceNodes of Object.values(data.nodes)) {
|
|
189
|
-
for (const node of serviceNodes) {
|
|
190
|
-
if (node.cpu.process >= rule.threshold) {
|
|
191
|
-
fired = true
|
|
192
|
-
severity = 'warning'
|
|
193
|
-
message = `High CPU Usage on ${node.service} (${node.id}): ${node.cpu.process}%`
|
|
194
|
-
break
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
if (fired) {
|
|
198
|
-
break
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
break
|
|
202
|
-
|
|
203
|
-
case 'node_ram':
|
|
204
|
-
for (const serviceNodes of Object.values(data.nodes)) {
|
|
205
|
-
for (const node of serviceNodes) {
|
|
206
|
-
const usagePercent = (node.memory.process.rss / node.memory.system.total) * 100
|
|
207
|
-
if (usagePercent >= rule.threshold) {
|
|
208
|
-
fired = true
|
|
209
|
-
severity = 'warning'
|
|
210
|
-
message = `High RAM Usage on ${node.service} (${node.id}): ${usagePercent.toFixed(1)}%`
|
|
211
|
-
break
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
if (fired) {
|
|
215
|
-
break
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
break
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
// 3. Dispatch if fired
|
|
222
|
-
if (fired) {
|
|
223
|
-
this.cooldowns.set(rule.id, now)
|
|
224
|
-
const event: AlertEvent = {
|
|
225
|
-
ruleId: rule.id,
|
|
226
|
-
timestamp: now,
|
|
227
|
-
message,
|
|
228
|
-
severity,
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
this.emitter.emit('alert', event)
|
|
232
|
-
this.notify(event)
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
private async notify(event: AlertEvent) {
|
|
238
|
-
const { slack, discord, email } = this.config.channels
|
|
239
|
-
|
|
240
|
-
// 1. Notify Slack
|
|
241
|
-
if (slack?.enabled && slack.webhookUrl) {
|
|
242
|
-
this.sendToWebhook(slack.webhookUrl, 'Slack', event).catch(console.error)
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
// 2. Notify Discord
|
|
246
|
-
if (discord?.enabled && discord.webhookUrl) {
|
|
247
|
-
this.sendToWebhook(discord.webhookUrl, 'Discord', event).catch(console.error)
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
// 3. Notify Email
|
|
251
|
-
if (email?.enabled) {
|
|
252
|
-
this.sendEmail(email, event).catch(console.error)
|
|
253
|
-
}
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
private async sendToWebhook(url: string, platform: string, event: AlertEvent) {
|
|
257
|
-
const payload = {
|
|
258
|
-
text: `*Flux Console Alert [${event.severity.toUpperCase()}]*\n${event.message}\n_Time: ${new Date(event.timestamp).toISOString()}_`,
|
|
259
|
-
attachments: [
|
|
260
|
-
{
|
|
261
|
-
color: event.severity === 'critical' ? '#ef4444' : '#f59e0b',
|
|
262
|
-
fields: [
|
|
263
|
-
{ title: 'Rule', value: event.ruleId, short: true },
|
|
264
|
-
{ title: 'Severity', value: event.severity, short: true },
|
|
265
|
-
{ title: 'Platform', value: platform, short: true },
|
|
266
|
-
],
|
|
267
|
-
},
|
|
268
|
-
],
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
const res = await fetch(url, {
|
|
272
|
-
method: 'POST',
|
|
273
|
-
headers: { 'Content-Type': 'application/json' },
|
|
274
|
-
body: JSON.stringify(payload),
|
|
275
|
-
})
|
|
276
|
-
|
|
277
|
-
if (!res.ok) {
|
|
278
|
-
throw new Error(`Failed to send to ${platform}: ${await res.text()}`)
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
private async sendEmail(config: any, event: AlertEvent) {
|
|
283
|
-
const transporter = nodemailer.createTransport({
|
|
284
|
-
host: config.smtpHost,
|
|
285
|
-
port: config.smtpPort,
|
|
286
|
-
secure: config.smtpPort === 465,
|
|
287
|
-
auth: {
|
|
288
|
-
user: config.smtpUser,
|
|
289
|
-
pass: config.smtpPass,
|
|
290
|
-
},
|
|
291
|
-
})
|
|
292
|
-
|
|
293
|
-
await transporter.sendMail({
|
|
294
|
-
from: config.from,
|
|
295
|
-
to: config.to,
|
|
296
|
-
subject: `[Zenith Alert] ${event.severity.toUpperCase()}: ${event.ruleId}`,
|
|
297
|
-
text: `${event.message}\n\nTimestamp: ${new Date(event.timestamp).toISOString()}\nSeverity: ${event.severity}`,
|
|
298
|
-
html: `
|
|
299
|
-
<div style="font-family: sans-serif; padding: 20px; border: 1px solid #eee; border-radius: 10px;">
|
|
300
|
-
<h2 style="color: ${event.severity === 'critical' ? '#ef4444' : '#f59e0b'}">
|
|
301
|
-
Zenith Alert: ${event.severity.toUpperCase()}
|
|
302
|
-
</h2>
|
|
303
|
-
<p style="font-size: 16px;">${event.message}</p>
|
|
304
|
-
<hr />
|
|
305
|
-
<p style="font-size: 12px; color: #666;">
|
|
306
|
-
Rule ID: ${event.ruleId}<br />
|
|
307
|
-
Time: ${new Date(event.timestamp).toISOString()}
|
|
308
|
-
</p>
|
|
309
|
-
</div>
|
|
310
|
-
`,
|
|
311
|
-
})
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
getRules() {
|
|
315
|
-
return this.rules
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
getConfig() {
|
|
319
|
-
return this.config
|
|
320
|
-
}
|
|
321
|
-
}
|
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
import type { CommandType, QuasarCommand } from '@gravito/quasar'
|
|
2
|
-
import { Redis } from 'ioredis'
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* CommandService handles sending commands from Zenith to Quasar agents.
|
|
6
|
-
*
|
|
7
|
-
* This is the "control center" that publishes commands to Redis Pub/Sub.
|
|
8
|
-
* Agents subscribe and execute commands locally.
|
|
9
|
-
*/
|
|
10
|
-
export class CommandService {
|
|
11
|
-
private redis: Redis
|
|
12
|
-
|
|
13
|
-
constructor(redisUrl: string) {
|
|
14
|
-
this.redis = new Redis(redisUrl, {
|
|
15
|
-
lazyConnect: true,
|
|
16
|
-
})
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
async connect(): Promise<void> {
|
|
20
|
-
if (this.redis.status !== 'ready' && this.redis.status !== 'connecting') {
|
|
21
|
-
await this.redis.connect()
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
* Send a command to a specific Quasar agent.
|
|
27
|
-
*
|
|
28
|
-
* @param service - Target service name
|
|
29
|
-
* @param nodeId - Target node ID (or '*' for broadcast)
|
|
30
|
-
* @param type - Command type
|
|
31
|
-
* @param payload - Command payload
|
|
32
|
-
* @returns Command ID
|
|
33
|
-
*/
|
|
34
|
-
async sendCommand(
|
|
35
|
-
service: string,
|
|
36
|
-
nodeId: string,
|
|
37
|
-
type: CommandType,
|
|
38
|
-
payload: QuasarCommand['payload']
|
|
39
|
-
): Promise<string> {
|
|
40
|
-
const commandId = crypto.randomUUID()
|
|
41
|
-
|
|
42
|
-
const command: QuasarCommand = {
|
|
43
|
-
id: commandId,
|
|
44
|
-
type,
|
|
45
|
-
targetNodeId: nodeId,
|
|
46
|
-
payload,
|
|
47
|
-
timestamp: Date.now(),
|
|
48
|
-
issuer: 'zenith',
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
const channel = `gravito:quasar:cmd:${service}:${nodeId}`
|
|
52
|
-
|
|
53
|
-
await this.redis.publish(channel, JSON.stringify(command))
|
|
54
|
-
|
|
55
|
-
console.log(`[CommandService] 📤 Sent ${type} to ${channel}`)
|
|
56
|
-
return commandId
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Retry a job on a specific node.
|
|
61
|
-
*/
|
|
62
|
-
async retryJob(
|
|
63
|
-
service: string,
|
|
64
|
-
nodeId: string,
|
|
65
|
-
queue: string,
|
|
66
|
-
jobKey: string,
|
|
67
|
-
driver: 'redis' | 'laravel' = 'redis'
|
|
68
|
-
): Promise<string> {
|
|
69
|
-
return this.sendCommand(service, nodeId, 'RETRY_JOB', {
|
|
70
|
-
queue,
|
|
71
|
-
jobKey,
|
|
72
|
-
driver,
|
|
73
|
-
})
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* Delete a job on a specific node.
|
|
78
|
-
*/
|
|
79
|
-
async deleteJob(
|
|
80
|
-
service: string,
|
|
81
|
-
nodeId: string,
|
|
82
|
-
queue: string,
|
|
83
|
-
jobKey: string,
|
|
84
|
-
driver: 'redis' | 'laravel' = 'redis'
|
|
85
|
-
): Promise<string> {
|
|
86
|
-
return this.sendCommand(service, nodeId, 'DELETE_JOB', {
|
|
87
|
-
queue,
|
|
88
|
-
jobKey,
|
|
89
|
-
driver,
|
|
90
|
-
})
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
/**
|
|
94
|
-
* Broadcast a retry command to all nodes of a service.
|
|
95
|
-
*/
|
|
96
|
-
async broadcastRetryJob(
|
|
97
|
-
service: string,
|
|
98
|
-
queue: string,
|
|
99
|
-
jobKey: string,
|
|
100
|
-
driver: 'redis' | 'laravel' = 'redis'
|
|
101
|
-
): Promise<string> {
|
|
102
|
-
return this.retryJob(service, '*', queue, jobKey, driver)
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
/**
|
|
106
|
-
* Broadcast a delete command to all nodes of a service.
|
|
107
|
-
*/
|
|
108
|
-
async broadcastDeleteJob(
|
|
109
|
-
service: string,
|
|
110
|
-
queue: string,
|
|
111
|
-
jobKey: string,
|
|
112
|
-
driver: 'redis' | 'laravel' = 'redis'
|
|
113
|
-
): Promise<string> {
|
|
114
|
-
return this.deleteJob(service, '*', queue, jobKey, driver)
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
* Send a Laravel-specific action (retry-all, restart) to a node.
|
|
119
|
-
*/
|
|
120
|
-
async laravelAction(
|
|
121
|
-
service: string,
|
|
122
|
-
nodeId: string,
|
|
123
|
-
action: 'retry-all' | 'restart' | 'retry',
|
|
124
|
-
jobId?: string
|
|
125
|
-
): Promise<string> {
|
|
126
|
-
return this.sendCommand(service, nodeId, 'LARAVEL_ACTION', {
|
|
127
|
-
queue: 'default',
|
|
128
|
-
jobKey: '*',
|
|
129
|
-
action,
|
|
130
|
-
jobId,
|
|
131
|
-
})
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
async disconnect(): Promise<void> {
|
|
135
|
-
await this.redis.quit()
|
|
136
|
-
}
|
|
137
|
-
}
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
import { EventEmitter } from 'node:events'
|
|
2
|
-
import type { Redis } from 'ioredis'
|
|
3
|
-
import type { SystemLog } from './QueueService'
|
|
4
|
-
|
|
5
|
-
export class LogStreamProcessor {
|
|
6
|
-
private static readonly MAX_LOGS_PER_SEC = 50
|
|
7
|
-
private logEmitter = new EventEmitter()
|
|
8
|
-
private logThrottleCount = 0
|
|
9
|
-
private logThrottleReset = Date.now()
|
|
10
|
-
|
|
11
|
-
constructor(
|
|
12
|
-
private redis: Redis,
|
|
13
|
-
private subRedis: Redis
|
|
14
|
-
) {
|
|
15
|
-
this.setupLogSubscription()
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* Setup Redis log subscription
|
|
20
|
-
*/
|
|
21
|
-
private setupLogSubscription(): void {
|
|
22
|
-
this.logEmitter.setMaxListeners(1000)
|
|
23
|
-
|
|
24
|
-
this.subRedis.on('message', (channel, message) => {
|
|
25
|
-
if (channel === 'flux_console:logs') {
|
|
26
|
-
this.processLogMessage(message)
|
|
27
|
-
}
|
|
28
|
-
})
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
/**
|
|
32
|
-
* Process incoming log message with throttling
|
|
33
|
-
*/
|
|
34
|
-
private processLogMessage(message: string): void {
|
|
35
|
-
try {
|
|
36
|
-
const now = Date.now()
|
|
37
|
-
|
|
38
|
-
if (now - this.logThrottleReset > 1000) {
|
|
39
|
-
this.logThrottleReset = now
|
|
40
|
-
this.logThrottleCount = 0
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
if (this.logThrottleCount < LogStreamProcessor.MAX_LOGS_PER_SEC) {
|
|
44
|
-
this.logThrottleCount++
|
|
45
|
-
const log = JSON.parse(message) as SystemLog
|
|
46
|
-
this.logEmitter.emit('log', log)
|
|
47
|
-
|
|
48
|
-
if (log.level === 'success' || log.level === 'error') {
|
|
49
|
-
this.updateThroughput()
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
} catch (_e) {
|
|
53
|
-
// Ignore parse errors
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* Update throughput counter for job completions
|
|
59
|
-
*/
|
|
60
|
-
private updateThroughput(): void {
|
|
61
|
-
const minute = Math.floor(Date.now() / 60000)
|
|
62
|
-
this.redis
|
|
63
|
-
.incr(`flux_console:throughput:${minute}`)
|
|
64
|
-
.then(() => {
|
|
65
|
-
this.redis.expire(`flux_console:throughput:${minute}`, 3600)
|
|
66
|
-
})
|
|
67
|
-
.catch(() => {})
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
/**
|
|
71
|
-
* Subscribe to log stream
|
|
72
|
-
*/
|
|
73
|
-
async subscribe(): Promise<void> {
|
|
74
|
-
await this.subRedis.subscribe('flux_console:logs')
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
/**
|
|
78
|
-
* Add listener for log events
|
|
79
|
-
*/
|
|
80
|
-
onLog(callback: (msg: SystemLog) => void): () => void {
|
|
81
|
-
this.logEmitter.on('log', callback)
|
|
82
|
-
return () => {
|
|
83
|
-
this.logEmitter.off('log', callback)
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
* Get emitter for direct access
|
|
89
|
-
*/
|
|
90
|
-
getEmitter(): EventEmitter {
|
|
91
|
-
return this.logEmitter
|
|
92
|
-
}
|
|
93
|
-
}
|