@gravito/zenith 0.1.0-beta.1 → 1.0.0

package/src/server/services/AlertService.ts

@@ -1,57 +1,110 @@
 import { EventEmitter } from 'events'
+import { Redis } from 'ioredis'
+import nodemailer from 'nodemailer'
+import type { AlertConfig, AlertEvent, AlertRule, PulseNode } from '../../shared/types'
 import type { WorkerReport } from './QueueService'
 
-export interface AlertRule {
-  id: string
-  name: string
-  type: 'backlog' | 'failure' | 'worker_lost'
-  threshold: number
-  queue?: string // Optional: specific queue or all
-  cooldownMinutes: number
-}
-
-export interface AlertEvent {
-  ruleId: string
-  timestamp: number
-  message: string
-  severity: 'warning' | 'critical'
-}
-
 export class AlertService {
+  private redis: Redis
   private rules: AlertRule[] = []
+  private config: AlertConfig = { channels: {} }
   private cooldowns: Map<string, number> = new Map()
-  private webhookUrl: string | null = process.env.SLACK_WEBHOOK_URL || null
   private emitter = new EventEmitter()
+  private readonly RULES_KEY = 'gravito:zenith:alerts:rules'
+  private readonly CONFIG_KEY = 'gravito:zenith:alerts:config'
 
-  constructor() {
-    // Default Rules
+  constructor(redisUrl: string) {
+    this.redis = new Redis(redisUrl, {
+      lazyConnect: true,
+    })
+
+    // Initial default rules
     this.rules = [
       {
         id: 'global_failure_spike',
         name: 'High Failure Rate',
         type: 'failure',
-        threshold: 50, // More than 50 failed jobs
+        threshold: 50,
         cooldownMinutes: 30,
       },
       {
         id: 'global_backlog_critical',
         name: 'Queue Backlog Warning',
         type: 'backlog',
-        threshold: 1000, // More than 1000 waiting jobs
+        threshold: 1000,
         cooldownMinutes: 60,
       },
       {
         id: 'no_workers_online',
         name: 'All Workers Offline',
         type: 'worker_lost',
-        threshold: 1, // < 1 worker
+        threshold: 1,
         cooldownMinutes: 15,
       },
     ]
+
+    // Default configuration (with env fallback for Slack)
+    if (process.env.SLACK_WEBHOOK_URL) {
+      this.config.channels.slack = {
+        enabled: true,
+        webhookUrl: process.env.SLACK_WEBHOOK_URL,
+      }
+    }
+
+    this.init().catch((err) => console.error('[AlertService] Failed to initialize:', err))
+  }
+
+  async connect() {
+    if (this.redis.status === 'wait') {
+      await this.redis.connect()
+    }
+  }
+
+  private async init() {
+    await this.loadRules()
+    await this.loadConfig()
+  }
+
+  async loadRules() {
+    try {
+      const data = await this.redis.get(this.RULES_KEY)
+      if (data) {
+        this.rules = JSON.parse(data)
+      }
+    } catch (err) {
+      console.error('[AlertService] Error loading rules from Redis:', err)
+    }
+  }
+
+  async loadConfig() {
+    try {
+      const data = await this.redis.get(this.CONFIG_KEY)
+      if (data) {
+        this.config = JSON.parse(data)
+      }
+    } catch (err) {
+      console.error('[AlertService] Error loading config from Redis:', err)
+    }
+  }
+
+  async saveRules(rules: AlertRule[]) {
+    this.rules = rules
+    await this.redis.set(this.RULES_KEY, JSON.stringify(rules))
+  }
+
+  async saveConfig(config: AlertConfig) {
+    this.config = config
+    await this.redis.set(this.CONFIG_KEY, JSON.stringify(config))
+  }
+
+  async addRule(rule: AlertRule) {
+    this.rules.push(rule)
+    await this.saveRules(this.rules)
   }
 
-  setWebhook(url: string | null) {
-    this.webhookUrl = url
+  async deleteRule(id: string) {
+    this.rules = this.rules.filter((r) => r.id !== id)
+    await this.saveRules(this.rules)
   }
 
   onAlert(callback: (event: AlertEvent) => void) {
@@ -61,10 +114,10 @@ export class AlertService {
 
   /**
    * Evaluates rules against provided data.
-   * Extremely lightweight: only uses existing metrics data.
    */
   async check(data: {
     queues: any[]
+    nodes: Record<string, PulseNode[]>
     workers: WorkerReport[]
     totals: { waiting: number; delayed: number; failed: number }
   }) {
@@ -83,21 +136,35 @@ export class AlertService {
 
       // 2. Evaluate Rule
       switch (rule.type) {
-        case 'backlog':
-          if (data.totals.waiting >= rule.threshold) {
+        case 'backlog': {
+          const targetValue = rule.queue
+            ? data.queues.find((q) => q.name === rule.queue)?.waiting || 0
+            : data.totals.waiting
+
+          if (targetValue >= rule.threshold) {
             fired = true
             severity = 'critical'
-            message = `Queue backlog detected: ${data.totals.waiting} jobs waiting across all queues.`
+            message = rule.queue
+              ? `Queue backlog on ${rule.queue}: ${targetValue} jobs waiting.`
+              : `Queue backlog detected: ${targetValue} jobs waiting across all queues.`
           }
           break
+        }
 
-        case 'failure':
-          if (data.totals.failed >= rule.threshold) {
+        case 'failure': {
+          const targetValue = rule.queue
+            ? data.queues.find((q) => q.name === rule.queue)?.failed || 0
+            : data.totals.failed
+
+          if (targetValue >= rule.threshold) {
             fired = true
             severity = 'warning'
-            message = `High failure count: ${data.totals.failed} jobs are currently in failed state.`
+            message = rule.queue
+              ? `High failure count on ${rule.queue}: ${targetValue} jobs failed.`
+              : `High failure count: ${targetValue} jobs are currently in failed state.`
           }
           break
+        }
 
         case 'worker_lost':
           if (data.workers.length < rule.threshold) {
@@ -106,6 +173,36 @@ export class AlertService {
             message = `System Incident: Zero worker nodes detected! Jobs will not be processed.`
           }
           break
+
+        case 'node_cpu':
+          // Check all pulse nodes
+          for (const serviceNodes of Object.values(data.nodes)) {
+            for (const node of serviceNodes) {
+              if (node.cpu.process >= rule.threshold) {
+                fired = true
+                severity = 'warning'
+                message = `High CPU Usage on ${node.service} (${node.id}): ${node.cpu.process}%`
+                break
+              }
+            }
+            if (fired) break
+          }
+          break
+
+        case 'node_ram':
+          for (const serviceNodes of Object.values(data.nodes)) {
+            for (const node of serviceNodes) {
+              const usagePercent = (node.memory.process.rss / node.memory.system.total) * 100
+              if (usagePercent >= rule.threshold) {
+                fired = true
+                severity = 'warning'
+                message = `High RAM Usage on ${node.service} (${node.id}): ${usagePercent.toFixed(1)}%`
+                break
+              }
+            }
+            if (fired) break
+          }
+          break
       }
 
       // 3. Dispatch if fired
@@ -124,14 +221,26 @@ export class AlertService {
     }
   }
 
-  /**
-   * Send notification to external channels.
-   * Fire-and-forget to ensure zero impact on main loop latency.
-   */
-  private notify(event: AlertEvent) {
-    if (!this.webhookUrl) return
+  private async notify(event: AlertEvent) {
+    const { slack, discord, email } = this.config.channels
 
-    // Simple Slack formatting
+    // 1. Notify Slack
+    if (slack?.enabled && slack.webhookUrl) {
+      this.sendToWebhook(slack.webhookUrl, 'Slack', event).catch(console.error)
+    }
+
+    // 2. Notify Discord
+    if (discord?.enabled && discord.webhookUrl) {
+      this.sendToWebhook(discord.webhookUrl, 'Discord', event).catch(console.error)
+    }
+
+    // 3. Notify Email
+    if (email?.enabled) {
+      this.sendEmail(email, event).catch(console.error)
+    }
+  }
+
+  private async sendToWebhook(url: string, platform: string, event: AlertEvent) {
     const payload = {
       text: `*Flux Console Alert [${event.severity.toUpperCase()}]*\n${event.message}\n_Time: ${new Date(event.timestamp).toISOString()}_`,
       attachments: [
@@ -140,21 +249,60 @@ export class AlertService {
           fields: [
             { title: 'Rule', value: event.ruleId, short: true },
             { title: 'Severity', value: event.severity, short: true },
+            { title: 'Platform', value: platform, short: true },
           ],
         },
       ],
     }
 
-    fetch(this.webhookUrl, {
+    const res = await fetch(url, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify(payload),
-    }).catch((err) => {
-      console.error('[AlertService] Failed to send notification:', err.message)
+    })
+
+    if (!res.ok) {
+      throw new Error(`Failed to send to ${platform}: ${await res.text()}`)
+    }
+  }
+
+  private async sendEmail(config: any, event: AlertEvent) {
+    const transporter = nodemailer.createTransport({
+      host: config.smtpHost,
+      port: config.smtpPort,
+      secure: config.smtpPort === 465,
+      auth: {
+        user: config.smtpUser,
+        pass: config.smtpPass,
+      },
+    })
+
+    await transporter.sendMail({
+      from: config.from,
+      to: config.to,
+      subject: `[Zenith Alert] ${event.severity.toUpperCase()}: ${event.ruleId}`,
+      text: `${event.message}\n\nTimestamp: ${new Date(event.timestamp).toISOString()}\nSeverity: ${event.severity}`,
+      html: `
+        <div style="font-family: sans-serif; padding: 20px; border: 1px solid #eee; border-radius: 10px;">
+          <h2 style="color: ${event.severity === 'critical' ? '#ef4444' : '#f59e0b'}">
+            Zenith Alert: ${event.severity.toUpperCase()}
+          </h2>
+          <p style="font-size: 16px;">${event.message}</p>
+          <hr />
+          <p style="font-size: 12px; color: #666;">
+            Rule ID: ${event.ruleId}<br />
+            Time: ${new Date(event.timestamp).toISOString()}
+          </p>
+        </div>
+      `,
     })
   }
 
   getRules() {
     return this.rules
   }
+
+  getConfig() {
+    return this.config
+  }
 }

package/src/server/services/CommandService.ts

@@ -0,0 +1,137 @@
+import type { CommandType, QuasarCommand } from '@gravito/quasar'
+import { Redis } from 'ioredis'
+
+/**
+ * CommandService handles sending commands from Zenith to Quasar agents.
+ *
+ * This is the "control center" that publishes commands to Redis Pub/Sub.
+ * Agents subscribe and execute commands locally.
+ */
+export class CommandService {
+  private redis: Redis
+
+  constructor(redisUrl: string) {
+    this.redis = new Redis(redisUrl, {
+      lazyConnect: true,
+    })
+  }
+
+  async connect(): Promise<void> {
+    if (this.redis.status !== 'ready' && this.redis.status !== 'connecting') {
+      await this.redis.connect()
+    }
+  }
+
+  /**
+   * Send a command to a specific Quasar agent.
+   *
+   * @param service - Target service name
+   * @param nodeId - Target node ID (or '*' for broadcast)
+   * @param type - Command type
+   * @param payload - Command payload
+   * @returns Command ID
+   */
+  async sendCommand(
+    service: string,
+    nodeId: string,
+    type: CommandType,
+    payload: QuasarCommand['payload']
+  ): Promise<string> {
+    const commandId = crypto.randomUUID()
+
+    const command: QuasarCommand = {
+      id: commandId,
+      type,
+      targetNodeId: nodeId,
+      payload,
+      timestamp: Date.now(),
+      issuer: 'zenith',
+    }
+
+    const channel = `gravito:quasar:cmd:${service}:${nodeId}`
+
+    await this.redis.publish(channel, JSON.stringify(command))
+
+    console.log(`[CommandService] 📤 Sent ${type} to ${channel}`)
+    return commandId
+  }
+
+  /**
+   * Retry a job on a specific node.
+   */
+  async retryJob(
+    service: string,
+    nodeId: string,
+    queue: string,
+    jobKey: string,
+    driver: 'redis' | 'laravel' = 'redis'
+  ): Promise<string> {
+    return this.sendCommand(service, nodeId, 'RETRY_JOB', {
+      queue,
+      jobKey,
+      driver,
+    })
+  }
+
+  /**
+   * Delete a job on a specific node.
+   */
+  async deleteJob(
+    service: string,
+    nodeId: string,
+    queue: string,
+    jobKey: string,
+    driver: 'redis' | 'laravel' = 'redis'
+  ): Promise<string> {
+    return this.sendCommand(service, nodeId, 'DELETE_JOB', {
+      queue,
+      jobKey,
+      driver,
+    })
+  }
+
+  /**
+   * Broadcast a retry command to all nodes of a service.
+   */
+  async broadcastRetryJob(
+    service: string,
+    queue: string,
+    jobKey: string,
+    driver: 'redis' | 'laravel' = 'redis'
+  ): Promise<string> {
+    return this.retryJob(service, '*', queue, jobKey, driver)
+  }
+
+  /**
+   * Broadcast a delete command to all nodes of a service.
+   */
+  async broadcastDeleteJob(
+    service: string,
+    queue: string,
+    jobKey: string,
+    driver: 'redis' | 'laravel' = 'redis'
+  ): Promise<string> {
+    return this.deleteJob(service, '*', queue, jobKey, driver)
+  }
+
+  /**
+   * Send a Laravel-specific action (retry-all, restart) to a node.
+   */
+  async laravelAction(
+    service: string,
+    nodeId: string,
+    action: 'retry-all' | 'restart' | 'retry',
+    jobId?: string
+  ): Promise<string> {
+    return this.sendCommand(service, nodeId, 'LARAVEL_ACTION', {
+      queue: 'default',
+      jobKey: '*',
+      action,
+      jobId,
+    })
+  }
+
+  async disconnect(): Promise<void> {
+    await this.redis.quit()
+  }
+}

package/src/server/services/PulseService.ts

@@ -0,0 +1,80 @@
+import { Redis } from 'ioredis'
+import type { PulseNode } from '../../shared/types'
+
+export class PulseService {
+  private redis: Redis
+  private prefix = 'gravito:quasar:node:'
+
+  constructor(redisUrl: string) {
+    this.redis = new Redis(redisUrl, {
+      lazyConnect: true,
+    })
+  }
+
+  async connect() {
+    await this.redis.connect()
+  }
+
+  /**
+   * Discovers active Pulse nodes using SCAN.
+   */
+  async getNodes(): Promise<Record<string, PulseNode[]>> {
+    const nodes: PulseNode[] = []
+    let cursor = '0'
+    const now = Date.now()
+
+    do {
+      // Scan for pulse keys
+      const result = await this.redis.scan(cursor, 'MATCH', `${this.prefix}*`, 'COUNT', 100)
+      cursor = result[0]
+      const keys = result[1]
+
+      if (keys.length > 0) {
+        // Fetch values
+        const values = await this.redis.mget(...keys)
+
+        values.forEach((v) => {
+          if (v) {
+            try {
+              const node = JSON.parse(v) as PulseNode
+              // Filter out stale nodes if TTL didn't catch them yet (grace period 60s)
+              if (now - node.timestamp < 60000) {
+                nodes.push(node)
+              }
+            } catch (_e) {
+              // Ignore malformed
+            }
+          }
+        })
+      }
+    } while (cursor !== '0')
+
+    // Group by service
+    const grouped: Record<string, PulseNode[]> = {}
+
+    // Sort nodes by service name, then by node id for stable UI positions
+    nodes.sort((a, b) => {
+      const sComp = a.service.localeCompare(b.service)
+      if (sComp !== 0) return sComp
+      return a.id.localeCompare(b.id)
+    })
+
+    for (const node of nodes) {
+      if (!grouped[node.service]) {
+        grouped[node.service] = []
+      }
+      grouped[node.service].push(node)
+    }
+
+    return grouped
+  }
+
+  /**
+   * Manually record a heartbeat (for this Zenith server itself).
+   */
+  async recordHeartbeat(node: PulseNode): Promise<void> {
+    const key = `${this.prefix}${node.service}:${node.id}`
+    // TTL 30 seconds
+    await this.redis.set(key, JSON.stringify(node), 'EX', 30)
+  }
+}

package/src/server/services/QueueService.ts

@@ -51,7 +51,7 @@ export class QueueService {
   private logThrottleReset = Date.now()
   private readonly MAX_LOGS_PER_SEC = 50
   private manager: QueueManager
-  public alerts = new AlertService()
+  public alerts: AlertService
 
   constructor(
     redisUrl: string,
@@ -84,10 +84,11 @@ export class QueueService {
       },
       persistence,
     })
+    this.alerts = new AlertService(redisUrl)
   }
 
   async connect() {
-    await Promise.all([this.redis.connect(), this.subRedis.connect()])
+    await Promise.all([this.redis.connect(), this.subRedis.connect(), this.alerts.connect()])
 
     // Setup single Redis subscription
     await this.subRedis.subscribe('flux_console:logs')
@@ -123,6 +124,58 @@ export class QueueService {
         }
       }
     })
+
+    // Start Maintenance Loop
+    this.runMaintenanceLoop()
+  }
+
+  private async runMaintenanceLoop() {
+    // Initial delay to avoid startup congestion
+    setTimeout(() => {
+      const loop = async () => {
+        try {
+          await this.checkMaintenance()
+        } catch (err) {
+          console.error('[Maintenance] Task Error:', err)
+        }
+        // Check every hour (3600000 ms)
+        setTimeout(loop, 3600000)
+      }
+      loop()
+    }, 1000 * 30) // 30 seconds after boot
+  }
+
+  private async checkMaintenance() {
+    const config = await this.getMaintenanceConfig()
+    if (!config.autoCleanup) return
+
+    const now = Date.now()
+    const lastRun = config.lastRun || 0
+    const ONE_DAY = 24 * 60 * 60 * 1000
+
+    if (now - lastRun >= ONE_DAY) {
+      console.log(
+        `[Maintenance] Starting Auto-Cleanup (Retention: ${config.retentionDays} days)...`
+      )
+      const deleted = await this.cleanupArchive(config.retentionDays)
+      console.log(`[Maintenance] Cleanup Complete. Removed ${deleted} records.`)
+
+      // Update Last Run
+      await this.saveMaintenanceConfig({
+        ...config,
+        lastRun: now,
+      })
+    }
+  }
+
+  async getMaintenanceConfig(): Promise<any> {
+    const data = await this.redis.get('gravito:zenith:maintenance:config')
+    if (data) return JSON.parse(data)
+    return { autoCleanup: false, retentionDays: 30 }
+  }
+
+  async saveMaintenanceConfig(config: any): Promise<void> {
+    await this.redis.set('gravito:zenith:maintenance:config', JSON.stringify(config))
   }
 
   /**
@@ -291,7 +344,10 @@ export class QueueService {
   /**
    * Records a snapshot of current global statistics for sparklines.
    */
-  async recordStatusMetrics(): Promise<void> {
+  async recordStatusMetrics(
+    nodes: Record<string, any> = {},
+    injectedWorkers?: any[]
+  ): Promise<void> {
     const stats = await this.listQueues()
     const totals = stats.reduce(
       (acc, q) => {
@@ -312,7 +368,7 @@ export class QueueService {
     pipe.set(`flux_console:metrics:failed:${now}`, totals.failed, 'EX', 3600)
 
     // Also record worker count
-    const workers = await this.listWorkers()
+    const workers = injectedWorkers || (await this.listWorkers())
     pipe.set(`flux_console:metrics:workers:${now}`, workers.length, 'EX', 3600)
 
     await pipe.exec()
@@ -328,7 +384,8 @@ export class QueueService {
     this.alerts
       .check({
         queues: stats,
-        workers,
+        nodes: nodes as any,
+        workers: workers as any,
         totals,
       })
       .catch((err) => console.error('[AlertService] Rule Evaluation Error:', err))
@@ -403,7 +460,7 @@ export class QueueService {
       }
     } while (cursor !== '0')
 
-    return workers
+    return workers.sort((a, b) => a.id.localeCompare(b.id))
   }
 
   /**