@gravito/zenith 0.1.0-beta.1 → 1.0.0-beta.1

package/src/client/pages/WorkersPage.tsx

@@ -1,7 +1,7 @@
 import { useQuery, useQueryClient } from '@tanstack/react-query'
 import { motion } from 'framer-motion'
 import { AlertCircle, Clock, Cpu, Gauge, MemoryStick, RefreshCcw, Server, Zap } from 'lucide-react'
-import React, { useEffect } from 'react'
+import { useEffect } from 'react'
 import { cn } from '../utils'
 
 interface Worker {

package/src/client/pages/index.ts

@@ -1,6 +1,7 @@
 export { LoginPage } from './LoginPage'
 export { MetricsPage } from './MetricsPage'
 export { OverviewPage } from './OverviewPage'
+export { PulsePage } from './PulsePage'
 export { QueuesPage } from './QueuesPage'
 export { SchedulesPage } from './SchedulesPage'
 export { SettingsPage } from './SettingsPage'

package/src/server/index.ts

@@ -1,9 +1,14 @@
 import { DB } from '@gravito/atlas'
 import { Photon } from '@gravito/photon'
+import { QuasarAgent } from '@gravito/quasar'
 import { MySQLPersistence, SQLitePersistence } from '@gravito/stream'
+import fs from 'fs'
 import { serveStatic } from 'hono/bun'
 import { getCookie } from 'hono/cookie'
 import { streamSSE } from 'hono/streaming'
+import os from 'os'
+import path from 'path'
+import { fileURLToPath } from 'url'
 import {
   authMiddleware,
   createSession,
@@ -11,6 +16,8 @@ import {
   isAuthEnabled,
   verifyPassword,
 } from './middleware/auth'
+import { CommandService } from './services/CommandService'
+import { PulseService } from './services/PulseService'
 import { QueueService } from './services/QueueService'
 
 const app = new Photon()
@@ -58,14 +65,26 @@ if (dbDriver === 'sqlite' || process.env.DB_HOST) {
 
 // Service Initialization
 const queueService = new QueueService(REDIS_URL, QUEUE_PREFIX, persistence)
+const pulseService = new PulseService(REDIS_URL)
+const commandService = new CommandService(REDIS_URL)
 
 queueService
   .connect()
+  .then(() => pulseService.connect())
+  .then(() => commandService.connect())
   .then(() => {
+    // Start Self-Monitoring (Quasar)
+    const agent = new QuasarAgent({
+      service: 'flux-console',
+      redisUrl: REDIS_URL,
+    })
+    agent.start().catch((err) => console.error('[FluxConsole] Quasar Agent Error:', err))
+
     console.log(`[FluxConsole] Connected to Redis at ${REDIS_URL}`)
     // Start background metrics recording (Reduced from 5s to 2s for better real-time feel)
-    setInterval(() => {
-      queueService.recordStatusMetrics().catch(console.error)
+    setInterval(async () => {
+      const nodes = await pulseService.getNodes()
+      queueService.recordStatusMetrics(nodes).catch(console.error)
     }, 2000)
 
     // Start Scheduler Tick (Reduced from 10s to 5s)
@@ -74,7 +93,10 @@ queueService
     }, 5000)
 
     // Record initial snapshot
-    queueService.recordStatusMetrics().catch(console.error)
+    pulseService
+      .getNodes()
+      .then((nodes) => queueService.recordStatusMetrics(nodes))
+      .catch(console.error)
   })
   .catch((err) => {
     console.error('[FluxConsole] Failed to connect to Redis', err)
@@ -328,19 +350,80 @@ api.get('/metrics/history', async (c) => {
 
 api.get('/system/status', (c) => {
   const mem = process.memoryUsage()
+  const totalMem = os.totalmem()
+
+  // Find package.json (relative to this file in src/server/index.ts)
+  const __dirname = path.dirname(fileURLToPath(import.meta.url))
+  const pkgPath = path.resolve(__dirname, '../../package.json')
+  let pkg = { version: '0.1.0-unknown', name: '@gravito/zenith' }
+  try {
+    pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'))
+  } catch (_e) {
+    // fallback
+  }
+
   return c.json({
     node: process.version,
     memory: {
       rss: `${(mem.rss / 1024 / 1024).toFixed(2)} MB`,
       heapUsed: `${(mem.heapUsed / 1024 / 1024).toFixed(2)} MB`,
-      total: '4.00 GB', // Hardcoded limit for demo aesthetic
+      total: `${(totalMem / 1024 / 1024 / 1024).toFixed(2)} GB`,
     },
-    engine: 'v0.1.0-beta.1',
+    version: pkg.version,
+    package: pkg.name,
+    engine: `Zenith ${pkg.version}`,
     uptime: process.uptime(),
-    env: process.env.NODE_ENV || 'production-east-1',
+    env:
+      process.env.NODE_ENV === 'production'
+        ? `production (${os.hostname()})`
+        : `development (${os.hostname()})`,
+    redisUrl: process.env.REDIS_URL || 'redis://localhost:6379',
   })
 })
 
+// --- Pulse Monitoring ---
+api.get('/pulse/nodes', async (c) => {
+  try {
+    const nodes = await pulseService.getNodes()
+    return c.json({ nodes })
+  } catch (_err) {
+    return c.json({ error: 'Failed to fetch pulse nodes' }, 500)
+  }
+})
+
+// --- Pulse Remote Control (Phase 3) ---
+api.post('/pulse/command', async (c) => {
+  try {
+    const { service, nodeId, type, queue, jobKey, driver, action } = await c.req.json()
+
+    // Validate required fields
+    if (!service || !nodeId || !type || !queue || !jobKey) {
+      return c.json({ error: 'Missing required fields: service, nodeId, type, queue, jobKey' }, 400)
+    }
+
+    // Validate command type
+    if (type !== 'RETRY_JOB' && type !== 'DELETE_JOB' && type !== 'LARAVEL_ACTION') {
+      return c.json({ error: 'Invalid command type. Allowed: RETRY_JOB, DELETE_JOB, LARAVEL_ACTION' }, 400)
+    }
+
+    const commandId = await commandService.sendCommand(service, nodeId, type, {
+      queue,
+      jobKey,
+      driver: driver || 'redis',
+      action,
+    })
+
+    return c.json({
+      success: true,
+      commandId,
+      message: `Command ${type} sent to ${nodeId}. Observe job state for result.`,
+    })
+  } catch (err) {
+    console.error('[CommandService] Error:', err)
+    return c.json({ error: 'Failed to send command' }, 500)
+  }
+})
+
 api.post('/queues/:name/jobs/delete', async (c) => {
   const queueName = c.req.param('name')
   const { type, raw } = await c.req.json()
@@ -454,9 +537,23 @@ api.get('/logs/stream', async (c) => {
       })
     })
 
+    // 4. Poll Pulse Nodes per client (simple polling for now)
+    const pulseInterval = setInterval(async () => {
+      try {
+        const nodes = await pulseService.getNodes()
+        await stream.writeSSE({
+          data: JSON.stringify({ nodes }),
+          event: 'pulse',
+        })
+      } catch (err) {
+        // ignore errors
+      }
+    }, 2000)
+
     stream.onAbort(() => {
       unsubscribeLogs()
       unsubscribeStats()
+      clearInterval(pulseInterval)
     })
 
     // Keep alive
@@ -508,17 +605,60 @@ api.delete('/schedules/:id', async (c) => {
 })
 
 // --- Alerting ---
-api.get('/alerts/config', (c) => {
+api.get('/alerts/config', async (c) => {
   return c.json({
     rules: queueService.alerts.getRules(),
-    webhookEnabled: !!process.env.SLACK_WEBHOOK_URL,
+    config: queueService.alerts.getConfig(),
+    maintenance: await queueService.getMaintenanceConfig(),
   })
 })
 
+api.post('/maintenance/config', async (c) => {
+  const config = await c.req.json()
+  try {
+    await queueService.saveMaintenanceConfig(config)
+    return c.json({ success: true })
+  } catch (err) {
+    return c.json({ error: 'Failed to save maintenance config' }, 500)
+  }
+})
+
+api.post('/alerts/config', async (c) => {
+  const config = await c.req.json()
+  try {
+    await queueService.alerts.saveConfig(config)
+    return c.json({ success: true })
+  } catch (err) {
+    return c.json({ error: 'Failed to save alert config' }, 500)
+  }
+})
+
+api.post('/alerts/rules', async (c) => {
+  const rule = await c.req.json()
+  try {
+    await queueService.alerts.addRule(rule)
+    return c.json({ success: true })
+  } catch (err) {
+    return c.json({ error: 'Failed to add rule' }, 500)
+  }
+})
+
+api.delete('/alerts/rules/:id', async (c) => {
+  const id = c.req.param('id')
+  try {
+    await queueService.alerts.deleteRule(id)
+    return c.json({ success: true })
+  } catch (err) {
+    return c.json({ error: 'Failed to delete rule' }, 500)
+  }
+})
+
 api.post('/alerts/test', async (c) => {
   try {
+    const nodes = await pulseService.getNodes()
     queueService.alerts.check({
       queues: [],
+      nodes,
       workers: [
         {
           id: 'test-node',

package/src/server/services/AlertService.ts

@@ -1,57 +1,110 @@
 import { EventEmitter } from 'events'
+import { Redis } from 'ioredis'
+import nodemailer from 'nodemailer'
+import type { AlertConfig, AlertEvent, AlertRule, PulseNode } from '../../shared/types'
 import type { WorkerReport } from './QueueService'
 
-export interface AlertRule {
-  id: string
-  name: string
-  type: 'backlog' | 'failure' | 'worker_lost'
-  threshold: number
-  queue?: string // Optional: specific queue or all
-  cooldownMinutes: number
-}
-
-export interface AlertEvent {
-  ruleId: string
-  timestamp: number
-  message: string
-  severity: 'warning' | 'critical'
-}
-
 export class AlertService {
+  private redis: Redis
   private rules: AlertRule[] = []
+  private config: AlertConfig = { channels: {} }
   private cooldowns: Map<string, number> = new Map()
-  private webhookUrl: string | null = process.env.SLACK_WEBHOOK_URL || null
   private emitter = new EventEmitter()
+  private readonly RULES_KEY = 'gravito:zenith:alerts:rules'
+  private readonly CONFIG_KEY = 'gravito:zenith:alerts:config'
 
-  constructor() {
-    // Default Rules
+  constructor(redisUrl: string) {
+    this.redis = new Redis(redisUrl, {
+      lazyConnect: true,
+    })
+
+    // Initial default rules
     this.rules = [
       {
         id: 'global_failure_spike',
         name: 'High Failure Rate',
         type: 'failure',
-        threshold: 50, // More than 50 failed jobs
+        threshold: 50,
         cooldownMinutes: 30,
       },
       {
         id: 'global_backlog_critical',
         name: 'Queue Backlog Warning',
         type: 'backlog',
-        threshold: 1000, // More than 1000 waiting jobs
+        threshold: 1000,
         cooldownMinutes: 60,
       },
       {
         id: 'no_workers_online',
         name: 'All Workers Offline',
         type: 'worker_lost',
-        threshold: 1, // < 1 worker
+        threshold: 1,
         cooldownMinutes: 15,
       },
     ]
+
+    // Default configuration (with env fallback for Slack)
+    if (process.env.SLACK_WEBHOOK_URL) {
+      this.config.channels.slack = {
+        enabled: true,
+        webhookUrl: process.env.SLACK_WEBHOOK_URL,
+      }
+    }
+
+    this.init().catch((err) => console.error('[AlertService] Failed to initialize:', err))
+  }
+
+  async connect() {
+    if (this.redis.status === 'wait') {
+      await this.redis.connect()
+    }
+  }
+
+  private async init() {
+    await this.loadRules()
+    await this.loadConfig()
+  }
+
+  async loadRules() {
+    try {
+      const data = await this.redis.get(this.RULES_KEY)
+      if (data) {
+        this.rules = JSON.parse(data)
+      }
+    } catch (err) {
+      console.error('[AlertService] Error loading rules from Redis:', err)
+    }
+  }
+
+  async loadConfig() {
+    try {
+      const data = await this.redis.get(this.CONFIG_KEY)
+      if (data) {
+        this.config = JSON.parse(data)
+      }
+    } catch (err) {
+      console.error('[AlertService] Error loading config from Redis:', err)
+    }
+  }
+
+  async saveRules(rules: AlertRule[]) {
+    this.rules = rules
+    await this.redis.set(this.RULES_KEY, JSON.stringify(rules))
+  }
+
+  async saveConfig(config: AlertConfig) {
+    this.config = config
+    await this.redis.set(this.CONFIG_KEY, JSON.stringify(config))
+  }
+
+  async addRule(rule: AlertRule) {
+    this.rules.push(rule)
+    await this.saveRules(this.rules)
   }
 
-  setWebhook(url: string | null) {
-    this.webhookUrl = url
+  async deleteRule(id: string) {
+    this.rules = this.rules.filter((r) => r.id !== id)
+    await this.saveRules(this.rules)
   }
 
   onAlert(callback: (event: AlertEvent) => void) {
@@ -61,10 +114,10 @@ export class AlertService {
 
   /**
    * Evaluates rules against provided data.
-   * Extremely lightweight: only uses existing metrics data.
    */
   async check(data: {
     queues: any[]
+    nodes: Record<string, PulseNode[]>
     workers: WorkerReport[]
     totals: { waiting: number; delayed: number; failed: number }
   }) {
@@ -83,21 +136,35 @@ export class AlertService {
 
       // 2. Evaluate Rule
       switch (rule.type) {
-        case 'backlog':
-          if (data.totals.waiting >= rule.threshold) {
+        case 'backlog': {
+          const targetValue = rule.queue
+            ? data.queues.find((q) => q.name === rule.queue)?.waiting || 0
+            : data.totals.waiting
+
+          if (targetValue >= rule.threshold) {
             fired = true
             severity = 'critical'
-            message = `Queue backlog detected: ${data.totals.waiting} jobs waiting across all queues.`
+            message = rule.queue
+              ? `Queue backlog on ${rule.queue}: ${targetValue} jobs waiting.`
+              : `Queue backlog detected: ${targetValue} jobs waiting across all queues.`
           }
           break
+        }
 
-        case 'failure':
-          if (data.totals.failed >= rule.threshold) {
+        case 'failure': {
+          const targetValue = rule.queue
+            ? data.queues.find((q) => q.name === rule.queue)?.failed || 0
+            : data.totals.failed
+
+          if (targetValue >= rule.threshold) {
             fired = true
             severity = 'warning'
-            message = `High failure count: ${data.totals.failed} jobs are currently in failed state.`
+            message = rule.queue
+              ? `High failure count on ${rule.queue}: ${targetValue} jobs failed.`
+              : `High failure count: ${targetValue} jobs are currently in failed state.`
           }
           break
+        }
 
         case 'worker_lost':
           if (data.workers.length < rule.threshold) {
@@ -106,6 +173,36 @@ export class AlertService {
             message = `System Incident: Zero worker nodes detected! Jobs will not be processed.`
           }
           break
+
+        case 'node_cpu':
+          // Check all pulse nodes
+          for (const serviceNodes of Object.values(data.nodes)) {
+            for (const node of serviceNodes) {
+              if (node.cpu.process >= rule.threshold) {
+                fired = true
+                severity = 'warning'
+                message = `High CPU Usage on ${node.service} (${node.id}): ${node.cpu.process}%`
+                break
+              }
+            }
+            if (fired) break
+          }
+          break
+
+        case 'node_ram':
+          for (const serviceNodes of Object.values(data.nodes)) {
+            for (const node of serviceNodes) {
+              const usagePercent = (node.memory.process.rss / node.memory.system.total) * 100
+              if (usagePercent >= rule.threshold) {
+                fired = true
+                severity = 'warning'
+                message = `High RAM Usage on ${node.service} (${node.id}): ${usagePercent.toFixed(1)}%`
+                break
+              }
+            }
+            if (fired) break
+          }
+          break
       }
 
       // 3. Dispatch if fired
@@ -124,14 +221,26 @@ export class AlertService {
     }
   }
 
-  /**
-   * Send notification to external channels.
-   * Fire-and-forget to ensure zero impact on main loop latency.
-   */
-  private notify(event: AlertEvent) {
-    if (!this.webhookUrl) return
+  private async notify(event: AlertEvent) {
+    const { slack, discord, email } = this.config.channels
 
-    // Simple Slack formatting
+    // 1. Notify Slack
+    if (slack?.enabled && slack.webhookUrl) {
+      this.sendToWebhook(slack.webhookUrl, 'Slack', event).catch(console.error)
+    }
+
+    // 2. Notify Discord
+    if (discord?.enabled && discord.webhookUrl) {
+      this.sendToWebhook(discord.webhookUrl, 'Discord', event).catch(console.error)
+    }
+
+    // 3. Notify Email
+    if (email?.enabled) {
+      this.sendEmail(email, event).catch(console.error)
+    }
+  }
+
+  private async sendToWebhook(url: string, platform: string, event: AlertEvent) {
     const payload = {
       text: `*Flux Console Alert [${event.severity.toUpperCase()}]*\n${event.message}\n_Time: ${new Date(event.timestamp).toISOString()}_`,
       attachments: [
@@ -140,21 +249,60 @@ export class AlertService {
           fields: [
             { title: 'Rule', value: event.ruleId, short: true },
             { title: 'Severity', value: event.severity, short: true },
+            { title: 'Platform', value: platform, short: true },
           ],
         },
       ],
     }
 
-    fetch(this.webhookUrl, {
+    const res = await fetch(url, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify(payload),
-    }).catch((err) => {
-      console.error('[AlertService] Failed to send notification:', err.message)
+    })
+
+    if (!res.ok) {
+      throw new Error(`Failed to send to ${platform}: ${await res.text()}`)
+    }
+  }
+
+  private async sendEmail(config: any, event: AlertEvent) {
+    const transporter = nodemailer.createTransport({
+      host: config.smtpHost,
+      port: config.smtpPort,
+      secure: config.smtpPort === 465,
+      auth: {
+        user: config.smtpUser,
+        pass: config.smtpPass,
+      },
+    })
+
+    await transporter.sendMail({
+      from: config.from,
+      to: config.to,
+      subject: `[Zenith Alert] ${event.severity.toUpperCase()}: ${event.ruleId}`,
+      text: `${event.message}\n\nTimestamp: ${new Date(event.timestamp).toISOString()}\nSeverity: ${event.severity}`,
+      html: `
+        <div style="font-family: sans-serif; padding: 20px; border: 1px solid #eee; border-radius: 10px;">
+          <h2 style="color: ${event.severity === 'critical' ? '#ef4444' : '#f59e0b'}">
+            Zenith Alert: ${event.severity.toUpperCase()}
+          </h2>
+          <p style="font-size: 16px;">${event.message}</p>
+          <hr />
+          <p style="font-size: 12px; color: #666;">
+            Rule ID: ${event.ruleId}<br />
+            Time: ${new Date(event.timestamp).toISOString()}
+          </p>
+        </div>
+      `,
     })
   }
 
   getRules() {
     return this.rules
   }
+
+  getConfig() {
+    return this.config
+  }
 }