@zintrust/workers 0.1.29 → 0.1.30

package/src/telemetry/api/TelemetryAPI.ts

@@ -0,0 +1,292 @@
+import { Logger } from '@zintrust/core';
+import type { WorkerData } from '../../dashboard';
+import { getWorkers } from '../../dashboard/workers-api';
+import { HealthMonitor } from '../../HealthMonitor';
+import type { WorkerResourceUsage } from '../../ResourceMonitor';
+import { ResourceMonitor } from '../../ResourceMonitor';
+
+export type TelemetrySettings = {
+  enabled: boolean;
+  basePath: string;
+  middleware: ReadonlyArray<string>;
+  autoRefresh: boolean;
+  refreshIntervalMs: number;
+};
+
+export type ResourceCurrentResponse = { ok: boolean; usage?: unknown };
+export type SystemSummaryResponse = { ok: boolean; summary?: unknown };
+
+const isOkWithUsage = (value: ResourceCurrentResponse): value is ResourceCurrentResponse =>
+  value.ok === true && 'usage' in value;
+
+const isOkWithSummary = (value: SystemSummaryResponse): value is SystemSummaryResponse =>
+  value.ok === true && 'summary' in value;
+
+export type ApiResponse<T> = { ok: boolean; error?: string } & T;
+export type AlertRep = {
+  type: string;
+  severity: string;
+  message: string;
+  timestamp: string;
+  recommendation?: string;
+};
+// Helper function to create stopped worker alert
+const createStoppedWorkerAlert = (worker: WorkerData): AlertRep => ({
+  type: 'worker-stopped',
+  severity: 'warning',
+  message: `Worker ${worker.name} is stopped`,
+  timestamp: worker.health?.lastCheck || new Date().toISOString(),
+});
+
+// Helper function to create health check alert
+const createHealthCheckAlert = (worker: WorkerData): AlertRep | null => {
+  const check = worker.health?.checks?.[0];
+  if (!check) return null;
+
+  return {
+    type: 'health-check-failed',
+    severity: check.status === 'fail' ? 'critical' : 'warning',
+    message: check.message || `Health check failed: ${check.name}`,
+    timestamp: worker.health?.lastCheck || new Date().toISOString(),
+  };
+};
+
+// Helper function to generate worker alerts
+const generateWorkerAlerts = (
+  workers: WorkerData[]
+): {
+  workerName: string;
+  status: string;
+  healthStatus: string;
+  lastCheck: string;
+  checks: WorkerData['health']['checks'];
+  alert: AlertRep | null;
+}[] => {
+  return workers
+    .filter((w) => {
+      return (
+        w.status !== 'running' ||
+        w.health?.status !== 'healthy' ||
+        (w.health?.checks && w.health.checks.length > 0)
+      );
+    })
+    .map((w) => {
+      const workerData = {
+        workerName: w.name,
+        status: w.status,
+        healthStatus: w.health?.status || 'unknown',
+        lastCheck: w.health?.lastCheck || new Date().toISOString(),
+        checks: w.health?.checks || [],
+      };
+
+      let alert = null;
+      if (w.status === 'stopped') {
+        alert = createStoppedWorkerAlert(w);
+      } else if (w.health?.checks && w.health.checks.length > 0) {
+        alert = createHealthCheckAlert(w);
+      }
+
+      return {
+        ...workerData,
+        alert,
+      };
+    })
+    .filter((w) => w.alert !== null);
+};
+
+// Helper function to generate resource alerts
+const generateResourceAlerts = (resourceUsage: WorkerResourceUsage): AlertRep[] => {
+  const resourceAlerts = [];
+  const cpuUsage = resourceUsage.resourceSnapshot?.cpu?.usage || 0;
+  const memoryUsage = resourceUsage.resourceSnapshot?.memory?.usage || 0;
+
+  if (cpuUsage > 90) {
+    resourceAlerts.push({
+      type: 'cpu-high',
+      severity: 'critical',
+      message: `Critical CPU usage: ${cpuUsage.toFixed(1)}%`,
+      timestamp: new Date().toISOString(),
+      recommendation: 'Consider scaling up or optimizing worker code',
+    });
+  } else if (cpuUsage > 80) {
+    resourceAlerts.push({
+      type: 'cpu-high',
+      severity: 'warning',
+      message: `High CPU usage: ${cpuUsage.toFixed(1)}%`,
+      timestamp: new Date().toISOString(),
+      recommendation: 'Monitor closely and consider scaling',
+    });
+  }
+
+  if (memoryUsage > 95) {
+    resourceAlerts.push({
+      type: 'memory-high',
+      severity: 'critical',
+      message: `Critical memory usage: ${memoryUsage.toFixed(1)}%`,
+      timestamp: new Date().toISOString(),
+      recommendation: 'Increase memory allocation or optimize memory usage',
+    });
+  } else if (memoryUsage > 85) {
+    resourceAlerts.push({
+      type: 'memory-high',
+      severity: 'warning',
+      message: `High memory usage: ${memoryUsage.toFixed(1)}%`,
+      timestamp: new Date().toISOString(),
+      recommendation: 'Monitor memory usage closely',
+    });
+  }
+
+  return resourceAlerts;
+};
+
+// Helper function to calculate monitoring summary
+const calculateMonitoringSummary = (
+  runningWorkers: WorkerData[]
+): {
+  total: number;
+  healthy: number;
+  degraded: number;
+  critical: number;
+  details: {
+    workerName: string;
+    status: string;
+    lastCheck: string;
+    checks: WorkerData['health']['checks'];
+  }[];
+} => {
+  const healthyCount = runningWorkers.filter((w) => w.health?.status === 'healthy').length;
+  const degradedCount = runningWorkers.filter((w) => w.health?.status === 'warning').length;
+  const criticalCount = runningWorkers.filter((w) => w.health?.status === 'unhealthy').length;
+
+  return {
+    total: runningWorkers.length,
+    healthy: healthyCount,
+    degraded: degradedCount,
+    critical: criticalCount,
+    details: runningWorkers.map((w) => ({
+      workerName: w.name,
+      status: w.health?.status || 'unknown',
+      lastCheck: w.health?.lastCheck || new Date().toISOString(),
+      checks: w.health?.checks || [],
+    })),
+  };
+};
+
+export const TelemetryAPI = Object.freeze({
+  async getSystemSummary(): Promise<ApiResponse<{ summary: unknown }>> {
+    try {
+      // Get all workers (both running and stopped) for complete alert visibility
+      const workersResult = await getWorkers({});
+      const runningWorkers = workersResult.workers.filter((w) => w.status === 'running');
+
+      // Calculate monitoring summary from running workers only
+      const monitoringSummary = calculateMonitoringSummary(runningWorkers);
+
+      // Generate alerts from workers and resources
+      const workerAlerts = generateWorkerAlerts(workersResult.workers);
+      const resourceUsage = ResourceMonitor.getCurrentUsage('system');
+      const resourceAlerts = generateResourceAlerts(resourceUsage);
+
+      // Combine all alerts
+      const allAlerts = [...workerAlerts, ...resourceAlerts];
+
+      return {
+        ok: true,
+        summary: {
+          workers: runningWorkers.length,
+          monitoring: {
+            ...monitoringSummary,
+            alerts: allAlerts, // Include all alerts (workers + resources)
+          },
+          resources: resourceUsage,
+          alerts: allAlerts, // Top-level alerts for easy access
+        },
+      };
+    } catch (error) {
+      Logger.error('Failed to get system summary', error as Error);
+      return {
+        ok: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+        summary: {},
+      };
+    }
+  },
+
+  async getMonitoringSummary(): Promise<ApiResponse<{ summary: unknown }>> {
+    try {
+      const summary = await HealthMonitor.getSummary();
+      return { ok: true, summary };
+    } catch (error) {
+      Logger.error('Failed to get monitoring summary', error as Error);
+      return {
+        ok: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+        summary: {},
+      };
+    }
+  },
+
+  async getResourceCurrent(): Promise<ApiResponse<{ usage: unknown }>> {
+    try {
+      const usage = ResourceMonitor.getCurrentUsage('system');
+      return { ok: true, usage };
+    } catch (error) {
+      Logger.error('Failed to get resource usage', error as Error);
+      return {
+        ok: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+        usage: null,
+      };
+    }
+  },
+
+  async getResourceTrends(): Promise<ApiResponse<{ trends: unknown }>> {
+    try {
+      const trends = ResourceMonitor.getAllTrends('system', 'day');
+      return { ok: true, trends };
+    } catch (error) {
+      Logger.error('Failed to get resource trends', error as Error);
+      return {
+        ok: false,
+        error: error instanceof Error ? error.message : 'Unknown error',
+        trends: null,
+      };
+    }
+  },
+});
+
+export function createSnapshotBuilder() {
+  return async (): Promise<{
+    ok: boolean;
+    summary: unknown;
+    resources: unknown;
+    cost: unknown;
+  }> => {
+    const [systemSummaryResult, resourceCurrentResult] = await Promise.allSettled([
+      TelemetryAPI.getSystemSummary(),
+      TelemetryAPI.getResourceCurrent(),
+    ]);
+
+    if (systemSummaryResult.status === 'rejected') {
+      Logger.error('Telemetry dashboard summary failed', systemSummaryResult.reason);
+    }
+
+    if (resourceCurrentResult.status === 'rejected') {
+      Logger.error('Telemetry resource summary failed', resourceCurrentResult.reason);
+    }
+
+    const systemSummary: SystemSummaryResponse =
+      systemSummaryResult.status === 'fulfilled' ? systemSummaryResult.value : { ok: false };
+    const resourceCurrent =
+      resourceCurrentResult.status === 'fulfilled'
+        ? resourceCurrentResult.value
+        : ({ ok: false } as ResourceCurrentResponse);
+
+    return {
+      ok: systemSummary.ok ?? false,
+      summary: isOkWithSummary(systemSummary) ? systemSummary.summary : {},
+      resources: isOkWithUsage(resourceCurrent) ? resourceCurrent.usage : null,
+      cost: null,
+    };
+  };
+}

package/src/telemetry/api/TelemetryMonitoringService.ts

@@ -0,0 +1,149 @@
+import type { IResponse } from '@zintrust/core';
+import { Logger, NodeSingletons } from '@zintrust/core';
+import type { createSnapshotBuilder, TelemetrySettings } from './TelemetryAPI';
+
+export type TelemetrySnapshotData = {
+  type: string;
+  ts: string;
+  ok: boolean;
+  summary: unknown;
+  resources: unknown;
+  cost: unknown;
+};
+
+// Internal state for singleton service
+const emitter = new NodeSingletons.EventEmitter();
+emitter.setMaxListeners(Infinity);
+let interval: NodeJS.Timeout | null = null;
+let subscribers = 0;
+let currentSettings: TelemetrySettings | null = null;
+let currentBuildSnapshot: ReturnType<typeof createSnapshotBuilder> | null = null;
+
+const broadcastTelemetrySnapshot = async (): Promise<void> => {
+  try {
+    if (subscribers <= 0 || !currentBuildSnapshot || !currentSettings) return;
+
+    const snapshot = await currentBuildSnapshot();
+
+    const payload: TelemetrySnapshotData = {
+      type: 'snapshot',
+      ts: new Date().toISOString(),
+      ...snapshot,
+    };
+
+    emitter.emit('snapshot', payload);
+  } catch (err) {
+    Logger.error('TelemetryMonitoringService.broadcastSnapshot failed', err);
+    emitter.emit('error', err);
+  }
+};
+
+const startPolling = (): void => {
+  if (interval || !currentSettings) return;
+
+  Logger.debug('Starting TelemetryMonitoringService polling');
+  // Initial fetch
+  void broadcastTelemetrySnapshot();
+
+  interval = setInterval(() => {
+    void broadcastTelemetrySnapshot();
+  }, currentSettings.refreshIntervalMs);
+};
+
+const stopPolling = (): void => {
+  if (interval) {
+    Logger.debug('Stopping TelemetryMonitoringService polling');
+    clearInterval(interval);
+    interval = null;
+  }
+};
+
+export const TelemetryMonitoringService = Object.freeze({
+  subscribe(callback: (data: TelemetrySnapshotData) => void): void {
+    emitter.on('snapshot', callback);
+    subscribers++;
+    if (subscribers === 1) {
+      startPolling();
+    }
+  },
+
+  unsubscribe(callback: (data: TelemetrySnapshotData) => void): void {
+    emitter.off('snapshot', callback);
+    subscribers--;
+    if (subscribers <= 0) {
+      stopPolling();
+      currentSettings = null;
+      currentBuildSnapshot = null;
+    }
+  },
+
+  startMonitoring(
+    settings: TelemetrySettings,
+    buildSnapshot: ReturnType<typeof createSnapshotBuilder>
+  ): void {
+    if (subscribers === 0) {
+      currentSettings = settings;
+      currentBuildSnapshot = buildSnapshot;
+    }
+  },
+
+  stopMonitoring(): void {
+    if (subscribers <= 0) {
+      stopPolling();
+      currentSettings = null;
+      currentBuildSnapshot = null;
+    }
+  },
+});
+
+export const teleStream = async (
+  res: IResponse,
+  settings: TelemetrySettings,
+  buildSnapshot: ReturnType<typeof createSnapshotBuilder>
+): Promise<void> => {
+  const raw = res.getRaw();
+
+  raw.writeHead(200, {
+    'Content-Type': 'text/event-stream',
+    'Cache-Control': 'no-cache, no-transform',
+    Connection: 'keep-alive',
+    'X-Accel-Buffering': 'no',
+  });
+
+  let closed = false;
+
+  const send = async (payload: unknown): Promise<void> => {
+    try {
+      raw.write(`data: ${JSON.stringify(payload)}\n\n`);
+    } catch (err) {
+      Logger.error('Telemetry SSE send failed', err);
+    }
+  };
+
+  // Send hello immediately
+  await send({ type: 'hello', ts: new Date().toISOString() });
+
+  // Start monitoring with the singleton service
+  TelemetryMonitoringService.startMonitoring(settings, buildSnapshot);
+
+  // Subscribe to telemetry snapshots
+  const onSnapshot = (data: TelemetrySnapshotData): void => {
+    if (!closed) {
+      void send(data);
+    }
+  };
+
+  TelemetryMonitoringService.subscribe(onSnapshot);
+
+  // Heartbeat to keep connection alive
+  const hb = setInterval(() => {
+    if (!closed) raw.write(': ping\n\n');
+  }, 15000);
+
+  raw.on('close', () => {
+    closed = true;
+    clearInterval(hb);
+    TelemetryMonitoringService.unsubscribe(onSnapshot);
+    TelemetryMonitoringService.stopMonitoring();
+  });
+};

package/src/telemetry/components/AlertPanel.ts

@@ -0,0 +1,13 @@
+export const renderAlertPanel = (): string => {
+  return `
+    <div class="zt-card">
+      <div class="zt-card-header">
+        <h3 class="zt-card-title">Alert History</h3>
+        <span class="zt-card-meta">Latest events</span>
+      </div>
+      <ul id="alertList" class="zt-alert-list">
+        <li class="zt-alert-item">No alerts yet.</li>
+      </ul>
+    </div>
+  `;
+};

package/src/telemetry/components/CostTracking.ts

@@ -0,0 +1,14 @@
+export const renderCostTracking = (): string => {
+  return `
+    <div class="zt-card">
+      <div class="zt-card-header">
+        <h3 class="zt-card-title">Cost Tracking</h3>
+        <span class="zt-card-meta">Daily estimate</span>
+      </div>
+      <div class="zt-card-body">
+        <p class="zt-cost-value" id="costTotal">$0.00</p>
+        <p class="zt-card-meta">Auto-updated from resource metrics</p>
+      </div>
+    </div>
+  `;
+};

package/src/telemetry/components/ResourceUsageChart.ts

@@ -0,0 +1,11 @@
+export const renderResourceUsageChart = (): string => {
+  return `
+    <div class="zt-card">
+      <div class="zt-card-header">
+        <h3 class="zt-card-title">Resource Usage</h3>
+        <span class="zt-card-meta">Current snapshot</span>
+      </div>
+      <canvas id="resourceUsageChart" class="zt-chart"></canvas>
+    </div>
+  `;
+};

package/src/telemetry/components/WorkerHealthChart.ts

@@ -0,0 +1,11 @@
+export const renderWorkerHealthChart = (): string => {
+  return `
+    <div class="zt-card">
+      <div class="zt-card-header">
+        <h3 class="zt-card-title">Worker Health</h3>
+        <span class="zt-card-meta">Last 24h</span>
+      </div>
+      <canvas id="workerHealthChart" class="zt-chart"></canvas>
+    </div>
+  `;
+};

package/src/telemetry/index.ts

@@ -0,0 +1,121 @@
+import type { IRequest, IResponse, IRouter, RouteOptions } from '@zintrust/core';
+import { Router } from '@zintrust/core';
+import type { TelemetrySettings } from './api/TelemetryAPI';
+import { createSnapshotBuilder } from './api/TelemetryAPI';
+import { teleStream } from './api/TelemetryMonitoringService';
+import { getDashboardHtml } from './routes/dashboard';
+
+export type TelemetryDashboardConfig = {
+  enabled?: boolean;
+  basePath?: string;
+  middleware?: ReadonlyArray<string>;
+  autoRefresh?: boolean;
+  refreshIntervalMs?: number;
+};
+
+const DEFAULTS = {
+  enabled: true,
+  basePath: '/telemetry',
+  autoRefresh: true,
+  refreshIntervalMs: 10000,
+};
+
+export type TelemetryDashboardApi = {
+  registerRoutes: (router: IRouter) => void;
+};
+
+export const TelemetryDashboard = Object.freeze({
+  create(config: TelemetryDashboardConfig): TelemetryDashboardApi {
+    const settings = buildSettings(config);
+    const buildSnapshot = createSnapshotBuilder();
+    const registerRoutes = createRouteRegistrar(settings, buildSnapshot);
+
+    return Object.freeze({ registerRoutes });
+  },
+});
+
+function buildSettings(config: TelemetryDashboardConfig): TelemetrySettings {
+  return {
+    enabled: config.enabled ?? DEFAULTS.enabled,
+    basePath: config.basePath ?? DEFAULTS.basePath,
+    middleware: config.middleware ?? [],
+    autoRefresh: config.autoRefresh ?? DEFAULTS.autoRefresh,
+    refreshIntervalMs:
+      typeof config.refreshIntervalMs === 'number' && Number.isFinite(config.refreshIntervalMs)
+        ? Math.max(1000, Math.floor(config.refreshIntervalMs))
+        : DEFAULTS.refreshIntervalMs,
+  };
+}
+
+function createRouteRegistrar(
+  settings: TelemetrySettings,
+  buildSnapshot: ReturnType<typeof createSnapshotBuilder>
+) {
+  return (router: IRouter): void => {
+    if (!settings.enabled) return;
+
+    const routeOptions: RouteOptions = (
+      settings.middleware.length > 0 ? { middleware: settings.middleware } : undefined
+    ) as RouteOptions;
+
+    registerDashboardRoute(router, settings, routeOptions);
+    registerSummaryApi(router, settings, routeOptions, buildSnapshot);
+    registerEventsApi(router, settings, routeOptions, buildSnapshot);
+  };
+}
+
+function registerDashboardRoute(
+  router: IRouter,
+  settings: TelemetrySettings,
+  routeOptions: RouteOptions
+): void {
+  Router.get(
+    router,
+    settings.basePath,
+    (_req, res) => {
+      res.html(
+        getDashboardHtml({
+          basePath: settings.basePath,
+          autoRefresh: settings.autoRefresh,
+          refreshIntervalMs: settings.refreshIntervalMs,
+        })
+      );
+    },
+    routeOptions
+  );
+}
+
+function registerSummaryApi(
+  router: IRouter,
+  settings: { basePath: string },
+  routeOptions: RouteOptions,
+  buildSnapshot: ReturnType<typeof createSnapshotBuilder>
+): void {
+  Router.get(
+    router,
+    `${settings.basePath}/api/summary`,
+    async (_req, res) => {
+      const snapshot = await buildSnapshot();
+      res.json(snapshot);
+    },
+    routeOptions
+  );
+}
+
+function registerEventsApi(
+  router: IRouter,
+  settings: TelemetrySettings,
+  routeOptions: RouteOptions,
+  buildSnapshot: ReturnType<typeof createSnapshotBuilder>
+): void {
+  Router.get(
+    router,
+    `${settings.basePath}/api/events`,
+    async (_req: IRequest, res: IResponse) => {
+      teleStream(res, settings, buildSnapshot);
+    },
+    routeOptions
+  );
+}
+
+export default TelemetryDashboard;

package/src/telemetry/public/assets/zintrust-logo.svg

@@ -0,0 +1,15 @@
+<svg width="96" height="96" viewBox="0 0 100 100" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <defs>
+    <linearGradient id="zt-logo" x1="10" y1="50" x2="90" y2="50" gradientUnits="userSpaceOnUse">
+      <stop stop-color="#22c55e" />
+      <stop offset="1" stop-color="#38bdf8" />
+    </linearGradient>
+  </defs>
+  <circle cx="50" cy="50" r="34" stroke="rgba(255,255,255,0.16)" stroke-width="4" />
+  <ellipse cx="50" cy="50" rx="40" ry="18" stroke="url(#zt-logo)" stroke-width="4" />
+  <ellipse cx="50" cy="50" rx="18" ry="40" stroke="url(#zt-logo)" stroke-width="4" opacity="0.75" />
+  <circle cx="50" cy="50" r="6" fill="url(#zt-logo)" />
+  <path d="M40 52C35 52 32 49 32 44C32 39 35 36 40 36H48" stroke="white" stroke-width="6" stroke-linecap="round" />
+  <path d="M60 48C65 48 68 51 68 56C68 61 65 64 60 64H52" stroke="white" stroke-width="6" stroke-linecap="round" />
+  <path d="M44 50H56" stroke="rgba(255,255,255,0.22)" stroke-width="6" stroke-linecap="round" />
+</svg>