@zintrust/workers 0.1.28 → 0.1.30

package/dist/telemetry/api/TelemetryAPI.js

@@ -0,0 +1,219 @@
+import { Logger } from '@zintrust/core';
+import { getWorkers } from '../../dashboard/workers-api';
+import { HealthMonitor } from '../../HealthMonitor';
+import { ResourceMonitor } from '../../ResourceMonitor';
+const isOkWithUsage = (value) => value.ok === true && 'usage' in value;
+const isOkWithSummary = (value) => value.ok === true && 'summary' in value;
+// Helper function to create stopped worker alert
+const createStoppedWorkerAlert = (worker) => ({
+    type: 'worker-stopped',
+    severity: 'warning',
+    message: `Worker ${worker.name} is stopped`,
+    timestamp: worker.health?.lastCheck || new Date().toISOString(),
+});
+// Helper function to create health check alert
+const createHealthCheckAlert = (worker) => {
+    const check = worker.health?.checks?.[0];
+    if (!check)
+        return null;
+    return {
+        type: 'health-check-failed',
+        severity: check.status === 'fail' ? 'critical' : 'warning',
+        message: check.message || `Health check failed: ${check.name}`,
+        timestamp: worker.health?.lastCheck || new Date().toISOString(),
+    };
+};
+// Helper function to generate worker alerts
+const generateWorkerAlerts = (workers) => {
+    return workers
+        .filter((w) => {
+        return (w.status !== 'running' ||
+            w.health?.status !== 'healthy' ||
+            (w.health?.checks && w.health.checks.length > 0));
+    })
+        .map((w) => {
+        const workerData = {
+            workerName: w.name,
+            status: w.status,
+            healthStatus: w.health?.status || 'unknown',
+            lastCheck: w.health?.lastCheck || new Date().toISOString(),
+            checks: w.health?.checks || [],
+        };
+        let alert = null;
+        if (w.status === 'stopped') {
+            alert = createStoppedWorkerAlert(w);
+        }
+        else if (w.health?.checks && w.health.checks.length > 0) {
+            alert = createHealthCheckAlert(w);
+        }
+        return {
+            ...workerData,
+            alert,
+        };
+    })
+        .filter((w) => w.alert !== null);
+};
+// Helper function to generate resource alerts
+const generateResourceAlerts = (resourceUsage) => {
+    const resourceAlerts = [];
+    const cpuUsage = resourceUsage.resourceSnapshot?.cpu?.usage || 0;
+    const memoryUsage = resourceUsage.resourceSnapshot?.memory?.usage || 0;
+    if (cpuUsage > 90) {
+        resourceAlerts.push({
+            type: 'cpu-high',
+            severity: 'critical',
+            message: `Critical CPU usage: ${cpuUsage.toFixed(1)}%`,
+            timestamp: new Date().toISOString(),
+            recommendation: 'Consider scaling up or optimizing worker code',
+        });
+    }
+    else if (cpuUsage > 80) {
+        resourceAlerts.push({
+            type: 'cpu-high',
+            severity: 'warning',
+            message: `High CPU usage: ${cpuUsage.toFixed(1)}%`,
+            timestamp: new Date().toISOString(),
+            recommendation: 'Monitor closely and consider scaling',
+        });
+    }
+    if (memoryUsage > 95) {
+        resourceAlerts.push({
+            type: 'memory-high',
+            severity: 'critical',
+            message: `Critical memory usage: ${memoryUsage.toFixed(1)}%`,
+            timestamp: new Date().toISOString(),
+            recommendation: 'Increase memory allocation or optimize memory usage',
+        });
+    }
+    else if (memoryUsage > 85) {
+        resourceAlerts.push({
+            type: 'memory-high',
+            severity: 'warning',
+            message: `High memory usage: ${memoryUsage.toFixed(1)}%`,
+            timestamp: new Date().toISOString(),
+            recommendation: 'Monitor memory usage closely',
+        });
+    }
+    return resourceAlerts;
+};
+// Helper function to calculate monitoring summary
+const calculateMonitoringSummary = (runningWorkers) => {
+    const healthyCount = runningWorkers.filter((w) => w.health?.status === 'healthy').length;
+    const degradedCount = runningWorkers.filter((w) => w.health?.status === 'warning').length;
+    const criticalCount = runningWorkers.filter((w) => w.health?.status === 'unhealthy').length;
+    return {
+        total: runningWorkers.length,
+        healthy: healthyCount,
+        degraded: degradedCount,
+        critical: criticalCount,
+        details: runningWorkers.map((w) => ({
+            workerName: w.name,
+            status: w.health?.status || 'unknown',
+            lastCheck: w.health?.lastCheck || new Date().toISOString(),
+            checks: w.health?.checks || [],
+        })),
+    };
+};
+export const TelemetryAPI = Object.freeze({
+    async getSystemSummary() {
+        try {
+            // Get all workers (both running and stopped) for complete alert visibility
+            const workersResult = await getWorkers({});
+            const runningWorkers = workersResult.workers.filter((w) => w.status === 'running');
+            // Calculate monitoring summary from running workers only
+            const monitoringSummary = calculateMonitoringSummary(runningWorkers);
+            // Generate alerts from workers and resources
+            const workerAlerts = generateWorkerAlerts(workersResult.workers);
+            const resourceUsage = ResourceMonitor.getCurrentUsage('system');
+            const resourceAlerts = generateResourceAlerts(resourceUsage);
+            // Combine all alerts
+            const allAlerts = [...workerAlerts, ...resourceAlerts];
+            return {
+                ok: true,
+                summary: {
+                    workers: runningWorkers.length,
+                    monitoring: {
+                        ...monitoringSummary,
+                        alerts: allAlerts, // Include all alerts (workers + resources)
+                    },
+                    resources: resourceUsage,
+                    alerts: allAlerts, // Top-level alerts for easy access
+                },
+            };
+        }
+        catch (error) {
+            Logger.error('Failed to get system summary', error);
+            return {
+                ok: false,
+                error: error instanceof Error ? error.message : 'Unknown error',
+                summary: {},
+            };
+        }
+    },
+    async getMonitoringSummary() {
+        try {
+            const summary = await HealthMonitor.getSummary();
+            return { ok: true, summary };
+        }
+        catch (error) {
+            Logger.error('Failed to get monitoring summary', error);
+            return {
+                ok: false,
+                error: error instanceof Error ? error.message : 'Unknown error',
+                summary: {},
+            };
+        }
+    },
+    async getResourceCurrent() {
+        try {
+            const usage = ResourceMonitor.getCurrentUsage('system');
+            return { ok: true, usage };
+        }
+        catch (error) {
+            Logger.error('Failed to get resource usage', error);
+            return {
+                ok: false,
+                error: error instanceof Error ? error.message : 'Unknown error',
+                usage: null,
+            };
+        }
+    },
+    async getResourceTrends() {
+        try {
+            const trends = ResourceMonitor.getAllTrends('system', 'day');
+            return { ok: true, trends };
+        }
+        catch (error) {
+            Logger.error('Failed to get resource trends', error);
+            return {
+                ok: false,
+                error: error instanceof Error ? error.message : 'Unknown error',
+                trends: null,
+            };
+        }
+    },
+});
+export function createSnapshotBuilder() {
+    return async () => {
+        const [systemSummaryResult, resourceCurrentResult] = await Promise.allSettled([
+            TelemetryAPI.getSystemSummary(),
+            TelemetryAPI.getResourceCurrent(),
+        ]);
+        if (systemSummaryResult.status === 'rejected') {
+            Logger.error('Telemetry dashboard summary failed', systemSummaryResult.reason);
+        }
+        if (resourceCurrentResult.status === 'rejected') {
+            Logger.error('Telemetry resource summary failed', resourceCurrentResult.reason);
+        }
+        const systemSummary = systemSummaryResult.status === 'fulfilled' ? systemSummaryResult.value : { ok: false };
+        const resourceCurrent = resourceCurrentResult.status === 'fulfilled'
+            ? resourceCurrentResult.value
+            : { ok: false };
+        return {
+            ok: systemSummary.ok ?? false,
+            summary: isOkWithSummary(systemSummary) ? systemSummary.summary : {},
+            resources: isOkWithUsage(resourceCurrent) ? resourceCurrent.usage : null,
+            cost: null,
+        };
+    };
+}

package/dist/telemetry/api/TelemetryMonitoringService.d.ts

@@ -0,0 +1,17 @@
+import type { IResponse } from '@zintrust/core';
+import type { createSnapshotBuilder, TelemetrySettings } from './TelemetryAPI';
+export type TelemetrySnapshotData = {
+    type: string;
+    ts: string;
+    ok: boolean;
+    summary: unknown;
+    resources: unknown;
+    cost: unknown;
+};
+export declare const TelemetryMonitoringService: Readonly<{
+    subscribe(callback: (data: TelemetrySnapshotData) => void): void;
+    unsubscribe(callback: (data: TelemetrySnapshotData) => void): void;
+    startMonitoring(settings: TelemetrySettings, buildSnapshot: ReturnType<typeof createSnapshotBuilder>): void;
+    stopMonitoring(): void;
+}>;
+export declare const teleStream: (res: IResponse, settings: TelemetrySettings, buildSnapshot: ReturnType<typeof createSnapshotBuilder>) => Promise<void>;

package/dist/telemetry/api/TelemetryMonitoringService.js

@@ -0,0 +1,113 @@
+import { Logger, NodeSingletons } from '@zintrust/core';
+// Internal state for singleton service
+const emitter = new NodeSingletons.EventEmitter();
+emitter.setMaxListeners(Infinity);
+let interval = null;
+let subscribers = 0;
+let currentSettings = null;
+let currentBuildSnapshot = null;
+const broadcastTelemetrySnapshot = async () => {
+    try {
+        if (subscribers <= 0 || !currentBuildSnapshot || !currentSettings)
+            return;
+        const snapshot = await currentBuildSnapshot();
+        const payload = {
+            type: 'snapshot',
+            ts: new Date().toISOString(),
+            ...snapshot,
+        };
+        emitter.emit('snapshot', payload);
+    }
+    catch (err) {
+        Logger.error('TelemetryMonitoringService.broadcastSnapshot failed', err);
+        emitter.emit('error', err);
+    }
+};
+const startPolling = () => {
+    if (interval || !currentSettings)
+        return;
+    Logger.debug('Starting TelemetryMonitoringService polling');
+    // Initial fetch
+    void broadcastTelemetrySnapshot();
+    interval = setInterval(() => {
+        void broadcastTelemetrySnapshot();
+    }, currentSettings.refreshIntervalMs);
+};
+const stopPolling = () => {
+    if (interval) {
+        Logger.debug('Stopping TelemetryMonitoringService polling');
+        clearInterval(interval);
+        interval = null;
+    }
+};
+export const TelemetryMonitoringService = Object.freeze({
+    subscribe(callback) {
+        emitter.on('snapshot', callback);
+        subscribers++;
+        if (subscribers === 1) {
+            startPolling();
+        }
+    },
+    unsubscribe(callback) {
+        emitter.off('snapshot', callback);
+        subscribers--;
+        if (subscribers <= 0) {
+            stopPolling();
+            currentSettings = null;
+            currentBuildSnapshot = null;
+        }
+    },
+    startMonitoring(settings, buildSnapshot) {
+        if (subscribers === 0) {
+            currentSettings = settings;
+            currentBuildSnapshot = buildSnapshot;
+        }
+    },
+    stopMonitoring() {
+        if (subscribers <= 0) {
+            stopPolling();
+            currentSettings = null;
+            currentBuildSnapshot = null;
+        }
+    },
+});
+export const teleStream = async (res, settings, buildSnapshot) => {
+    const raw = res.getRaw();
+    raw.writeHead(200, {
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache, no-transform',
+        Connection: 'keep-alive',
+        'X-Accel-Buffering': 'no',
+    });
+    let closed = false;
+    const send = async (payload) => {
+        try {
+            raw.write(`data: ${JSON.stringify(payload)}\n\n`);
+        }
+        catch (err) {
+            Logger.error('Telemetry SSE send failed', err);
+        }
+    };
+    // Send hello immediately
+    await send({ type: 'hello', ts: new Date().toISOString() });
+    // Start monitoring with the singleton service
+    TelemetryMonitoringService.startMonitoring(settings, buildSnapshot);
+    // Subscribe to telemetry snapshots
+    const onSnapshot = (data) => {
+        if (!closed) {
+            void send(data);
+        }
+    };
+    TelemetryMonitoringService.subscribe(onSnapshot);
+    // Heartbeat to keep connection alive
+    const hb = setInterval(() => {
+        if (!closed)
+            raw.write(': ping\n\n');
+    }, 15000);
+    raw.on('close', () => {
+        closed = true;
+        clearInterval(hb);
+        TelemetryMonitoringService.unsubscribe(onSnapshot);
+        TelemetryMonitoringService.stopMonitoring();
+    });
+};

package/dist/telemetry/components/AlertPanel.d.ts

@@ -0,0 +1 @@
+export declare const renderAlertPanel: () => string;

package/dist/telemetry/components/AlertPanel.js

@@ -0,0 +1,13 @@
+export const renderAlertPanel = () => {
+    return `
+    <div class="zt-card">
+      <div class="zt-card-header">
+        <h3 class="zt-card-title">Alert History</h3>
+        <span class="zt-card-meta">Latest events</span>
+      </div>
+      <ul id="alertList" class="zt-alert-list">
+        <li class="zt-alert-item">No alerts yet.</li>
+      </ul>
+    </div>
+  `;
+};

package/dist/telemetry/components/CostTracking.d.ts

@@ -0,0 +1 @@
+export declare const renderCostTracking: () => string;

package/dist/telemetry/components/CostTracking.js

@@ -0,0 +1,14 @@
+export const renderCostTracking = () => {
+    return `
+    <div class="zt-card">
+      <div class="zt-card-header">
+        <h3 class="zt-card-title">Cost Tracking</h3>
+        <span class="zt-card-meta">Daily estimate</span>
+      </div>
+      <div class="zt-card-body">
+        <p class="zt-cost-value" id="costTotal">$0.00</p>
+        <p class="zt-card-meta">Auto-updated from resource metrics</p>
+      </div>
+    </div>
+  `;
+};

package/dist/telemetry/components/ResourceUsageChart.d.ts

@@ -0,0 +1 @@
+export declare const renderResourceUsageChart: () => string;

package/dist/telemetry/components/ResourceUsageChart.js

@@ -0,0 +1,11 @@
+export const renderResourceUsageChart = () => {
+    return `
+    <div class="zt-card">
+      <div class="zt-card-header">
+        <h3 class="zt-card-title">Resource Usage</h3>
+        <span class="zt-card-meta">Current snapshot</span>
+      </div>
+      <canvas id="resourceUsageChart" class="zt-chart"></canvas>
+    </div>
+  `;
+};

package/dist/telemetry/components/WorkerHealthChart.d.ts

@@ -0,0 +1 @@
+export declare const renderWorkerHealthChart: () => string;

package/dist/telemetry/components/WorkerHealthChart.js

@@ -0,0 +1,11 @@
+export const renderWorkerHealthChart = () => {
+    return `
+    <div class="zt-card">
+      <div class="zt-card-header">
+        <h3 class="zt-card-title">Worker Health</h3>
+        <span class="zt-card-meta">Last 24h</span>
+      </div>
+      <canvas id="workerHealthChart" class="zt-chart"></canvas>
+    </div>
+  `;
+};

package/dist/telemetry/index.d.ts

@@ -0,0 +1,15 @@
+import type { IRouter } from '@zintrust/core';
+export type TelemetryDashboardConfig = {
+    enabled?: boolean;
+    basePath?: string;
+    middleware?: ReadonlyArray<string>;
+    autoRefresh?: boolean;
+    refreshIntervalMs?: number;
+};
+export type TelemetryDashboardApi = {
+    registerRoutes: (router: IRouter) => void;
+};
+export declare const TelemetryDashboard: Readonly<{
+    create(config: TelemetryDashboardConfig): TelemetryDashboardApi;
+}>;
+export default TelemetryDashboard;

package/dist/telemetry/index.js

@@ -0,0 +1,60 @@
+import { Router } from '@zintrust/core';
+import { createSnapshotBuilder } from './api/TelemetryAPI';
+import { teleStream } from './api/TelemetryMonitoringService';
+import { getDashboardHtml } from './routes/dashboard';
+const DEFAULTS = {
+    enabled: true,
+    basePath: '/telemetry',
+    autoRefresh: true,
+    refreshIntervalMs: 10000,
+};
+export const TelemetryDashboard = Object.freeze({
+    create(config) {
+        const settings = buildSettings(config);
+        const buildSnapshot = createSnapshotBuilder();
+        const registerRoutes = createRouteRegistrar(settings, buildSnapshot);
+        return Object.freeze({ registerRoutes });
+    },
+});
+function buildSettings(config) {
+    return {
+        enabled: config.enabled ?? DEFAULTS.enabled,
+        basePath: config.basePath ?? DEFAULTS.basePath,
+        middleware: config.middleware ?? [],
+        autoRefresh: config.autoRefresh ?? DEFAULTS.autoRefresh,
+        refreshIntervalMs: typeof config.refreshIntervalMs === 'number' && Number.isFinite(config.refreshIntervalMs)
+            ? Math.max(1000, Math.floor(config.refreshIntervalMs))
+            : DEFAULTS.refreshIntervalMs,
+    };
+}
+function createRouteRegistrar(settings, buildSnapshot) {
+    return (router) => {
+        if (!settings.enabled)
+            return;
+        const routeOptions = (settings.middleware.length > 0 ? { middleware: settings.middleware } : undefined);
+        registerDashboardRoute(router, settings, routeOptions);
+        registerSummaryApi(router, settings, routeOptions, buildSnapshot);
+        registerEventsApi(router, settings, routeOptions, buildSnapshot);
+    };
+}
+function registerDashboardRoute(router, settings, routeOptions) {
+    Router.get(router, settings.basePath, (_req, res) => {
+        res.html(getDashboardHtml({
+            basePath: settings.basePath,
+            autoRefresh: settings.autoRefresh,
+            refreshIntervalMs: settings.refreshIntervalMs,
+        }));
+    }, routeOptions);
+}
+function registerSummaryApi(router, settings, routeOptions, buildSnapshot) {
+    Router.get(router, `${settings.basePath}/api/summary`, async (_req, res) => {
+        const snapshot = await buildSnapshot();
+        res.json(snapshot);
+    }, routeOptions);
+}
+function registerEventsApi(router, settings, routeOptions, buildSnapshot) {
+    Router.get(router, `${settings.basePath}/api/events`, async (_req, res) => {
+        teleStream(res, settings, buildSnapshot);
+    }, routeOptions);
+}
+export default TelemetryDashboard;

package/dist/telemetry/routes/dashboard.d.ts

@@ -0,0 +1,6 @@
+export type DashboardUiOptions = {
+    basePath: string;
+    autoRefresh: boolean;
+    refreshIntervalMs: number;
+};
+export declare const getDashboardHtml: (options: DashboardUiOptions) => string;