@orkify/cli 1.0.0-beta.5

package/dist/telemetry/TelemetryReporter.js

@@ -0,0 +1,451 @@
+import { EventEmitter } from 'node:events';
+import { arch, cpus, hostname, platform, totalmem } from 'node:os';
+import { getAgentName } from '../agent-name.js';
+import { TELEMETRY_FLUSH_TIMEOUT, TELEMETRY_LOG_FLUSH_MAX_LINES, TELEMETRY_LOG_MAX_LINE_LENGTH, TELEMETRY_LOG_RING_SIZE, TELEMETRY_MAX_BATCH_SIZE, TELEMETRY_METRICS_INTERVAL, TELEMETRY_REQUEST_TIMEOUT, } from '../constants.js';
+import { getMachineId } from '../machine-id.js';
+import { computeFingerprint, parseFunctionName } from '../probe/compute-fingerprint.js';
+import { resolveSourceMaps } from '../probe/resolve-sourcemaps.js';
+export class TelemetryReporter extends EventEmitter {
+    config;
+    orchestrator;
+    events = [];
+    metrics = [];
+    errors = [];
+    logRings = new Map();
+    logFlushBuffer = [];
+    logFlushDropped = 0;
+    timer = null;
+    hostName;
+    agentName;
+    machineId;
+    hostInfo;
+    _deployStatus = null;
+    configStore;
+    alertEvaluator;
+    mcpCapable = false;
+    /** Previous cumulative cache counters per worker — keyed by "processName:workerId" */
+    prevCacheCounters = new Map();
+    constructor(config, orchestrator, configStore, alertEvaluator) {
+        super();
+        this.config = config;
+        this.orchestrator = orchestrator;
+        this.configStore = configStore ?? null;
+        this.alertEvaluator = alertEvaluator ?? null;
+        this.hostName = hostname();
+        this.agentName = getAgentName();
+        this.machineId = getMachineId();
+        this.hostInfo = {
+            os: platform(),
+            arch: arch(),
+            nodeVersion: process.version,
+            cpuCount: cpus().length,
+            totalMemory: totalmem(),
+        };
+    }
+    start() {
+        this.bindEvents();
+        this.timer = setInterval(() => {
+            void this.collectAndFlush();
+        }, TELEMETRY_METRICS_INTERVAL);
+        this.timer.unref();
+    }
+    async shutdown() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+        await Promise.race([
+            this.collectAndFlush(),
+            new Promise((resolve) => setTimeout(resolve, TELEMETRY_FLUSH_TIMEOUT).unref()),
+        ]);
+    }
+    bindEvents() {
+        this.orchestrator.on('process:start', (data) => {
+            this.pushEvent('process:start', data.processName, { processId: data.processId });
+        });
+        this.orchestrator.on('process:stop', (data) => {
+            this.pushEvent('process:stop', data.processName, { processId: data.processId });
+        });
+        this.orchestrator.on('reload:start', (data) => {
+            this.pushEvent('process:reload', data.processName, { processId: data.processId });
+        });
+        this.orchestrator.on('reload:complete', (data) => {
+            this.pushEvent('process:reloaded', data.processName, { processId: data.processId });
+        });
+        this.orchestrator.on('worker:ready', (data) => {
+            this.pushEvent('worker:ready', data.processName, { workerId: data.workerId });
+        });
+        this.orchestrator.on('worker:exit', (data) => {
+            if (data.code !== 0 && data.code !== null) {
+                const lastLogs = this.getWorkerLogs(data.processName, data.workerId);
+                this.pushEvent('worker:crash', data.processName, {
+                    workerId: data.workerId,
+                    exitCode: data.code,
+                    signal: data.signal,
+                    lastLogs,
+                });
+                this.clearWorkerLogs(data.processName, data.workerId);
+            }
+            else {
+                this.pushEvent('worker:exit', data.processName, {
+                    workerId: data.workerId,
+                    exitCode: data.code,
+                    signal: data.signal,
+                });
+            }
+        });
+        this.orchestrator.on('worker:maxRestarts', (data) => {
+            this.pushEvent('worker:maxRestarts', data.processName, { workerId: data.workerId });
+        });
+        this.orchestrator.on('worker:memoryRestart', (data) => {
+            this.pushEvent('worker:memoryRestart', data.processName, {
+                workerId: data.workerId,
+                details: { memory: data.memory, limit: data.limit },
+            });
+        });
+        this.orchestrator.on('log', (data) => {
+            // Ring buffer for crash/error context (last N lines per worker)
+            let processRings = this.logRings.get(data.processName);
+            if (!processRings) {
+                processRings = new Map();
+                this.logRings.set(data.processName, processRings);
+            }
+            let ring = processRings.get(data.workerId);
+            if (!ring) {
+                ring = [];
+                processRings.set(data.workerId, ring);
+            }
+            ring.push(data.data);
+            if (ring.length > TELEMETRY_LOG_RING_SIZE) {
+                ring.shift();
+            }
+            // Flush buffer for log ingestion (drained each flush cycle)
+            // Skip primary process logs (workerId -1) — they're internal ClusterWrapper messages
+            if (data.workerId < 0)
+                return;
+            const line = data.data.length > TELEMETRY_LOG_MAX_LINE_LENGTH
+                ? data.data.slice(0, TELEMETRY_LOG_MAX_LINE_LENGTH) + '...'
+                : data.data;
+            // Detect level: stderr defaults to error, but downgrade to warn if the
+            // line contains warn/warning without error (e.g. console.warn output)
+            let level = data.type === 'err' ? 'error' : 'info';
+            if (level === 'error') {
+                const prefix = line.slice(0, 80).toLowerCase();
+                if (/\bwarn(ing)?\b/.test(prefix) && !/\berror\b/.test(prefix)) {
+                    level = 'warn';
+                }
+            }
+            this.logFlushBuffer.push({
+                processName: data.processName,
+                workerId: data.workerId,
+                timestamp: Date.now(),
+                level,
+                message: line,
+            });
+            // If buffer grows too large between flushes, start dropping oldest
+            const maxBuffer = TELEMETRY_LOG_FLUSH_MAX_LINES * 10;
+            if (this.logFlushBuffer.length > maxBuffer) {
+                const excess = this.logFlushBuffer.length - maxBuffer;
+                this.logFlushBuffer.splice(0, excess);
+                this.logFlushDropped += excess;
+            }
+        });
+        this.orchestrator.on('worker:error:captured', (data) => {
+            const err = data.error;
+            const lastLogs = this.getWorkerLogs(data.processName, data.workerId);
+            const message = err.message || '';
+            const errorEvent = {
+                processName: data.processName,
+                workerId: data.workerId,
+                timestamp: err.timestamp || Date.now(),
+                errorType: err.errorType,
+                name: err.name || 'Error',
+                message,
+                stack: err.stack || '',
+                fingerprint: err.fingerprint || '',
+                sourceContext: err.sourceContext || null,
+                topFrame: err.topFrame || null,
+                diagnostics: err.diagnostics || null,
+                nodeVersion: err.nodeVersion || '',
+                pid: err.pid || 0,
+                lastLogs,
+                url: err.url || undefined,
+                userAgent: err.userAgent || undefined,
+            };
+            // Resolve source maps for bundled/minified code
+            const originalTopFrame = errorEvent.topFrame;
+            const resolved = resolveSourceMaps(errorEvent.sourceContext, errorEvent.topFrame);
+            errorEvent.sourceContext = resolved.sourceContext;
+            errorEvent.topFrame = resolved.topFrame;
+            errorEvent.resolved = resolved.resolved;
+            // Compute stable fingerprint: prefer source map function name, fall back to raw stack
+            let functionName = resolved.resolvedFunctionName;
+            if (!functionName && originalTopFrame) {
+                functionName = parseFunctionName(errorEvent.stack, originalTopFrame.file, originalTopFrame.line);
+            }
+            const effectiveTop = errorEvent.topFrame;
+            errorEvent.fingerprint = computeFingerprint({
+                errorName: errorEvent.name,
+                message,
+                file: effectiveTop?.file ?? '',
+                line: effectiveTop?.line ?? 0,
+                functionName,
+            });
+            this.errors.push(errorEvent);
+            if (this.errors.length >= TELEMETRY_MAX_BATCH_SIZE) {
+                void this.collectAndFlush();
+            }
+        });
+    }
+    /**
+     * Get logs for a specific worker. In cluster mode, worker stdout/stderr flows
+     * through the primary process (workerId -1), so if the worker-specific ring
+     * is empty we fall back to the primary's ring.
+     */
+    getWorkerLogs(processName, workerId) {
+        const processRings = this.logRings.get(processName);
+        if (!processRings)
+            return [];
+        const workerRing = processRings.get(workerId);
+        if (workerRing && workerRing.length > 0)
+            return workerRing.slice();
+        // Fallback: cluster primary ring (workerId -1) captures all worker output
+        const primaryRing = processRings.get(-1);
+        return primaryRing?.slice() ?? [];
+    }
+    clearWorkerLogs(processName, workerId) {
+        const processRings = this.logRings.get(processName);
+        if (!processRings)
+            return;
+        processRings.delete(workerId);
+        // Also clear primary ring on crash — it contained this worker's logs
+        if (workerId !== -1)
+            processRings.delete(-1);
+    }
+    pushEvent(type, processName, fields) {
+        this.events.push({
+            type,
+            processName,
+            timestamp: Date.now(),
+            ...fields,
+        });
+        if (this.events.length >= TELEMETRY_MAX_BATCH_SIZE) {
+            void this.collectAndFlush();
+        }
+    }
+    emitEvent(type, processName, fields) {
+        this.pushEvent(type, processName, fields);
+    }
+    setDeployStatus(status) {
+        this._deployStatus = status;
+    }
+    setMcpCapable(v) {
+        this.mcpCapable = v;
+    }
+    async collectAndFlush() {
+        // Collect metrics snapshot
+        const processList = this.orchestrator.list();
+        const now = Date.now();
+        const metricsSnapshots = processList.map((p) => ({
+            processName: p.name,
+            processId: p.id,
+            execMode: p.execMode,
+            status: p.status,
+            workers: p.workers.map((w) => {
+                const base = {
+                    id: w.id,
+                    pid: w.pid,
+                    cpu: w.cpu,
+                    memory: w.memory,
+                    uptime: w.uptime,
+                    restarts: w.restarts,
+                    crashes: w.crashes,
+                    status: w.status,
+                    stale: w.stale,
+                    heapUsed: w.heapUsed,
+                    heapTotal: w.heapTotal,
+                    external: w.external,
+                    arrayBuffers: w.arrayBuffers,
+                    eventLoopLag: w.eventLoopLag,
+                    eventLoopLagP95: w.eventLoopLagP95,
+                    activeHandles: w.activeHandles,
+                };
+                if (w.cacheSize === undefined)
+                    return base;
+                // Compute per-flush deltas for cumulative counters
+                const key = `${p.name}:${w.id}`;
+                const prev = this.prevCacheCounters.get(key);
+                const curHits = w.cacheHits ?? 0;
+                const curMisses = w.cacheMisses ?? 0;
+                const deltaHits = prev ? Math.max(0, curHits - prev.hits) : curHits;
+                const deltaMisses = prev ? Math.max(0, curMisses - prev.misses) : curMisses;
+                this.prevCacheCounters.set(key, { hits: curHits, misses: curMisses });
+                return {
+                    ...base,
+                    cacheSize: w.cacheSize,
+                    cacheTotalBytes: w.cacheTotalBytes,
+                    cacheHits: deltaHits,
+                    cacheMisses: deltaMisses,
+                    cacheHitRate: w.cacheHitRate,
+                };
+            }),
+            timestamp: now,
+        }));
+        // Drain flush buffer — cap at TELEMETRY_LOG_FLUSH_MAX_LINES per worker
+        const logsToSend = this.drainLogFlushBuffer();
+        // Buffer swap
+        const eventsToSend = this.events;
+        const metricsToSend = [...this.metrics, ...metricsSnapshots];
+        const errorsToSend = this.errors;
+        this.events = [];
+        this.metrics = [];
+        this.errors = [];
+        // Drain alert events before the emptiness check
+        const alertsToSend = this.alertEvaluator?.drainAlerts() ?? [];
+        if (eventsToSend.length === 0 &&
+            metricsToSend.length === 0 &&
+            errorsToSend.length === 0 &&
+            logsToSend.length === 0 &&
+            alertsToSend.length === 0) {
+            return;
+        }
+        const daemonStatus = this.orchestrator.getDaemonStatus();
+        const payload = {
+            daemonPid: daemonStatus.pid,
+            daemonUptime: daemonStatus.uptime,
+            hostname: this.hostName,
+            agentName: this.agentName,
+            machineId: this.machineId,
+            host: this.hostInfo,
+            events: eventsToSend,
+            metrics: metricsToSend,
+            errors: errorsToSend,
+            logs: logsToSend,
+            sentAt: Date.now(),
+        };
+        // Include config hash for sync
+        const configHash = this.configStore?.getHash();
+        if (configHash !== undefined) {
+            payload.configHash = configHash;
+        }
+        // Include MCP capability flag
+        if (this.mcpCapable) {
+            payload.mcpCapable = true;
+        }
+        // Include alert events
+        if (alertsToSend.length > 0) {
+            payload.alerts = alertsToSend;
+        }
+        // Include deploy status if active
+        if (this._deployStatus) {
+            payload.deployStatus = this._deployStatus;
+            const terminalPhases = ['success', 'failed', 'rolled_back'];
+            if (terminalPhases.includes(this._deployStatus.phase)) {
+                this._deployStatus = null; // Clear after terminal state is sent
+            }
+        }
+        try {
+            const response = await fetch(`${this.config.apiHost}/api/v1/ingest/telemetry`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${this.config.apiKey}`,
+                },
+                body: JSON.stringify(payload),
+                signal: AbortSignal.timeout(TELEMETRY_REQUEST_TIMEOUT),
+            });
+            if (!response.ok) {
+                const body = await response.text().catch(() => '');
+                console.error(`Telemetry flush failed: ${response.status} ${response.statusText} ${body}`);
+                this.restoreBuffers(eventsToSend, metricsToSend, errorsToSend, logsToSend, alertsToSend);
+            }
+            else {
+                // Parse response for pending commands and config sync
+                try {
+                    const responseBody = (await response.json());
+                    if (responseBody.has_commands) {
+                        this.emit('commands:pending');
+                    }
+                    if (responseBody.config && responseBody.config_hash) {
+                        this.configStore?.update(responseBody.config, responseBody.config_hash);
+                    }
+                }
+                catch {
+                    // Ignore JSON parse errors on response
+                }
+            }
+        }
+        catch (err) {
+            console.error('Telemetry flush error:', err instanceof Error ? err.message : err);
+            this.restoreBuffers(eventsToSend, metricsToSend, errorsToSend, logsToSend, alertsToSend);
+        }
+    }
+    /**
+     * Drain the log flush buffer, capping at TELEMETRY_LOG_FLUSH_MAX_LINES per
+     * worker. When lines are dropped, prepend a truncation marker.
+     */
+    drainLogFlushBuffer() {
+        const buffer = this.logFlushBuffer;
+        const dropped = this.logFlushDropped;
+        this.logFlushBuffer = [];
+        this.logFlushDropped = 0;
+        if (buffer.length === 0)
+            return [];
+        // Group by processName:workerId, keep the last N lines per worker
+        const groups = new Map();
+        for (const entry of buffer) {
+            const key = `${entry.processName}:${entry.workerId}`;
+            let group = groups.get(key);
+            if (!group) {
+                group = [];
+                groups.set(key, group);
+            }
+            group.push(entry);
+        }
+        const result = [];
+        for (const [, entries] of groups) {
+            if (entries.length > TELEMETRY_LOG_FLUSH_MAX_LINES) {
+                const excess = entries.length - TELEMETRY_LOG_FLUSH_MAX_LINES;
+                const kept = entries.slice(-TELEMETRY_LOG_FLUSH_MAX_LINES);
+                // Prepend truncation marker
+                const first = kept[0];
+                result.push({
+                    processName: first.processName,
+                    workerId: first.workerId,
+                    timestamp: first.timestamp - 1,
+                    level: 'info',
+                    message: `\u22EF ${excess} log lines truncated`,
+                });
+                result.push(...kept);
+            }
+            else {
+                result.push(...entries);
+            }
+        }
+        // If there were globally dropped lines (buffer overflow between flushes)
+        if (dropped > 0 && result.length > 0) {
+            const first = result[0];
+            result.unshift({
+                processName: first.processName,
+                workerId: first.workerId,
+                timestamp: first.timestamp - 1,
+                level: 'info',
+                message: `\u22EF ${dropped} log lines dropped (buffer overflow)`,
+            });
+        }
+        return result;
+    }
+    restoreBuffers(events, metrics, errors, logs, alerts) {
+        const maxSize = TELEMETRY_MAX_BATCH_SIZE * 2;
+        this.events = [...events, ...this.events].slice(-maxSize);
+        this.metrics = [...metrics, ...this.metrics].slice(-maxSize);
+        this.errors = [...errors, ...this.errors].slice(-maxSize);
+        // Restore logs back to flush buffer (prepend so they'll be sent first next time)
+        this.logFlushBuffer = [...logs, ...this.logFlushBuffer].slice(-maxSize);
+        // Re-queue unsent alerts back into the evaluator's buffer
+        if (alerts && alerts.length > 0 && this.alertEvaluator) {
+            this.alertEvaluator.restoreAlerts(alerts);
+        }
+    }
+}
+//# sourceMappingURL=TelemetryReporter.js.map