@tenex-chat/backend 0.9.1

package/src/telemetry/ConversationSpanManager.ts

@@ -0,0 +1,111 @@
+import { logger } from "@/utils/logger";
+import type { Span } from "@opentelemetry/api";
+
+/**
+ * Manages conversation-level metadata for tracing.
+ *
+ * Instead of long-lived parent spans, this tracks conversation message counts
+ * and adds conversation.id attributes to all spans. This allows querying Jaeger
+ * for all traces with the same conversation.id to see the full timeline.
+ *
+ * NOTE: Conversation IDs in Jaeger spans are shortened to 12 characters (PREFIX_LENGTH)
+ * for better readability in the Jaeger UI with low collision risk.
+ * Use shortenConversationId() from @/utils/conversation-id for consistency.
+ *
+ * Benefits:
+ * - No long-lived spans (avoids OTEL issues with spans that never end)
+ * - Immediate visibility in Jaeger (spans export as soon as they complete)
+ * - Easy querying: Search for conversation.id tag in Jaeger
+ * - Readable UI: Shortened IDs reduce visual clutter
+ */
+export class ConversationSpanManager {
+    private conversationMessageCounts = new Map<string, number>();
+
+    constructor() {
+        this.startCleanupTimer();
+    }
+
+    /**
+     * Increment message count for a conversation and add to span attributes
+     */
+    incrementMessageCount(conversationId: string, span: Span): void {
+        const currentCount = (this.conversationMessageCounts.get(conversationId) || 0) + 1;
+        this.conversationMessageCounts.set(conversationId, currentCount);
+
+        span.setAttributes({
+            "conversation.message_sequence": currentCount,
+        });
+
+        logger.debug("Incremented conversation message count", {
+            conversationId: conversationId.substring(0, 8),
+            messageSequence: currentCount,
+        });
+    }
+
+    /**
+     * Get total message count for a conversation
+     */
+    getMessageCount(conversationId: string): number {
+        return this.conversationMessageCounts.get(conversationId) || 0;
+    }
+
+    /**
+     * Start automatic cleanup timer (not used in this implementation but kept for future)
+     */
+    private startCleanupTimer(): void {
+        // Cleanup not needed with attribute-based approach
+        // Message counts are lightweight and don't need cleanup
+    }
+
+    /**
+     * Shutdown (no-op for this implementation)
+     */
+    shutdown(): void {
+        logger.info("Shutting down ConversationSpanManager", {
+            trackedConversations: this.conversationMessageCounts.size,
+        });
+        this.conversationMessageCounts.clear();
+    }
+
+    /**
+     * Get stats about tracked conversations
+     */
+    getStats(): {
+        trackedConversations: number;
+        totalMessages: number;
+    } {
+        let totalMessages = 0;
+
+        for (const count of this.conversationMessageCounts.values()) {
+            totalMessages += count;
+        }
+
+        return {
+            trackedConversations: this.conversationMessageCounts.size,
+            totalMessages,
+        };
+    }
+}
+
+// Singleton instance
+let conversationSpanManager: ConversationSpanManager | null = null;
+
+/**
+ * Get or create the conversation span manager instance
+ */
+export function getConversationSpanManager(): ConversationSpanManager {
+    if (!conversationSpanManager) {
+        conversationSpanManager = new ConversationSpanManager();
+    }
+    return conversationSpanManager;
+}
+
+/**
+ * Reset the conversation span manager (for testing)
+ */
+export function resetConversationSpanManager(): void {
+    if (conversationSpanManager) {
+        conversationSpanManager.shutdown();
+    }
+    conversationSpanManager = null;
+}

package/src/telemetry/EventLoopMonitor.ts

@@ -0,0 +1,206 @@
+/**
+ * EventLoopMonitor - Tracks event loop lag to identify blocking operations
+ *
+ * This module monitors the Node.js event loop for blocking/slow operations
+ * that could cause unresponsiveness when multiple agents are streaming.
+ *
+ * DIAGNOSTIC: This is part of the concurrent streaming bottleneck investigation.
+ */
+
+import { trace } from "@opentelemetry/api";
+import { logger } from "@/utils/logger";
+
+interface LagSample {
+    timestamp: number;
+    lagMs: number;
+    activeOperations: number;
+}
+
+interface EventLoopStats {
+    samples: LagSample[];
+    peakLagMs: number;
+    avgLagMs: number;
+    sampleCount: number;
+    blockedCount: number; // samples where lag > threshold
+}
+
+const DEFAULT_SAMPLE_INTERVAL_MS = 100; // Sample every 100ms
+const DEFAULT_LAG_THRESHOLD_MS = 50; // Consider >50ms as "blocked"
+const MAX_SAMPLES = 1000; // Keep last 1000 samples
+
+class EventLoopMonitor {
+    private static instance: EventLoopMonitor;
+    private isRunning = false;
+    private samples: LagSample[] = [];
+    private peakLagMs = 0;
+    private totalLagMs = 0;
+    private sampleCount = 0;
+    private blockedCount = 0;
+    private intervalHandle: ReturnType<typeof setInterval> | null = null;
+    private lastCheckTime = 0;
+    private getActiveOperationsCount: () => number = () => 0;
+
+    private constructor() {}
+
+    static getInstance(): EventLoopMonitor {
+        if (!EventLoopMonitor.instance) {
+            EventLoopMonitor.instance = new EventLoopMonitor();
+        }
+        return EventLoopMonitor.instance;
+    }
+
+    /**
+     * Start monitoring the event loop.
+     * @param getActiveOperationsCount - Function to get current active LLM operations count
+     * @param sampleIntervalMs - How often to sample (default 100ms)
+     * @param lagThresholdMs - What lag value is considered "blocked" (default 50ms)
+     */
+    start(
+        getActiveOperationsCount: () => number,
+        sampleIntervalMs = DEFAULT_SAMPLE_INTERVAL_MS,
+        lagThresholdMs = DEFAULT_LAG_THRESHOLD_MS
+    ): void {
+        if (this.isRunning) {
+            return;
+        }
+
+        this.getActiveOperationsCount = getActiveOperationsCount;
+        this.isRunning = true;
+        this.lastCheckTime = Date.now();
+
+        // Use setInterval but measure actual elapsed time vs expected
+        this.intervalHandle = setInterval(() => {
+            const now = Date.now();
+            const expectedElapsed = sampleIntervalMs;
+            const actualElapsed = now - this.lastCheckTime;
+            const lag = actualElapsed - expectedElapsed;
+
+            // Only record positive lag (negative would mean timer fired early)
+            const lagMs = Math.max(0, lag);
+            const activeOps = this.getActiveOperationsCount();
+
+            const sample: LagSample = {
+                timestamp: now,
+                lagMs,
+                activeOperations: activeOps,
+            };
+
+            this.samples.push(sample);
+            if (this.samples.length > MAX_SAMPLES) {
+                this.samples.shift();
+            }
+
+            this.sampleCount++;
+            this.totalLagMs += lagMs;
+            if (lagMs > this.peakLagMs) {
+                this.peakLagMs = lagMs;
+            }
+            if (lagMs > lagThresholdMs) {
+                this.blockedCount++;
+
+                // Log significant lag events
+                if (lagMs > lagThresholdMs * 2) {
+                    const activeSpan = trace.getActiveSpan();
+                    activeSpan?.addEvent("event_loop.significant_lag", {
+                        "lag.ms": lagMs,
+                        "lag.threshold_ms": lagThresholdMs,
+                        "concurrent.active_operations": activeOps,
+                        "process.memory_heap_used_mb": Math.round(
+                            process.memoryUsage().heapUsed / 1024 / 1024
+                        ),
+                    });
+
+                    logger.warn("[EventLoopMonitor] Significant event loop lag detected", {
+                        lagMs,
+                        activeOperations: activeOps,
+                        threshold: lagThresholdMs,
+                    });
+                }
+            }
+
+            this.lastCheckTime = now;
+        }, sampleIntervalMs);
+
+        logger.info("[EventLoopMonitor] Started monitoring", {
+            sampleIntervalMs,
+            lagThresholdMs,
+        });
+    }
+
+    /**
+     * Stop monitoring
+     */
+    stop(): void {
+        if (!this.isRunning || !this.intervalHandle) {
+            return;
+        }
+
+        clearInterval(this.intervalHandle);
+        this.intervalHandle = null;
+        this.isRunning = false;
+
+        logger.info("[EventLoopMonitor] Stopped monitoring");
+    }
+
+    /**
+     * Get current statistics
+     */
+    getStats(): EventLoopStats {
+        return {
+            samples: [...this.samples],
+            peakLagMs: this.peakLagMs,
+            avgLagMs: this.sampleCount > 0 ? this.totalLagMs / this.sampleCount : 0,
+            sampleCount: this.sampleCount,
+            blockedCount: this.blockedCount,
+        };
+    }
+
+    /**
+     * Get recent samples where lag was above threshold
+     */
+    getRecentBlockedSamples(thresholdMs = DEFAULT_LAG_THRESHOLD_MS, maxCount = 10): LagSample[] {
+        return this.samples
+            .filter((s) => s.lagMs > thresholdMs)
+            .slice(-maxCount);
+    }
+
+    /**
+     * Emit current stats as an OTL span event
+     */
+    emitStatsEvent(contextLabel: string): void {
+        const stats = this.getStats();
+        const activeSpan = trace.getActiveSpan();
+
+        activeSpan?.addEvent("event_loop.stats_snapshot", {
+            "stats.context": contextLabel,
+            "stats.peak_lag_ms": stats.peakLagMs,
+            "stats.avg_lag_ms": Math.round(stats.avgLagMs * 100) / 100,
+            "stats.sample_count": stats.sampleCount,
+            "stats.blocked_count": stats.blockedCount,
+            "stats.blocked_percentage":
+                stats.sampleCount > 0
+                    ? Math.round((stats.blockedCount / stats.sampleCount) * 10000) / 100
+                    : 0,
+        });
+    }
+
+    /**
+     * Reset all metrics (for testing)
+     */
+    reset(): void {
+        this.samples = [];
+        this.peakLagMs = 0;
+        this.totalLagMs = 0;
+        this.sampleCount = 0;
+        this.blockedCount = 0;
+    }
+
+    /**
+     * Check if monitoring is currently active
+     */
+    isActive(): boolean {
+        return this.isRunning;
+    }
+}
+
+export const eventLoopMonitor = EventLoopMonitor.getInstance();

package/src/telemetry/LLMSpanRegistry.ts

@@ -0,0 +1,20 @@
+/**
+ * Registry to track LLM execution span IDs by trace ID.
+ * Allows published events to reference the LLM span that generated their content.
+ *
+ * When an ai.streamText.doStream span ends, its span ID is stored here keyed by trace ID.
+ * When publishing events, we look up the LLM span ID to link directly to the LLM execution.
+ */
+const llmSpansByTrace = new Map<string, string>();
+
+export function setLLMSpanId(traceId: string, spanId: string): void {
+    llmSpansByTrace.set(traceId, spanId);
+}
+
+export function getLLMSpanId(traceId: string): string | undefined {
+    return llmSpansByTrace.get(traceId);
+}
+
+export function clearLLMSpanId(traceId: string): void {
+    llmSpansByTrace.delete(traceId);
+}

package/src/telemetry/NostrSpanProcessor.ts

@@ -0,0 +1,89 @@
+import type { Context } from "@opentelemetry/api";
+import type { ReadableSpan, Span, SpanProcessor } from "@opentelemetry/sdk-trace-base";
+
+/**
+ * Internal OTEL Span structure - accessing private fields to rewrite span IDs
+ * This is necessary because OTEL doesn't expose a way to set spanId after creation
+ */
+interface SpanInternals {
+    _spanContext?: { spanId: string };
+    _parentSpanId?: string;
+    _parentSpanContext?: { spanId: string };
+    parentSpanContext?: { spanId: string };
+}
+
+/**
+ * Convert a Nostr hex ID to OpenTelemetry spanID (16 hex chars)
+ */
+function nostrIdToSpanId(nostrId: string): string {
+    return nostrId.substring(0, 16);
+}
+
+/**
+ * SpanProcessor that rewrites span IDs for Nostr event processing spans.
+ *
+ * Problem: OpenTelemetry generates random spanIds, but we derive parentSpanId
+ * from Nostr event IDs (e-tags). This causes parent-child relationships to break
+ * because the random spanId doesn't match the derived parentSpanId.
+ *
+ * Solution: For spans with event.id attribute, rewrite their spanId to be
+ * deterministically derived from the event ID. This ensures that when another
+ * span references this event via e-tag, the parentSpanId will match.
+ */
+export class NostrSpanProcessor implements SpanProcessor {
+    forceFlush(): Promise<void> {
+        return Promise.resolve();
+    }
+
+    shutdown(): Promise<void> {
+        return Promise.resolve();
+    }
+
+    onStart(_span: Span, _parentContext: Context): void {
+        // Nothing to do on start
+    }
+
+    onEnd(span: ReadableSpan): void {
+        // Only rewrite spanIDs for tenex.event.process spans
+        // Other spans (tenex.dispatch.chat_message, tenex.delegation.completion_check) share
+        // event.id attribute but should NOT have their spanIDs rewritten to avoid collisions
+        if (span.name !== "tenex.event.process") {
+            return;
+        }
+
+        // Only process spans that have event.id attribute (Nostr event processing spans)
+        const eventId = span.attributes["event.id"];
+        if (typeof eventId !== "string" || !eventId) {
+            return;
+        }
+
+        // Rewrite spanId to be derived from event.id
+        const derivedSpanId = nostrIdToSpanId(eventId);
+
+        // Access internal state - the Span class stores context in _spanContext
+        const spanInternal = span as unknown as SpanInternals;
+
+        // Modify the spanContext's spanId
+        if (spanInternal._spanContext) {
+            spanInternal._spanContext.spanId = derivedSpanId;
+        }
+
+        // Fix parentSpanId based on event.reply_to
+        const replyTo = span.attributes["event.reply_to"];
+        if (typeof replyTo === "string" && replyTo) {
+            // This is a reply - set parent to the event being replied to
+            const derivedParentSpanId = nostrIdToSpanId(replyTo);
+
+            // Modify the parent span context
+            if (spanInternal._parentSpanContext) {
+                spanInternal._parentSpanContext.spanId = derivedParentSpanId;
+            } else if (spanInternal.parentSpanContext) {
+                spanInternal.parentSpanContext.spanId = derivedParentSpanId;
+            }
+        } else {
+            // This is a root message - clear parent context so it becomes a true root span
+            spanInternal._parentSpanContext = undefined;
+            spanInternal.parentSpanContext = undefined;
+        }
+    }
+}

package/src/telemetry/ToolCallSpanProcessor.ts

@@ -0,0 +1,66 @@
+import type { Context } from "@opentelemetry/api";
+import type { ReadableSpan, SpanProcessor } from "@opentelemetry/sdk-trace-base";
+import type { Span } from "@opentelemetry/sdk-trace-base";
+import { setLLMSpanId } from "./LLMSpanRegistry";
+
+/**
+ * Span processor that enriches tool call span names with the actual tool name.
+ * Transforms generic "ai.toolCall" spans into "ai.toolCall.{toolName}" for better visibility.
+ */
+export class ToolCallSpanProcessor implements SpanProcessor {
+    onStart(_span: Span, _parentContext: Context): void {
+        // This is called when a span starts, but we can't modify the name here
+        // because the attributes might not be set yet
+    }
+
+    onEnd(span: ReadableSpan): void {
+        // Extract agent context if available
+        const agentName = span.attributes?.["agent.name"];
+        const agentSlug = span.attributes?.["agent.slug"];
+        const agentPrefix = agentSlug || agentName;
+
+        // Check if this is a tool call span
+        if (span.name === "ai.toolCall" && span.attributes) {
+            const toolName = span.attributes["ai.toolCall.name"];
+
+            if (toolName && typeof toolName === "string") {
+                // Include agent slug in the span name for better visibility
+                const prefix = agentPrefix ? `[${agentPrefix}] ` : "";
+                // ReadableSpan has readonly name, but we can mutate it via the underlying object
+                (span as ReadableSpan & { name: string }).name = `${prefix}ai.toolCall.${toolName}`;
+            }
+        }
+
+        // Also enhance other AI operation spans with more context if available
+        if (span.name === "ai.streamText" || span.name === "ai.generateText") {
+            const model = span.attributes?.["ai.model.id"];
+            if (model && typeof model === "string") {
+                const shortModel = model.split("/").pop() || model;
+                const prefix = agentPrefix ? `[${agentPrefix}] ` : "";
+                (span as ReadableSpan & { name: string }).name = `${prefix}${span.name}.${shortModel}`;
+            }
+        }
+
+        // Enhance agent execution spans
+        if (span.name === "tenex.agent.execute" && agentPrefix) {
+            const phase = span.attributes?.["conversation.phase"];
+            const phaseStr = phase ? `.${phase}` : "";
+            (span as ReadableSpan & { name: string }).name = `[${agentPrefix}] agent.execute${phaseStr}`;
+        }
+
+        // Capture ai.streamText.doStream span IDs for event linking
+        // This allows published events to reference the LLM span that generated their content
+        if (span.name === "ai.streamText.doStream") {
+            const spanContext = span.spanContext();
+            setLLMSpanId(spanContext.traceId, spanContext.spanId);
+        }
+    }
+
+    shutdown(): Promise<void> {
+        return Promise.resolve();
+    }
+
+    forceFlush(): Promise<void> {
+        return Promise.resolve();
+    }
+}

package/src/telemetry/diagnostics.ts

@@ -0,0 +1,27 @@
+/**
+ * Diagnostics Configuration - Feature flag for diagnostic instrumentation
+ *
+ * Controls whether detailed diagnostic telemetry is collected.
+ * Disable in production for performance; enable when investigating issues.
+ *
+ * Set TENEX_DIAGNOSTICS=true to enable all diagnostic instrumentation.
+ */
+
+/**
+ * Check if diagnostics are enabled via environment variable.
+ * Caches the result at module load time.
+ */
+export function isDiagnosticsEnabled(): boolean {
+    return process.env.TENEX_DIAGNOSTICS === "true";
+}
+
+/**
+ * Get the diagnostics enabled state (for logging/tracing).
+ */
+export function getDiagnosticsState(): { enabled: boolean; source: string } {
+    const enabled = isDiagnosticsEnabled();
+    return {
+        enabled,
+        source: enabled ? "TENEX_DIAGNOSTICS=true" : "default (disabled)",
+    };
+}

package/src/telemetry/setup.ts

@@ -0,0 +1,120 @@
+import type { ExportResult } from "@opentelemetry/core";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base";
+import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
+import {
+    SEMRESATTRS_SERVICE_NAME,
+    SEMRESATTRS_SERVICE_VERSION,
+} from "@opentelemetry/semantic-conventions";
+import { ToolCallSpanProcessor } from "./ToolCallSpanProcessor.js";
+import { NostrSpanProcessor } from "./NostrSpanProcessor.js";
+
+const DEFAULT_SERVICE_NAME = "tenex-daemon";
+const DEFAULT_ENDPOINT = "http://localhost:4318/v1/traces";
+
+class ErrorHandlingExporterWrapper implements SpanExporter {
+    private disabled = false;
+
+    constructor(
+        private traceExporter: OTLPTraceExporter,
+        private exporterUrl: string
+    ) {}
+
+    export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void {
+        // Once disabled, drop all spans silently
+        if (this.disabled) {
+            resultCallback({ code: 0 }); // ExportResultCode.SUCCESS
+            return;
+        }
+
+        this.traceExporter.export(spans, (result) => {
+            if (result.error && !this.disabled) {
+                const errorMessage = result.error?.message || String(result.error);
+                const isConnectionError = errorMessage.includes("ECONNREFUSED") || errorMessage.includes("connect");
+                if (isConnectionError) {
+                    console.warn(`[Telemetry] ⚠️  Collector not available at ${this.exporterUrl}`);
+                } else {
+                    console.error("[Telemetry] Export error:", errorMessage);
+                }
+                console.warn("[Telemetry] Disabling trace export");
+                this.disabled = true;
+            }
+            resultCallback(result);
+        });
+    }
+
+    shutdown(): Promise<void> {
+        return this.traceExporter.shutdown();
+    }
+}
+
+// Create a wrapper processor that enriches span names and fixes Nostr IDs before exporting
+class EnrichedBatchSpanProcessor extends BatchSpanProcessor {
+    private enricher = new ToolCallSpanProcessor();
+    private nostrProcessor = new NostrSpanProcessor();
+
+    onEnd(span: ReadableSpan): void {
+        // First fix Nostr span IDs (must happen before export)
+        this.nostrProcessor.onEnd(span);
+        // Then enrich tool call spans
+        this.enricher.onEnd(span);
+        // Then pass to batch processor
+        super.onEnd(span);
+    }
+}
+
+let sdk: NodeSDK | null = null;
+
+function createSDK(serviceName: string, wrappedExporter: SpanExporter): NodeSDK {
+    const resource = resourceFromAttributes({
+        [SEMRESATTRS_SERVICE_NAME]: serviceName,
+        [SEMRESATTRS_SERVICE_VERSION]: process.env.npm_package_version || "0.8.0",
+        "deployment.environment": process.env.NODE_ENV || "development",
+    });
+
+    const spanProcessor = new EnrichedBatchSpanProcessor(wrappedExporter, {
+        maxQueueSize: 2048,
+        maxExportBatchSize: 512,
+        scheduledDelayMillis: 5000, // Send every 5 seconds
+    });
+
+    return new NodeSDK({
+        resource,
+        spanProcessor,
+        // NO sampling - capture everything (100%)
+        // NO instrumentation filters - capture all
+    });
+}
+
+export function initializeTelemetry(
+    enabled = true,
+    serviceName = DEFAULT_SERVICE_NAME,
+    endpoint = DEFAULT_ENDPOINT
+): void {
+    if (!enabled) {
+        return;
+    }
+
+    // Use environment variable if set, otherwise use config endpoint
+    const exporterUrl = process.env.OTEL_EXPORTER_OTLP_ENDPOINT || endpoint;
+
+    // Create exporter with the configured URL
+    const traceExporter = new OTLPTraceExporter({
+        url: exporterUrl,
+    });
+
+    // Wrap the exporter with error handling
+    const wrappedExporter = new ErrorHandlingExporterWrapper(traceExporter, exporterUrl);
+
+    sdk = createSDK(serviceName, wrappedExporter);
+    sdk.start();
+}
+
+export function shutdownTelemetry(): Promise<void> {
+    if (!sdk) {
+        return Promise.resolve();
+    }
+    return sdk.shutdown();
+}