@smythos/sre 1.7.42 → 1.8.1

package/src/subsystems/ObservabilityManager/Telemetry.service/connectors/OTel/OTel.class.ts

@@ -4,6 +4,7 @@ import { ACL } from '@sre/Security/AccessControl/ACL.class';
 import { IAccessCandidate } from '@sre/types/ACL.types';
 import { TelemetryConnector } from '../../TelemetryConnector';
 import { AgentCallLog } from '@sre/types/AgentLogger.types';
+import { redactSensitiveString, redactData, redactHeaders } from './OTel.redaction.helper';
 
 import { trace, context, SpanStatusCode, Tracer, propagation } from '@opentelemetry/api';
 import { Logger as OTelLogger, logs, SeverityNumber } from '@opentelemetry/api-logs';
@@ -50,6 +51,14 @@ export type OTelLogConfig = {
      * These will be replaced with '[REDACTED]' in logs
      */
     redactFields?: string[];
+    /**
+     * Enable automatic redaction of sensitive data in logs and traces.
+     * When true (or omitted), sensitive data such as passwords, tokens,
+     * API keys, and JWT tokens are automatically replaced with '[REDACTED]'.
+     * Set to false to disable all automatic redaction.
+     * Default: true
+     */
+    enableRedaction?: boolean;
 };
 const OTEL_DEBUG_LOGS = true;
 export class OTel extends TelemetryConnector {
@@ -62,6 +71,9 @@ export class OTel extends TelemetryConnector {
 
     constructor(protected _settings: OTelLogConfig) {
         super();
+        // Default enableRedaction to true when not explicitly provided
+        _settings.enableRedaction = _settings.enableRedaction ?? true;
+
         if (!_settings.endpoint) {
             outputLogger.warn('OTel initialization skipped, endpoint is not set');
             return;
@@ -141,6 +153,7 @@ export class OTel extends TelemetryConnector {
      * Redact sensitive fields from an object
      */
     private redactSensitiveData(data: any, redactFields?: string[]): any {
+        if (!this._settings.enableRedaction) return data;
         if (!redactFields || redactFields.length === 0) return data;
         if (typeof data !== 'object' || data === null) return data;
 
@@ -157,6 +170,35 @@ export class OTel extends TelemetryConnector {
         return redacted;
     }
 
+    /**
+     * Redact sensitive patterns from a string value.
+     * Skips redaction when enableRedaction is explicitly set to false.
+     */
+    private redactString(value: string): string {
+        if (!this._settings.enableRedaction) return value;
+        return redactSensitiveString(value);
+    }
+
+    /**
+     * Redact sensitive data from any data type (objects, arrays, strings).
+     * Skips redaction when enableRedaction is explicitly set to false.
+     */
+    private redactObject<T>(data: T): T {
+        if (!this._settings.enableRedaction) return data;
+        return redactData(data);
+    }
+
+    /**
+     * Redact sensitive HTTP headers.
+     * Skips redaction when enableRedaction is explicitly set to false.
+     */
+    private redactRequestHeaders(
+        headers: Record<string, unknown> | string | undefined | null,
+    ): Record<string, unknown> | string | undefined | null {
+        if (!this._settings.enableRedaction) return headers;
+        return redactHeaders(headers);
+    }
+
     /**
      * Safely format output for logging with size limits and redaction
      */
@@ -170,11 +212,17 @@ export class OTel extends TelemetryConnector {
             return undefined;
         }
 
-        // Redact sensitive fields
-        const redacted = this.redactSensitiveData(output, config.redactFields);
+        // Redact sensitive fields (config-based)
+        let redacted = this.redactSensitiveData(output, config.redactFields);
+
+        // Apply SENSITIVE_WORDS-based redaction on the object (automatic key-based redaction)
+        redacted = this.redactObject(redacted);
 
         // Stringify
-        const outputStr = JSON.stringify(redacted);
+        let outputStr = JSON.stringify(redacted);
+
+        // Apply string-based redaction on the stringified output to catch embedded JSON
+        outputStr = this.redactString(outputStr);
 
         // Check size limit
         if (outputStr && outputStr.length > maxSize) {
@@ -200,12 +248,38 @@ export class OTel extends TelemetryConnector {
         for (let key in data) {
             result[prefix ? `${prefix}.${key}` : key] = (typeof data[key] === 'object' ? JSON.stringify(data[key]) : data[key].toString()).substring(
                 0,
-                maxEntryLength
+                maxEntryLength,
             );
         }
 
         return result;
     }
+
+    private prepareContext(contextWindow: Array<{ role: string; content: string; [key: string]: unknown }>): string {
+        if (!contextWindow || !Array.isArray(contextWindow)) return '[]';
+
+        const filtered = contextWindow.filter((msg) => {
+            if (typeof msg !== 'object' || msg === null) return false;
+            const keys = Object.keys(msg);
+            return !keys.some((k) => k.includes('___smyth_metadata___'));
+        });
+
+        const lastAssistant = [...filtered].reverse().find((msg) => msg.role === 'assistant' && msg.content);
+        const lastUser = [...filtered].reverse().find((msg) => msg.role === 'user' && msg.content);
+
+        const messages: Array<{ role: string; content: string }> = [];
+        if (lastAssistant) {
+            const raw = typeof lastAssistant.content === 'string' ? lastAssistant.content : JSON.stringify(lastAssistant.content);
+            messages.push({ role: 'assistant', content: raw.substring(0, 2000) });
+        }
+        if (lastUser) {
+            const raw = typeof lastUser.content === 'string' ? lastUser.content : JSON.stringify(lastUser.content);
+            messages.push({ role: 'user', content: raw.substring(0, 2000) });
+        }
+
+        return JSON.stringify(messages);
+    }
+
     protected setupHooks(): Promise<void> {
         const tracer = this.tracer;
         const logger = this.logger;
@@ -219,15 +293,18 @@ export class OTel extends TelemetryConnector {
 
                 const modelId = toolInfo.model;
                 const contextWindow = toolInfo.contextWindow;
-                const lastContext = contextWindow.filter((context) => context.role === 'user').slice(-2);
 
-                const toolNames = toolInfo.map((tool) => tool.name + '(' + tool.arguments + ')');
+                const toolNames = toolInfo.map((tool) => {
+                    const args = typeof tool.arguments === 'string' ? tool.arguments : JSON.stringify(tool.arguments);
+                    return `${tool.name}(${args})`;
+                });
                 hookContext.curLLMGenSpan.addEvent('llm.gen.tool.calls', {
-                    'tool.calls': toolNames.join(', '),
-                    'llm.model': modelId || 'unknown',
-                    'context.preview': JSON.stringify(lastContext).substring(0, 200),
+                    'tool.calls': oTelInstance.redactString(toolNames.join(', ')),
+                    'llm.model': modelId || '',
+                    'context.preview': oTelInstance.redactString(oTelInstance.prepareContext(contextWindow).substring(0, 200)),
                 });
 
+                const llmSpanCtx = hookContext.curLLMGenSpan.spanContext();
                 const spanContext = trace.setSpan(context.active(), hookContext.curLLMGenSpan);
                 context.with(spanContext, () => {
                     logger.emit({
@@ -235,10 +312,15 @@ export class OTel extends TelemetryConnector {
                         severityText: 'INFO',
                         body: `LLM tool calls: ${toolNames.join(', ')}`,
                         attributes: {
+                            // Explicit trace correlation (some backends need these)
+                            trace_id: llmSpanCtx.traceId,
+                            span_id: llmSpanCtx.spanId,
+                            trace_flags: llmSpanCtx.traceFlags,
+
                             'agent.id': hookContext.agentId,
                             'conv.id': hookContext.processId,
-                            'llm.model': modelId || 'unknown',
-                            'context.preview': JSON.stringify(lastContext).substring(0, 5000),
+                            'llm.model': modelId || '',
+                            'context.preview': oTelInstance.redactString(oTelInstance.prepareContext(contextWindow)),
                         },
                     });
                 });
@@ -259,7 +341,6 @@ export class OTel extends TelemetryConnector {
                 const modelId = reqInfo.model;
                 const contextWindow = reqInfo.contextWindow;
 
-                const lastContext = contextWindow.filter((context) => context.role === 'user').slice(-2);
                 // End TTFB span when first data arrives
                 if (hookContext?.latencySpans?.[reqInfo.requestId]) {
                     const ttfbSpan = hookContext.latencySpans[reqInfo.requestId];
@@ -268,7 +349,7 @@ export class OTel extends TelemetryConnector {
                     ttfbSpan.addEvent('llm.first.byte.received', {
                         'request.id': reqInfo.requestId,
                         'data.size': JSON.stringify(data || {}).length,
-                        'llm.model': modelId || 'unknown',
+                        'llm.model': modelId || '',
                     });
 
                     ttfbSpan.setStatus({ code: SpanStatusCode.OK });
@@ -284,22 +365,98 @@ export class OTel extends TelemetryConnector {
                             'agent.id': hookContext.agentId,
                             'conv.id': hookContext.processId,
                             'team.id': hookContext.teamId,
-                            'llm.model': modelId || 'unknown',
+                            'llm.model': modelId || '',
                         },
                     },
-                    trace.setSpan(context.active(), hookContext.convSpan)
+                    trace.setSpan(context.active(), hookContext.convSpan),
                 );
                 llmGenSpan.addEvent('llm.gen.started', {
                     'request.id': reqInfo.requestId,
                     timestamp: Date.now(),
-                    'llm.model': modelId || 'unknown',
-                    'context.preview': JSON.stringify(lastContext).substring(0, 200),
+                    'llm.model': modelId || '',
+                    'context.preview': oTelInstance.redactString(oTelInstance.prepareContext(contextWindow).substring(0, 200)),
+                });
+
+                const llmGenSpanCtx = llmGenSpan.spanContext();
+                const llmGenSpanContext = trace.setSpan(context.active(), llmGenSpan);
+                context.with(llmGenSpanContext, () => {
+                    logger.emit({
+                        severityNumber: SeverityNumber.INFO,
+                        severityText: 'INFO',
+                        body: `LLM generation started: ${hookContext.processId}`,
+                        attributes: {
+                            // Explicit trace correlation (some backends need these)
+                            trace_id: llmGenSpanCtx.traceId,
+                            span_id: llmGenSpanCtx.spanId,
+                            trace_flags: llmGenSpanCtx.traceFlags,
+
+                            'agent.id': hookContext.agentId,
+                            'conv.id': hookContext.processId,
+                            'team.id': hookContext.teamId,
+                            'llm.model': modelId || '',
+                            'request.id': reqInfo.requestId,
+                            'context.preview': oTelInstance.redactString(oTelInstance.prepareContext(contextWindow)),
+                        },
+                    });
                 });
+
                 hookContext.curLLMGenSpan = llmGenSpan;
                 if (OTEL_DEBUG_LOGS) outputLogger.debug('createDataHandler completed', reqInfo?.requestId, accessCandidate);
             };
         };
 
+        const createErrorHandler = function (hookContext: any) {
+            return function (error: Error, metadata?: { requestId?: string }) {
+                if (!hookContext.convSpan) return;
+                const accessCandidate = AccessCandidate.agent(hookContext?.agentId);
+                if (OTEL_DEBUG_LOGS)
+                    outputLogger.debug('Error event received', { error: error?.message, requestId: metadata?.requestId }, accessCandidate);
+
+                // Mark that an error occurred so after hook knows not to log success
+                hookContext.hasError = true;
+                hookContext.errorDetails = error;
+
+                const convSpan = hookContext.convSpan;
+                const spanCtx = convSpan.spanContext();
+                const spanContext = trace.setSpan(context.active(), convSpan);
+
+                // Record exception on span
+                convSpan.recordException(error);
+                convSpan.setStatus({ code: SpanStatusCode.ERROR, message: error?.message || 'Unknown error' });
+                convSpan.addEvent('conv.error', {
+                    'error.message': error?.message || 'Unknown error',
+                    'request.id': metadata?.requestId || 'unknown',
+                });
+
+                context.with(spanContext, () => {
+                    logger.emit({
+                        severityNumber: SeverityNumber.ERROR,
+                        severityText: 'ERROR',
+                        body: `Conversation error: ${hookContext.processId}`,
+                        attributes: {
+                            // Explicit trace correlation (some backends need these)
+                            trace_id: spanCtx.traceId,
+                            span_id: spanCtx.spanId,
+                            trace_flags: spanCtx.traceFlags,
+
+                            'agent.id': hookContext.agentId,
+                            'agent.name': hookContext.agentName,
+                            'conv.id': hookContext.processId,
+                            'error.message': error?.message || 'Unknown error',
+                            'error.stack': error?.stack,
+                            'team.id': hookContext.teamId,
+                            'org.slot': hookContext.orgSlot,
+                            'agent.debug': hookContext.isDebugSession,
+                            'agent.isTest': hookContext.isTestDomain,
+                            'request.id': metadata?.requestId || 'unknown',
+                        },
+                    });
+                });
+
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('Error event handled', { error: error?.message }, accessCandidate);
+            };
+        };
+
         const createRequestedHandler = function (hookContext) {
             return function (reqInfo: any) {
                 if (!hookContext.convSpan) return;
@@ -308,8 +465,6 @@ export class OTel extends TelemetryConnector {
                 if (!hookContext.latencySpans) hookContext.latencySpans = {};
                 const contextWindow = reqInfo.contextWindow;
 
-                const lastContext = contextWindow.filter((context) => context.role === 'user').slice(-2);
-
                 const modelId = reqInfo.model;
                 const llmGenLatencySpan = tracer.startSpan(
                     'Conv.GenAI.TTFB',
@@ -319,16 +474,16 @@ export class OTel extends TelemetryConnector {
                             'conv.id': hookContext.processId,
                             'team.id': hookContext.teamId,
                             'request.id': reqInfo.requestId,
-                            'llm.model': modelId || 'unknown',
+                            'llm.model': modelId || '',
                             'metric.type': 'ttfb',
                         },
                     },
-                    trace.setSpan(context.active(), hookContext.convSpan)
+                    trace.setSpan(context.active(), hookContext.convSpan),
                 );
                 llmGenLatencySpan.addEvent('llm.requested', {
                     'request.id': reqInfo.requestId,
                     timestamp: Date.now(),
-                    'context.preview': JSON.stringify(lastContext).substring(0, 200),
+                    'context.preview': oTelInstance.redactString(oTelInstance.prepareContext(contextWindow).substring(0, 200)),
                 });
                 hookContext.latencySpans[reqInfo.requestId] = llmGenLatencySpan;
                 if (OTEL_DEBUG_LOGS) outputLogger.debug('createRequestedHandler completed', reqInfo?.requestId, accessCandidate);
@@ -351,6 +506,7 @@ export class OTel extends TelemetryConnector {
                 const sessionId = processId;
                 const workflowId = agentData?.workflowReqId || agentData?.workflowID || agentData?.workflowId || undefined;
                 const logTags = agentData?.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
+                const agentName = agentData?.name || undefined;
 
                 if (message == null) {
                     //this is a conversation step, will be handled by createRequestedHandler
@@ -366,16 +522,17 @@ export class OTel extends TelemetryConnector {
                     attributes: {
                         // OTel standard attributes
                         'gen_ai.operation.name': 'chat',
-                        'gen_ai.provider.name': conversation?.llmInference?.llmProviderName || 'unknown',
+                        'gen_ai.provider.name': conversation?.llmInference?.llmProviderName || '',
                         'gen_ai.conversation.id': processId,
-                        'gen_ai.request.model': modelId || 'unknown',
+                        'gen_ai.request.model': modelId || '',
                         ////////////////////////////////
                         'team.id': teamId,
                         'org.tier': orgTier,
                         'org.slot': orgSlot,
                         'agent.id': agentId,
+                        'agent.name': agentName,
                         'conv.id': processId,
-                        'llm.model': modelId || 'unknown',
+                        'llm.model': modelId || '',
                         'agent.debug': isDebugSession,
                         'agent.isTest': isTestDomain,
                         'session.id': sessionId,
@@ -384,6 +541,7 @@ export class OTel extends TelemetryConnector {
                 });
                 hookContext.convSpan = convSpan;
                 hookContext.agentId = agentId;
+                hookContext.agentName = agentData?.name || undefined;
                 hookContext.processId = processId;
                 hookContext.teamId = teamId;
                 hookContext.orgSlot = orgSlot;
@@ -410,12 +568,15 @@ export class OTel extends TelemetryConnector {
                 hookContext.toolInfoHandler = createToolInfoHandler(hookContext);
                 conversation.on(TLLMEvent.ToolInfo, hookContext.toolInfoHandler);
 
+                hookContext.errorHandler = createErrorHandler(hookContext);
+                conversation.on(TLLMEvent.Error, hookContext.errorHandler);
+
                 // Add start event
 
                 convSpan.addEvent('skill.process.started', {
                     'input.size': JSON.stringify(message || {}).length,
-                    'input.preview': message.substring(0, 200),
-                    'llm.model': modelId || 'unknown',
+                    'input.preview': oTelInstance.redactString(message.substring(0, 200)),
+                    'llm.model': modelId || '',
                 });
 
                 OTelContextRegistry.startProcess(agentId, processId, convSpan);
@@ -438,9 +599,10 @@ export class OTel extends TelemetryConnector {
                             'org.slot': orgSlot,
 
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'conv.id': processId,
                             'input.size': JSON.stringify(message || {}).length,
-                            'input.preview': message.substring(0, 2000),
+                            'input.preview': oTelInstance.redactString(message.substring(0, 4000)),
                             'agent.debug': isDebugSession,
                             'agent.isTest': isTestDomain,
                             'session.id': sessionId,
@@ -450,7 +612,7 @@ export class OTel extends TelemetryConnector {
                     });
                 });
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
 
         HookService.registerAfter(
@@ -471,6 +633,7 @@ export class OTel extends TelemetryConnector {
                 const sessionId = processId;
                 const workflowId = agentData?.workflowReqId || agentData?.workflowID || agentData?.workflowId || undefined;
                 const logTags = agentData?.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
+                const agentName = agentData?.name || undefined;
 
                 if (message == null) {
                     return;
@@ -489,7 +652,9 @@ export class OTel extends TelemetryConnector {
                     }
                     hookContext.curLLMGenSpan.addEvent('llm.gen.content', {
                         'content.size': JSON.stringify(result || {}).length,
-                        'content.preview': typeof result === 'string' ? result.substring(0, 200) : JSON.stringify(result || {}).substring(0, 200),
+                        'content.preview': oTelInstance.redactString(
+                            typeof result === 'string' ? result.substring(0, 200) : JSON.stringify(result || {}).substring(0, 200),
+                        ),
                     });
                     hookContext.curLLMGenSpan.end();
 
@@ -498,46 +663,52 @@ export class OTel extends TelemetryConnector {
                     if (hookContext.requestedHandler) conversation.off(TLLMEvent.Requested, hookContext.requestedHandler);
                 }
 
+                if (hookContext.errorHandler) conversation.off(TLLMEvent.Error, hookContext.errorHandler);
+
                 const { rootSpan: convSpan } = ctx;
 
                 const spanCtx = convSpan.spanContext();
                 const spanContext = trace.setSpan(context.active(), convSpan);
 
-                if (error) {
-                    // Error handling for conversation span
-                    convSpan.recordException(error);
-                    convSpan.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
-                    convSpan.addEvent('conv.error', {
-                        'error.message': error.message,
-                    });
+                // Check for errors - either thrown (error param) or emitted via event (hookContext.hasError)
+                const hasError = error || hookContext.hasError;
 
-                    // Emit ERROR log
-                    context.with(spanContext, () => {
-                        logger.emit({
-                            severityNumber: SeverityNumber.ERROR,
-                            severityText: 'ERROR',
-                            body: `Conversation.streamPrompt failed: ${processId}`,
-                            attributes: {
-                                // Explicit trace correlation
-                                trace_id: spanCtx.traceId,
-                                span_id: spanCtx.spanId,
-                                trace_flags: spanCtx.traceFlags,
+                if (hasError) {
+                    if (error && !hookContext.hasError) {
+                        convSpan.recordException(error);
+                        convSpan.setStatus({ code: SpanStatusCode.ERROR, message: error.message || 'Unknown error' });
+                        convSpan.addEvent('conv.error', {
+                            'error.message': error.message || 'Unknown error',
+                        });
 
-                                'agent.id': agentId,
-                                'conv.id': processId,
-                                'error.message': error.message,
-                                'error.stack': error.stack,
-                                'team.id': teamId,
-                                'org.tier': orgTier,
-                                'org.slot': orgSlot,
-                                'agent.debug': isDebugSession,
-                                'agent.isTest': isTestDomain,
-                                'session.id': sessionId,
-                                'workflow.id': workflowId,
-                                'log.tags': logTags,
-                            },
+                        context.with(spanContext, () => {
+                            logger.emit({
+                                severityNumber: SeverityNumber.ERROR,
+                                severityText: 'ERROR',
+                                body: `Conversation.streamPrompt failed: ${processId}`,
+                                attributes: {
+                                    // Explicit trace correlation (some backends need these)
+                                    trace_id: spanCtx.traceId,
+                                    span_id: spanCtx.spanId,
+                                    trace_flags: spanCtx.traceFlags,
+
+                                    'agent.id': agentId,
+                                    'agent.name': agentName,
+                                    'conv.id': processId,
+                                    'error.message': error.message || 'Unknown error',
+                                    'error.stack': error.stack,
+                                    'team.id': teamId,
+                                    'org.tier': orgTier,
+                                    'org.slot': orgSlot,
+                                    'agent.debug': isDebugSession,
+                                    'agent.isTest': isTestDomain,
+                                    'session.id': sessionId,
+                                    'workflow.id': workflowId,
+                                    'log.tags': logTags,
+                                },
+                            });
                         });
-                    });
+                    }
                 } else {
                     // Success handling
                     convSpan.setStatus({ code: SpanStatusCode.OK });
@@ -548,11 +719,18 @@ export class OTel extends TelemetryConnector {
                             severityText: 'INFO',
                             body: `Conversation.streamPrompt completed: ${processId}`,
                             attributes: {
+                                // Explicit trace correlation (some backends need these)
+                                trace_id: spanCtx.traceId,
+                                span_id: spanCtx.spanId,
+                                trace_flags: spanCtx.traceFlags,
+
                                 'agent.id': agentId,
+                                'agent.name': agentName,
                                 'conv.id': processId,
                                 'output.size': JSON.stringify(result || {}).length,
-                                'output.preview':
-                                    typeof result === 'string' ? result.substring(0, 2000) : JSON.stringify(result || {}).substring(0, 2000),
+                                'output.preview': oTelInstance.redactString(
+                                    (typeof result === 'string' ? result : JSON.stringify(result || {})).substring(0, 4000),
+                                ),
                                 'team.id': teamId,
                                 'org.tier': orgTier,
                                 'org.slot': orgSlot,
@@ -570,7 +748,7 @@ export class OTel extends TelemetryConnector {
 
                 OTelContextRegistry.endProcess(agentId, processId);
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
 
         HookService.register(
@@ -597,6 +775,7 @@ export class OTel extends TelemetryConnector {
                 const logTags = agent.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
                 const isTestDomain = agent.usingTestDomain || false;
                 const domain = agent.domain || undefined;
+                const agentName = agent.name || undefined;
 
                 const accessCandidate = AccessCandidate.agent(agentId);
                 if (OTEL_DEBUG_LOGS) outputLogger.debug('SREAgent.process started', { processId, agentProcessId, endpointPath }, accessCandidate);
@@ -608,6 +787,11 @@ export class OTel extends TelemetryConnector {
 
                 const input = { body, query, headers, processInput: agentInput };
 
+                const logBody = oTelInstance.prepareComponentData(agentRequest.body || {}, undefined, 4000);
+                const logQuery = oTelInstance.prepareComponentData(agentRequest.query || {}, undefined, 4000);
+                const logHeaders = oTelInstance.prepareComponentData(agentRequest.headers || {}, undefined, 4000);
+                const logAgentInput = oTelInstance.prepareComponentData(inputData || {}, undefined, 4000);
+
                 let convSpan;
                 let parentContext = context.active();
 
@@ -640,6 +824,7 @@ export class OTel extends TelemetryConnector {
                     {
                         attributes: {
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'team.id': teamId,
                             'conv.id': conversationId,
                             'process.id': agentProcessId,
@@ -652,11 +837,11 @@ export class OTel extends TelemetryConnector {
                             'agent.domain': domain,
                         },
                     },
-                    parentContext
+                    parentContext,
                 );
 
                 // Add start event
-                const inputPreview = JSON.stringify(input || {}).substring(0, 200);
+                const inputPreview = oTelInstance.redactString(JSON.stringify(input || {}).substring(0, 200));
                 agentSpan.addEvent('skill.process.started', {
                     endpoint: endpointPath,
                     'input.size': JSON.stringify(input || {}).length,
@@ -678,12 +863,14 @@ export class OTel extends TelemetryConnector {
                             trace_id: spanCtx.traceId,
                             span_id: spanCtx.spanId,
                             trace_flags: spanCtx.traceFlags,
+
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'process.id': agentProcessId,
-                            input: agentInput,
-                            body,
-                            query,
-                            headers,
+                            input: oTelInstance.redactObject(logAgentInput),
+                            body: oTelInstance.redactObject(logBody),
+                            query: oTelInstance.redactObject(logQuery),
+                            headers: oTelInstance.redactRequestHeaders(logHeaders),
                             'team.id': teamId,
                             'org.slot': orgSlot,
                             'org.tier': orgTier,
@@ -698,7 +885,7 @@ export class OTel extends TelemetryConnector {
                     } as any);
                 });
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
 
         HookService.registerAfter(
@@ -720,6 +907,7 @@ export class OTel extends TelemetryConnector {
                 const logTags = agent.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
                 const isTestDomain = agent.usingTestDomain || false;
                 const domain = agent.domain || undefined;
+                const agentName = agent.name || undefined;
 
                 const ctx = OTelContextRegistry.get(agentId, agentProcessId);
                 if (!ctx) return;
@@ -774,7 +962,9 @@ export class OTel extends TelemetryConnector {
                     trace_id: spanCtx.traceId,
                     span_id: spanCtx.spanId,
                     trace_flags: spanCtx.traceFlags,
+
                     'agent.id': agentId,
+                    'agent.name': agentName,
                     'process.id': agentProcessId,
                     hasError: isError,
                     'error.message': isError ? errorMessage : undefined,
@@ -813,7 +1003,7 @@ export class OTel extends TelemetryConnector {
 
                 OTelContextRegistry.endProcess(agentId, agentProcessId);
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
 
         // In setupHooks() - Enhanced Component.process hook
@@ -844,6 +1034,7 @@ export class OTel extends TelemetryConnector {
                 const isDebugSession = agent.debugSessionEnabled || agent.agentRuntime?.debug || false;
                 const logTags = agent.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
                 const isTestDomain = agent.usingTestDomain || false;
+                const agentName = agent.name || undefined;
 
                 const inputAction = input?.__action || undefined;
                 const inputStatus = input?.__status || undefined;
@@ -860,6 +1051,7 @@ export class OTel extends TelemetryConnector {
                     {
                         attributes: {
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'process.id': processId,
                             'event.id': eventId,
                             'cmp.id': componentId,
@@ -878,22 +1070,30 @@ export class OTel extends TelemetryConnector {
                             ...compSettingsData,
                         },
                     },
-                    parentSpan ? trace.setSpan(context.active(), parentSpan) : undefined
+                    parentSpan ? trace.setSpan(context.active(), parentSpan) : undefined,
                 );
 
                 // Add event: Component started - includes input.action and input.status for workflow tracking
-                const inputStr = JSON.stringify(input || {});
-
-                const compInputData = oTelInstance.prepareComponentData(input || {});
+                // Use component-specific input (from predecessor nodes), not the merged object with agent variables
+                // For APIEndpoint, use HTTP request body/query as the actual user input
+                const componentInput =
+                    componentType === 'APIEndpoint'
+                        ? agent.agentRequest?.method === 'GET'
+                            ? agent.agentRequest?.query
+                            : agent.agentRequest?.body
+                        : componentData?.runtimeData?.input || {};
+
+                const compInputData = oTelInstance.prepareComponentData(componentInput || {});
                 span.addEvent('cmp.call', {
                     'event.id': eventId,
-                    'cmp.input.size': JSON.stringify(input || {}).length,
-                    'cmp.input': JSON.stringify(compInputData),
+                    'cmp.input.size': JSON.stringify(componentInput || {}).length,
+                    'cmp.input': oTelInstance.redactString(JSON.stringify(compInputData)),
                     'input.action': inputAction,
                     'input.status': inputStatus,
                 });
 
                 // Emit structured log with full details
+                const cmpSpanCtx = span.spanContext();
                 const spanContext = trace.setSpan(context.active(), span);
                 context.with(spanContext, () => {
                     logger.emit({
@@ -901,13 +1101,19 @@ export class OTel extends TelemetryConnector {
                         severityText: 'INFO',
                         body: `Component ${componentType} started`,
                         attributes: {
+                            // Explicit trace correlation (some backends need these)
+                            trace_id: cmpSpanCtx.traceId,
+                            span_id: cmpSpanCtx.spanId,
+                            trace_flags: cmpSpanCtx.traceFlags,
+
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'process.id': processId,
                             'event.id': eventId,
                             'cmp.id': componentId,
                             'cmp.type': componentType,
                             'cmp.name': componentName,
-                            'cmp.input': input,
+                            'cmp.input': oTelInstance.redactObject(componentInput),
                             'team.id': teamId,
                             'org.slot': orgSlot,
                             'org.tier': orgTier,
@@ -926,7 +1132,7 @@ export class OTel extends TelemetryConnector {
                 // Store span in hook context (isolated per component execution, concurrency-safe)
                 this.context.otelSpan = span;
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
 
         HookService.registerAfter(
@@ -961,6 +1167,7 @@ export class OTel extends TelemetryConnector {
                 const isDebugSession = agent.debugSessionEnabled || agent.agentRuntime?.debug || false;
                 const logTags = agent.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
                 const isTestDomain = agent.usingTestDomain || false;
+                const agentName = agent.name || undefined;
 
                 const accessCandidate = AccessCandidate.agent(agentId);
                 if (OTEL_DEBUG_LOGS) outputLogger.debug('Component.process completed', { componentId }, accessCandidate);
@@ -982,6 +1189,7 @@ export class OTel extends TelemetryConnector {
                     });
 
                     // Emit error log
+                    const cmpErrorSpanCtx = span.spanContext();
                     const spanContext = trace.setSpan(context.active(), span);
                     context.with(spanContext, () => {
                         logger.emit({
@@ -989,7 +1197,13 @@ export class OTel extends TelemetryConnector {
                             severityText: 'ERROR',
                             body: `Component ${componentType} (${componentId}) failed: ${error.message}`,
                             attributes: {
+                                // Explicit trace correlation (some backends need these)
+                                trace_id: cmpErrorSpanCtx.traceId,
+                                span_id: cmpErrorSpanCtx.spanId,
+                                trace_flags: cmpErrorSpanCtx.traceFlags,
+
                                 'agent.id': agentId,
+                                'agent.name': agentName,
                                 'process.id': processId,
                                 'event.id': eventId,
                                 'cmp.id': componentId,
@@ -1039,6 +1253,7 @@ export class OTel extends TelemetryConnector {
                         });
 
                         // Emit ERROR log for result error
+                        const cmpResultErrorSpanCtx = span.spanContext();
                         const spanContext = trace.setSpan(context.active(), span);
                         context.with(spanContext, () => {
                             logger.emit({
@@ -1046,7 +1261,13 @@ export class OTel extends TelemetryConnector {
                                 severityText: 'ERROR',
                                 body: `Component ${componentType} (${componentId}) failed: ${errorMessage}`,
                                 attributes: {
+                                    // Explicit trace correlation (some backends need these)
+                                    trace_id: cmpResultErrorSpanCtx.traceId,
+                                    span_id: cmpResultErrorSpanCtx.spanId,
+                                    trace_flags: cmpResultErrorSpanCtx.traceFlags,
+
                                     'agent.id': agentId,
+                                    'agent.name': agentName,
                                     'process.id': processId,
                                     'event.id': eventId,
                                     'cmp.id': componentId,
@@ -1054,7 +1275,7 @@ export class OTel extends TelemetryConnector {
                                     'cmp.type': componentType,
                                     'error.type': 'result_error',
                                     'error.message': errorMessage,
-                                    'cmp.output': result,
+                                    'cmp.output': oTelInstance.redactObject(result),
                                     'team.id': teamId,
                                     'org.slot': orgSlot,
                                     'org.tier': orgTier,
@@ -1076,7 +1297,7 @@ export class OTel extends TelemetryConnector {
                         // Add success event with output summary
                         span.addEvent('cmp.call.result', {
                             'output.size': resultStr.length,
-                            'output.preview': resultStr.substring(0, 200),
+                            'output.preview': oTelInstance.redactString(resultStr.substring(0, 200)),
                         });
 
                         // Add output attributes to span
@@ -1086,14 +1307,21 @@ export class OTel extends TelemetryConnector {
                         });
 
                         // Emit success log with output (formatted safely)
+                        const cmpSuccessSpanCtx = span.spanContext();
                         const logAttributes: Record<string, any> = {
+                            // Explicit trace correlation (some backends need these)
+                            trace_id: cmpSuccessSpanCtx.traceId,
+                            span_id: cmpSuccessSpanCtx.spanId,
+                            trace_flags: cmpSuccessSpanCtx.traceFlags,
+
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'cmp.id': componentId,
                             'cmp.type': componentType,
                             'cmp.name': componentName,
                             'process.id': processId,
                             'event.id': eventId,
-                            'cmp.output': result,
+                            'cmp.output': oTelInstance.redactObject(result),
                             'team.id': teamId,
                             'org.slot': orgSlot,
                             'org.tier': orgTier,
@@ -1121,7 +1349,7 @@ export class OTel extends TelemetryConnector {
 
                 span.end();
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
         return Promise.resolve();
     }