@smythos/sre 1.7.41 → 1.8.0

package/src/subsystems/ObservabilityManager/Telemetry.service/connectors/OTel/OTel.class.ts

@@ -4,6 +4,7 @@ import { ACL } from '@sre/Security/AccessControl/ACL.class';
 import { IAccessCandidate } from '@sre/types/ACL.types';
 import { TelemetryConnector } from '../../TelemetryConnector';
 import { AgentCallLog } from '@sre/types/AgentLogger.types';
+import { redactSensitiveString, redactData, redactHeaders } from './OTel.redaction.helper';
 
 import { trace, context, SpanStatusCode, Tracer, propagation } from '@opentelemetry/api';
 import { Logger as OTelLogger, logs, SeverityNumber } from '@opentelemetry/api-logs';
@@ -50,6 +51,14 @@ export type OTelLogConfig = {
      * These will be replaced with '[REDACTED]' in logs
      */
     redactFields?: string[];
+    /**
+     * Enable automatic redaction of sensitive data in logs and traces.
+     * When true (or omitted), sensitive data such as passwords, tokens,
+     * API keys, and JWT tokens are automatically replaced with '[REDACTED]'.
+     * Set to false to disable all automatic redaction.
+     * Default: true
+     */
+    enableRedaction?: boolean;
 };
 const OTEL_DEBUG_LOGS = true;
 export class OTel extends TelemetryConnector {
@@ -62,6 +71,9 @@ export class OTel extends TelemetryConnector {
 
     constructor(protected _settings: OTelLogConfig) {
         super();
+        // Default enableRedaction to true when not explicitly provided
+        _settings.enableRedaction = _settings.enableRedaction ?? true;
+
         if (!_settings.endpoint) {
             outputLogger.warn('OTel initialization skipped, endpoint is not set');
             return;
@@ -141,6 +153,7 @@ export class OTel extends TelemetryConnector {
      * Redact sensitive fields from an object
      */
     private redactSensitiveData(data: any, redactFields?: string[]): any {
+        if (!this._settings.enableRedaction) return data;
         if (!redactFields || redactFields.length === 0) return data;
         if (typeof data !== 'object' || data === null) return data;
 
@@ -157,6 +170,35 @@ export class OTel extends TelemetryConnector {
         return redacted;
     }
 
+    /**
+     * Redact sensitive patterns from a string value.
+     * Skips redaction when enableRedaction is explicitly set to false.
+     */
+    private redactString(value: string): string {
+        if (!this._settings.enableRedaction) return value;
+        return redactSensitiveString(value);
+    }
+
+    /**
+     * Redact sensitive data from any data type (objects, arrays, strings).
+     * Skips redaction when enableRedaction is explicitly set to false.
+     */
+    private redactObject<T>(data: T): T {
+        if (!this._settings.enableRedaction) return data;
+        return redactData(data);
+    }
+
+    /**
+     * Redact sensitive HTTP headers.
+     * Skips redaction when enableRedaction is explicitly set to false.
+     */
+    private redactRequestHeaders(
+        headers: Record<string, unknown> | string | undefined | null,
+    ): Record<string, unknown> | string | undefined | null {
+        if (!this._settings.enableRedaction) return headers;
+        return redactHeaders(headers);
+    }
+
     /**
      * Safely format output for logging with size limits and redaction
      */
@@ -170,11 +212,17 @@ export class OTel extends TelemetryConnector {
             return undefined;
         }
 
-        // Redact sensitive fields
-        const redacted = this.redactSensitiveData(output, config.redactFields);
+        // Redact sensitive fields (config-based)
+        let redacted = this.redactSensitiveData(output, config.redactFields);
+
+        // Apply SENSITIVE_WORDS-based redaction on the object (automatic key-based redaction)
+        redacted = this.redactObject(redacted);
 
         // Stringify
-        const outputStr = JSON.stringify(redacted);
+        let outputStr = JSON.stringify(redacted);
+
+        // Apply string-based redaction on the stringified output to catch embedded JSON
+        outputStr = this.redactString(outputStr);
 
         // Check size limit
         if (outputStr && outputStr.length > maxSize) {
@@ -200,12 +248,38 @@ export class OTel extends TelemetryConnector {
         for (let key in data) {
             result[prefix ? `${prefix}.${key}` : key] = (typeof data[key] === 'object' ? JSON.stringify(data[key]) : data[key].toString()).substring(
                 0,
-                maxEntryLength
+                maxEntryLength,
             );
         }
 
         return result;
     }
+
+    private prepareContext(contextWindow: Array<{ role: string; content: string; [key: string]: unknown }>): string {
+        if (!contextWindow || !Array.isArray(contextWindow)) return '[]';
+
+        const filtered = contextWindow.filter((msg) => {
+            if (typeof msg !== 'object' || msg === null) return false;
+            const keys = Object.keys(msg);
+            return !keys.some((k) => k.includes('___smyth_metadata___'));
+        });
+
+        const lastAssistant = [...filtered].reverse().find((msg) => msg.role === 'assistant' && msg.content);
+        const lastUser = [...filtered].reverse().find((msg) => msg.role === 'user' && msg.content);
+
+        const messages: Array<{ role: string; content: string }> = [];
+        if (lastAssistant) {
+            const raw = typeof lastAssistant.content === 'string' ? lastAssistant.content : JSON.stringify(lastAssistant.content);
+            messages.push({ role: 'assistant', content: raw.substring(0, 2000) });
+        }
+        if (lastUser) {
+            const raw = typeof lastUser.content === 'string' ? lastUser.content : JSON.stringify(lastUser.content);
+            messages.push({ role: 'user', content: raw.substring(0, 2000) });
+        }
+
+        return JSON.stringify(messages);
+    }
+
     protected setupHooks(): Promise<void> {
         const tracer = this.tracer;
         const logger = this.logger;
@@ -219,15 +293,18 @@ export class OTel extends TelemetryConnector {
 
                 const modelId = toolInfo.model;
                 const contextWindow = toolInfo.contextWindow;
-                const lastContext = contextWindow.filter((context) => context.role === 'user').slice(-2);
 
-                const toolNames = toolInfo.map((tool) => tool.name + '(' + tool.arguments + ')');
+                const toolNames = toolInfo.map((tool) => {
+                    const args = typeof tool.arguments === 'string' ? tool.arguments : JSON.stringify(tool.arguments);
+                    return `${tool.name}(${args})`;
+                });
                 hookContext.curLLMGenSpan.addEvent('llm.gen.tool.calls', {
-                    'tool.calls': toolNames.join(', '),
-                    'llm.model': modelId || 'unknown',
-                    'context.preview': JSON.stringify(lastContext).substring(0, 200),
+                    'tool.calls': oTelInstance.redactString(toolNames.join(', ')),
+                    'llm.model': modelId || '',
+                    'context.preview': oTelInstance.redactString(oTelInstance.prepareContext(contextWindow).substring(0, 200)),
                 });
 
+                const llmSpanCtx = hookContext.curLLMGenSpan.spanContext();
                 const spanContext = trace.setSpan(context.active(), hookContext.curLLMGenSpan);
                 context.with(spanContext, () => {
                     logger.emit({
@@ -235,10 +312,15 @@ export class OTel extends TelemetryConnector {
                         severityText: 'INFO',
                         body: `LLM tool calls: ${toolNames.join(', ')}`,
                         attributes: {
+                            // Explicit trace correlation (some backends need these)
+                            trace_id: llmSpanCtx.traceId,
+                            span_id: llmSpanCtx.spanId,
+                            trace_flags: llmSpanCtx.traceFlags,
+
                             'agent.id': hookContext.agentId,
                             'conv.id': hookContext.processId,
-                            'llm.model': modelId || 'unknown',
-                            'context.preview': JSON.stringify(lastContext).substring(0, 5000),
+                            'llm.model': modelId || '',
+                            'context.preview': oTelInstance.redactString(oTelInstance.prepareContext(contextWindow)),
                         },
                     });
                 });
@@ -259,7 +341,6 @@ export class OTel extends TelemetryConnector {
                 const modelId = reqInfo.model;
                 const contextWindow = reqInfo.contextWindow;
 
-                const lastContext = contextWindow.filter((context) => context.role === 'user').slice(-2);
                 // End TTFB span when first data arrives
                 if (hookContext?.latencySpans?.[reqInfo.requestId]) {
                     const ttfbSpan = hookContext.latencySpans[reqInfo.requestId];
@@ -268,7 +349,7 @@ export class OTel extends TelemetryConnector {
                     ttfbSpan.addEvent('llm.first.byte.received', {
                         'request.id': reqInfo.requestId,
                         'data.size': JSON.stringify(data || {}).length,
-                        'llm.model': modelId || 'unknown',
+                        'llm.model': modelId || '',
                     });
 
                     ttfbSpan.setStatus({ code: SpanStatusCode.OK });
@@ -284,22 +365,98 @@ export class OTel extends TelemetryConnector {
                             'agent.id': hookContext.agentId,
                             'conv.id': hookContext.processId,
                             'team.id': hookContext.teamId,
-                            'llm.model': modelId || 'unknown',
+                            'llm.model': modelId || '',
                         },
                     },
-                    trace.setSpan(context.active(), hookContext.convSpan)
+                    trace.setSpan(context.active(), hookContext.convSpan),
                 );
                 llmGenSpan.addEvent('llm.gen.started', {
                     'request.id': reqInfo.requestId,
                     timestamp: Date.now(),
-                    'llm.model': modelId || 'unknown',
-                    'context.preview': JSON.stringify(lastContext).substring(0, 200),
+                    'llm.model': modelId || '',
+                    'context.preview': oTelInstance.redactString(oTelInstance.prepareContext(contextWindow).substring(0, 200)),
+                });
+
+                const llmGenSpanCtx = llmGenSpan.spanContext();
+                const llmGenSpanContext = trace.setSpan(context.active(), llmGenSpan);
+                context.with(llmGenSpanContext, () => {
+                    logger.emit({
+                        severityNumber: SeverityNumber.INFO,
+                        severityText: 'INFO',
+                        body: `LLM generation started: ${hookContext.processId}`,
+                        attributes: {
+                            // Explicit trace correlation (some backends need these)
+                            trace_id: llmGenSpanCtx.traceId,
+                            span_id: llmGenSpanCtx.spanId,
+                            trace_flags: llmGenSpanCtx.traceFlags,
+
+                            'agent.id': hookContext.agentId,
+                            'conv.id': hookContext.processId,
+                            'team.id': hookContext.teamId,
+                            'llm.model': modelId || '',
+                            'request.id': reqInfo.requestId,
+                            'context.preview': oTelInstance.redactString(oTelInstance.prepareContext(contextWindow)),
+                        },
+                    });
                 });
+
                 hookContext.curLLMGenSpan = llmGenSpan;
                 if (OTEL_DEBUG_LOGS) outputLogger.debug('createDataHandler completed', reqInfo?.requestId, accessCandidate);
             };
         };
 
+        const createErrorHandler = function (hookContext: any) {
+            return function (error: Error, metadata?: { requestId?: string }) {
+                if (!hookContext.convSpan) return;
+                const accessCandidate = AccessCandidate.agent(hookContext?.agentId);
+                if (OTEL_DEBUG_LOGS)
+                    outputLogger.debug('Error event received', { error: error?.message, requestId: metadata?.requestId }, accessCandidate);
+
+                // Mark that an error occurred so after hook knows not to log success
+                hookContext.hasError = true;
+                hookContext.errorDetails = error;
+
+                const convSpan = hookContext.convSpan;
+                const spanCtx = convSpan.spanContext();
+                const spanContext = trace.setSpan(context.active(), convSpan);
+
+                // Record exception on span
+                convSpan.recordException(error);
+                convSpan.setStatus({ code: SpanStatusCode.ERROR, message: error?.message || 'Unknown error' });
+                convSpan.addEvent('conv.error', {
+                    'error.message': error?.message || 'Unknown error',
+                    'request.id': metadata?.requestId || 'unknown',
+                });
+
+                context.with(spanContext, () => {
+                    logger.emit({
+                        severityNumber: SeverityNumber.ERROR,
+                        severityText: 'ERROR',
+                        body: `Conversation error: ${hookContext.processId}`,
+                        attributes: {
+                            // Explicit trace correlation (some backends need these)
+                            trace_id: spanCtx.traceId,
+                            span_id: spanCtx.spanId,
+                            trace_flags: spanCtx.traceFlags,
+
+                            'agent.id': hookContext.agentId,
+                            'agent.name': hookContext.agentName,
+                            'conv.id': hookContext.processId,
+                            'error.message': error?.message || 'Unknown error',
+                            'error.stack': error?.stack,
+                            'team.id': hookContext.teamId,
+                            'org.slot': hookContext.orgSlot,
+                            'agent.debug': hookContext.isDebugSession,
+                            'agent.isTest': hookContext.isTestDomain,
+                            'request.id': metadata?.requestId || 'unknown',
+                        },
+                    });
+                });
+
+                if (OTEL_DEBUG_LOGS) outputLogger.debug('Error event handled', { error: error?.message }, accessCandidate);
+            };
+        };
+
         const createRequestedHandler = function (hookContext) {
             return function (reqInfo: any) {
                 if (!hookContext.convSpan) return;
@@ -308,8 +465,6 @@ export class OTel extends TelemetryConnector {
                 if (!hookContext.latencySpans) hookContext.latencySpans = {};
                 const contextWindow = reqInfo.contextWindow;
 
-                const lastContext = contextWindow.filter((context) => context.role === 'user').slice(-2);
-
                 const modelId = reqInfo.model;
                 const llmGenLatencySpan = tracer.startSpan(
                     'Conv.GenAI.TTFB',
@@ -319,16 +474,16 @@ export class OTel extends TelemetryConnector {
                             'conv.id': hookContext.processId,
                             'team.id': hookContext.teamId,
                             'request.id': reqInfo.requestId,
-                            'llm.model': modelId || 'unknown',
+                            'llm.model': modelId || '',
                             'metric.type': 'ttfb',
                         },
                     },
-                    trace.setSpan(context.active(), hookContext.convSpan)
+                    trace.setSpan(context.active(), hookContext.convSpan),
                 );
                 llmGenLatencySpan.addEvent('llm.requested', {
                     'request.id': reqInfo.requestId,
                     timestamp: Date.now(),
-                    'context.preview': JSON.stringify(lastContext).substring(0, 200),
+                    'context.preview': oTelInstance.redactString(oTelInstance.prepareContext(contextWindow).substring(0, 200)),
                 });
                 hookContext.latencySpans[reqInfo.requestId] = llmGenLatencySpan;
                 if (OTEL_DEBUG_LOGS) outputLogger.debug('createRequestedHandler completed', reqInfo?.requestId, accessCandidate);
@@ -351,6 +506,7 @@ export class OTel extends TelemetryConnector {
                 const sessionId = processId;
                 const workflowId = agentData?.workflowReqId || agentData?.workflowID || agentData?.workflowId || undefined;
                 const logTags = agentData?.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
+                const agentName = agentData?.name || undefined;
 
                 if (message == null) {
                     //this is a conversation step, will be handled by createRequestedHandler
@@ -366,16 +522,17 @@ export class OTel extends TelemetryConnector {
                     attributes: {
                         // OTel standard attributes
                         'gen_ai.operation.name': 'chat',
-                        'gen_ai.provider.name': conversation?.llmInference?.llmProviderName || 'unknown',
+                        'gen_ai.provider.name': conversation?.llmInference?.llmProviderName || '',
                         'gen_ai.conversation.id': processId,
-                        'gen_ai.request.model': modelId || 'unknown',
+                        'gen_ai.request.model': modelId || '',
                         ////////////////////////////////
                         'team.id': teamId,
                         'org.tier': orgTier,
                         'org.slot': orgSlot,
                         'agent.id': agentId,
+                        'agent.name': agentName,
                         'conv.id': processId,
-                        'llm.model': modelId || 'unknown',
+                        'llm.model': modelId || '',
                         'agent.debug': isDebugSession,
                         'agent.isTest': isTestDomain,
                         'session.id': sessionId,
@@ -384,6 +541,7 @@ export class OTel extends TelemetryConnector {
                 });
                 hookContext.convSpan = convSpan;
                 hookContext.agentId = agentId;
+                hookContext.agentName = agentData?.name || undefined;
                 hookContext.processId = processId;
                 hookContext.teamId = teamId;
                 hookContext.orgSlot = orgSlot;
@@ -410,12 +568,15 @@ export class OTel extends TelemetryConnector {
                 hookContext.toolInfoHandler = createToolInfoHandler(hookContext);
                 conversation.on(TLLMEvent.ToolInfo, hookContext.toolInfoHandler);
 
+                hookContext.errorHandler = createErrorHandler(hookContext);
+                conversation.on(TLLMEvent.Error, hookContext.errorHandler);
+
                 // Add start event
 
                 convSpan.addEvent('skill.process.started', {
                     'input.size': JSON.stringify(message || {}).length,
-                    'input.preview': message.substring(0, 200),
-                    'llm.model': modelId || 'unknown',
+                    'input.preview': oTelInstance.redactString(message.substring(0, 200)),
+                    'llm.model': modelId || '',
                 });
 
                 OTelContextRegistry.startProcess(agentId, processId, convSpan);
@@ -438,9 +599,10 @@ export class OTel extends TelemetryConnector {
                             'org.slot': orgSlot,
 
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'conv.id': processId,
                             'input.size': JSON.stringify(message || {}).length,
-                            'input.preview': message.substring(0, 2000),
+                            'input.preview': oTelInstance.redactString(message.substring(0, 4000)),
                             'agent.debug': isDebugSession,
                             'agent.isTest': isTestDomain,
                             'session.id': sessionId,
@@ -450,7 +612,7 @@ export class OTel extends TelemetryConnector {
                     });
                 });
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
 
         HookService.registerAfter(
@@ -471,6 +633,7 @@ export class OTel extends TelemetryConnector {
                 const sessionId = processId;
                 const workflowId = agentData?.workflowReqId || agentData?.workflowID || agentData?.workflowId || undefined;
                 const logTags = agentData?.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
+                const agentName = agentData?.name || undefined;
 
                 if (message == null) {
                     return;
@@ -482,10 +645,16 @@ export class OTel extends TelemetryConnector {
                 const accessCandidate = AccessCandidate.agent(agentId);
                 if (OTEL_DEBUG_LOGS) outputLogger.debug('Conversation.streamPrompt completed', { processId }, accessCandidate);
 
+                // Handle curLLMGenSpan with error awareness
                 if (hookContext.curLLMGenSpan) {
+                    if (error) {
+                        hookContext.curLLMGenSpan.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+                    }
                     hookContext.curLLMGenSpan.addEvent('llm.gen.content', {
                         'content.size': JSON.stringify(result || {}).length,
-                        'content.preview': result.substring(0, 200),
+                        'content.preview': oTelInstance.redactString(
+                            typeof result === 'string' ? result.substring(0, 200) : JSON.stringify(result || {}).substring(0, 200),
+                        ),
                     });
                     hookContext.curLLMGenSpan.end();
 
@@ -494,37 +663,92 @@ export class OTel extends TelemetryConnector {
                     if (hookContext.requestedHandler) conversation.off(TLLMEvent.Requested, hookContext.requestedHandler);
                 }
 
+                if (hookContext.errorHandler) conversation.off(TLLMEvent.Error, hookContext.errorHandler);
+
                 const { rootSpan: convSpan } = ctx;
 
                 const spanCtx = convSpan.spanContext();
                 const spanContext = trace.setSpan(context.active(), convSpan);
-                context.with(spanContext, () => {
-                    logger.emit({
-                        severityNumber: SeverityNumber.INFO,
-                        severityText: 'INFO',
-                        body: `Conversation.streamPrompt completed: ${processId}`,
-                        attributes: {
-                            'agent.id': agentId,
-                            'conv.id': processId,
-                            'output.size': JSON.stringify(result || {}).length,
-                            'output.preview': result.substring(0, 2000),
-                            'team.id': teamId,
-                            'org.tier': orgTier,
-                            'org.slot': orgSlot,
-                            'agent.debug': isDebugSession,
-                            'agent.isTest': isTestDomain,
-                            'session.id': sessionId,
-                            'workflow.id': workflowId,
-                            'log.tags': logTags,
-                        },
+
+                // Check for errors - either thrown (error param) or emitted via event (hookContext.hasError)
+                const hasError = error || hookContext.hasError;
+
+                if (hasError) {
+                    if (error && !hookContext.hasError) {
+                        convSpan.recordException(error);
+                        convSpan.setStatus({ code: SpanStatusCode.ERROR, message: error.message || 'Unknown error' });
+                        convSpan.addEvent('conv.error', {
+                            'error.message': error.message || 'Unknown error',
+                        });
+
+                        context.with(spanContext, () => {
+                            logger.emit({
+                                severityNumber: SeverityNumber.ERROR,
+                                severityText: 'ERROR',
+                                body: `Conversation.streamPrompt failed: ${processId}`,
+                                attributes: {
+                                    // Explicit trace correlation (some backends need these)
+                                    trace_id: spanCtx.traceId,
+                                    span_id: spanCtx.spanId,
+                                    trace_flags: spanCtx.traceFlags,
+
+                                    'agent.id': agentId,
+                                    'agent.name': agentName,
+                                    'conv.id': processId,
+                                    'error.message': error.message || 'Unknown error',
+                                    'error.stack': error.stack,
+                                    'team.id': teamId,
+                                    'org.tier': orgTier,
+                                    'org.slot': orgSlot,
+                                    'agent.debug': isDebugSession,
+                                    'agent.isTest': isTestDomain,
+                                    'session.id': sessionId,
+                                    'workflow.id': workflowId,
+                                    'log.tags': logTags,
+                                },
+                            });
+                        });
+                    }
+                } else {
+                    // Success handling
+                    convSpan.setStatus({ code: SpanStatusCode.OK });
+
+                    context.with(spanContext, () => {
+                        logger.emit({
+                            severityNumber: SeverityNumber.INFO,
+                            severityText: 'INFO',
+                            body: `Conversation.streamPrompt completed: ${processId}`,
+                            attributes: {
+                                // Explicit trace correlation (some backends need these)
+                                trace_id: spanCtx.traceId,
+                                span_id: spanCtx.spanId,
+                                trace_flags: spanCtx.traceFlags,
+
+                                'agent.id': agentId,
+                                'agent.name': agentName,
+                                'conv.id': processId,
+                                'output.size': JSON.stringify(result || {}).length,
+                                'output.preview': oTelInstance.redactString(
+                                    (typeof result === 'string' ? result : JSON.stringify(result || {})).substring(0, 4000),
+                                ),
+                                'team.id': teamId,
+                                'org.tier': orgTier,
+                                'org.slot': orgSlot,
+                                'agent.debug': isDebugSession,
+                                'agent.isTest': isTestDomain,
+                                'session.id': sessionId,
+                                'workflow.id': workflowId,
+                                'log.tags': logTags,
+                            },
+                        });
                     });
-                });
+                }
 
                 convSpan.end();
 
                 OTelContextRegistry.endProcess(agentId, processId);
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
 
         HookService.register(
@@ -551,6 +775,7 @@ export class OTel extends TelemetryConnector {
                 const logTags = agent.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
                 const isTestDomain = agent.usingTestDomain || false;
                 const domain = agent.domain || undefined;
+                const agentName = agent.name || undefined;
 
                 const accessCandidate = AccessCandidate.agent(agentId);
                 if (OTEL_DEBUG_LOGS) outputLogger.debug('SREAgent.process started', { processId, agentProcessId, endpointPath }, accessCandidate);
@@ -562,6 +787,11 @@ export class OTel extends TelemetryConnector {
 
                 const input = { body, query, headers, processInput: agentInput };
 
+                const logBody = oTelInstance.prepareComponentData(agentRequest.body || {}, undefined, 4000);
+                const logQuery = oTelInstance.prepareComponentData(agentRequest.query || {}, undefined, 4000);
+                const logHeaders = oTelInstance.prepareComponentData(agentRequest.headers || {}, undefined, 4000);
+                const logAgentInput = oTelInstance.prepareComponentData(inputData || {}, undefined, 4000);
+
                 let convSpan;
                 let parentContext = context.active();
 
@@ -594,6 +824,7 @@ export class OTel extends TelemetryConnector {
                     {
                         attributes: {
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'team.id': teamId,
                             'conv.id': conversationId,
                             'process.id': agentProcessId,
@@ -606,11 +837,11 @@ export class OTel extends TelemetryConnector {
                             'agent.domain': domain,
                         },
                     },
-                    parentContext
+                    parentContext,
                 );
 
                 // Add start event
-                const inputPreview = JSON.stringify(input || {}).substring(0, 200);
+                const inputPreview = oTelInstance.redactString(JSON.stringify(input || {}).substring(0, 200));
                 agentSpan.addEvent('skill.process.started', {
                     endpoint: endpointPath,
                     'input.size': JSON.stringify(input || {}).length,
@@ -632,12 +863,14 @@ export class OTel extends TelemetryConnector {
                             trace_id: spanCtx.traceId,
                             span_id: spanCtx.spanId,
                             trace_flags: spanCtx.traceFlags,
+
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'process.id': agentProcessId,
-                            input: agentInput,
-                            body,
-                            query,
-                            headers,
+                            input: oTelInstance.redactObject(logAgentInput),
+                            body: oTelInstance.redactObject(logBody),
+                            query: oTelInstance.redactObject(logQuery),
+                            headers: oTelInstance.redactRequestHeaders(logHeaders),
                             'team.id': teamId,
                             'org.slot': orgSlot,
                             'org.tier': orgTier,
@@ -652,7 +885,7 @@ export class OTel extends TelemetryConnector {
                     } as any);
                 });
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
 
         HookService.registerAfter(
@@ -674,6 +907,7 @@ export class OTel extends TelemetryConnector {
                 const logTags = agent.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
                 const isTestDomain = agent.usingTestDomain || false;
                 const domain = agent.domain || undefined;
+                const agentName = agent.name || undefined;
 
                 const ctx = OTelContextRegistry.get(agentId, agentProcessId);
                 if (!ctx) return;
@@ -684,12 +918,32 @@ export class OTel extends TelemetryConnector {
                 const accessCandidate = AccessCandidate.agent(agentId);
                 if (OTEL_DEBUG_LOGS) outputLogger.debug('SREAgent.process completed', { agentProcessId }, accessCandidate);
 
+                // Check for error indicators in result (process returned error without throwing)
+                const hasResultError = !error && (!!result?._error || !!result?.error);
+                const resultError = hasResultError ? result._error || result.error : null;
+                const resultErrorMessage = resultError?.message || (typeof resultError === 'string' ? resultError : null);
+
+                // Determine if this is an error case (either thrown error or result error)
+                const isError = !!error || hasResultError;
+                const errorMessage = error?.message || resultErrorMessage || 'Process returned error';
+
                 if (error) {
                     agentSpan.recordException(error);
                     agentSpan.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
                     agentSpan.addEvent('skill.process.error', {
                         'error.message': error.message,
                     });
+                } else if (hasResultError) {
+                    // Handle error in result (no exception thrown)
+                    agentSpan.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage });
+                    agentSpan.addEvent('skill.process.error', {
+                        'error.message': errorMessage,
+                        'error.type': 'result_error',
+                    });
+                    agentSpan.setAttributes({
+                        'output.size': JSON.stringify(result || {}).length,
+                        'output.has_error': true,
+                    });
                 } else {
                     agentSpan.setStatus({ code: SpanStatusCode.OK });
                     agentSpan.addEvent('skill.process.completed', {
@@ -701,18 +955,21 @@ export class OTel extends TelemetryConnector {
                 }
 
                 // Emit log BEFORE ending span to ensure context is active
-                const outputForLog = oTelInstance.formatOutputForLog(result, !!error);
+                const outputForLog = oTelInstance.formatOutputForLog(result, isError);
                 const spanCtx = agentSpan.spanContext();
                 const logAttributes: Record<string, any> = {
                     // Explicit trace correlation (some backends need these)
                     trace_id: spanCtx.traceId,
                     span_id: spanCtx.spanId,
                     trace_flags: spanCtx.traceFlags,
+
                     'agent.id': agentId,
+                    'agent.name': agentName,
                     'process.id': agentProcessId,
-                    hasError: !!error,
-                    'error.message': error?.message,
+                    hasError: isError,
+                    'error.message': isError ? errorMessage : undefined,
                     'error.stack': error?.stack,
+                    'error.type': hasResultError ? 'result_error' : undefined,
                     'team.id': teamId,
                     'org.slot': orgSlot,
                     'org.tier': orgTier,
@@ -734,9 +991,9 @@ export class OTel extends TelemetryConnector {
                 const spanContext = trace.setSpan(context.active(), agentSpan);
                 context.with(spanContext, () => {
                     logger.emit({
-                        severityNumber: error ? SeverityNumber.ERROR : SeverityNumber.INFO,
-                        severityText: error ? 'ERROR' : 'INFO',
-                        body: `Agent process ${error ? 'failed' : 'completed'}: ${agentProcessId}`,
+                        severityNumber: isError ? SeverityNumber.ERROR : SeverityNumber.INFO,
+                        severityText: isError ? 'ERROR' : 'INFO',
+                        body: `Agent process ${isError ? 'failed' : 'completed'}: ${agentProcessId}`,
                         attributes: logAttributes,
                     } as any);
                 });
@@ -746,7 +1003,7 @@ export class OTel extends TelemetryConnector {
 
                 OTelContextRegistry.endProcess(agentId, agentProcessId);
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
 
         // In setupHooks() - Enhanced Component.process hook
@@ -777,6 +1034,7 @@ export class OTel extends TelemetryConnector {
                 const isDebugSession = agent.debugSessionEnabled || agent.agentRuntime?.debug || false;
                 const logTags = agent.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
                 const isTestDomain = agent.usingTestDomain || false;
+                const agentName = agent.name || undefined;
 
                 const inputAction = input?.__action || undefined;
                 const inputStatus = input?.__status || undefined;
@@ -793,6 +1051,7 @@ export class OTel extends TelemetryConnector {
                     {
                         attributes: {
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'process.id': processId,
                             'event.id': eventId,
                             'cmp.id': componentId,
@@ -811,22 +1070,30 @@ export class OTel extends TelemetryConnector {
                             ...compSettingsData,
                         },
                     },
-                    parentSpan ? trace.setSpan(context.active(), parentSpan) : undefined
+                    parentSpan ? trace.setSpan(context.active(), parentSpan) : undefined,
                 );
 
                 // Add event: Component started - includes input.action and input.status for workflow tracking
-                const inputStr = JSON.stringify(input || {});
-
-                const compInputData = oTelInstance.prepareComponentData(input || {});
+                // Use component-specific input (from predecessor nodes), not the merged object with agent variables
+                // For APIEndpoint, use HTTP request body/query as the actual user input
+                const componentInput =
+                    componentType === 'APIEndpoint'
+                        ? agent.agentRequest?.method === 'GET'
+                            ? agent.agentRequest?.query
+                            : agent.agentRequest?.body
+                        : componentData?.runtimeData?.input || {};
+
+                const compInputData = oTelInstance.prepareComponentData(componentInput || {});
                 span.addEvent('cmp.call', {
                     'event.id': eventId,
-                    'cmp.input.size': JSON.stringify(input || {}).length,
-                    'cmp.input': JSON.stringify(compInputData),
+                    'cmp.input.size': JSON.stringify(componentInput || {}).length,
+                    'cmp.input': oTelInstance.redactString(JSON.stringify(compInputData)),
                     'input.action': inputAction,
                     'input.status': inputStatus,
                 });
 
                 // Emit structured log with full details
+                const cmpSpanCtx = span.spanContext();
                 const spanContext = trace.setSpan(context.active(), span);
                 context.with(spanContext, () => {
                     logger.emit({
@@ -834,13 +1101,19 @@ export class OTel extends TelemetryConnector {
                         severityText: 'INFO',
                         body: `Component ${componentType} started`,
                         attributes: {
+                            // Explicit trace correlation (some backends need these)
+                            trace_id: cmpSpanCtx.traceId,
+                            span_id: cmpSpanCtx.spanId,
+                            trace_flags: cmpSpanCtx.traceFlags,
+
                             'agent.id': agentId,
+                            'agent.name': agentName,
                             'process.id': processId,
                             'event.id': eventId,
                             'cmp.id': componentId,
                             'cmp.type': componentType,
                             'cmp.name': componentName,
-                            'cmp.input': input,
+                            'cmp.input': oTelInstance.redactObject(componentInput),
                             'team.id': teamId,
                             'org.slot': orgSlot,
                             'org.tier': orgTier,
@@ -859,7 +1132,7 @@ export class OTel extends TelemetryConnector {
                 // Store span in hook context (isolated per component execution, concurrency-safe)
                 this.context.otelSpan = span;
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
 
         HookService.registerAfter(
@@ -894,6 +1167,7 @@ export class OTel extends TelemetryConnector {
                 const isDebugSession = agent.debugSessionEnabled || agent.agentRuntime?.debug || false;
                 const logTags = agent.sessionTag || (isDebugSession ? 'DEBUG' : undefined);
                 const isTestDomain = agent.usingTestDomain || false;
+                const agentName = agent.name || undefined;
 
                 const accessCandidate = AccessCandidate.agent(agentId);
                 if (OTEL_DEBUG_LOGS) outputLogger.debug('Component.process completed', { componentId }, accessCandidate);
@@ -915,6 +1189,7 @@ export class OTel extends TelemetryConnector {
                     });
 
                     // Emit error log
+                    const cmpErrorSpanCtx = span.spanContext();
                     const spanContext = trace.setSpan(context.active(), span);
                     context.with(spanContext, () => {
                         logger.emit({
@@ -922,7 +1197,13 @@ export class OTel extends TelemetryConnector {
                             severityText: 'ERROR',
                             body: `Component ${componentType} (${componentId}) failed: ${error.message}`,
                             attributes: {
+                                // Explicit trace correlation (some backends need these)
+                                trace_id: cmpErrorSpanCtx.traceId,
+                                span_id: cmpErrorSpanCtx.spanId,
+                                trace_flags: cmpErrorSpanCtx.traceFlags,
+
                                 'agent.id': agentId,
+                                'agent.name': agentName,
                                 'process.id': processId,
                                 'event.id': eventId,
                                 'cmp.id': componentId,
@@ -946,63 +1227,129 @@ export class OTel extends TelemetryConnector {
                         });
                     });
                 } else {
-                    span.setStatus({ code: SpanStatusCode.OK });
-
-                    // Add success event with output summary
+                    // Check if result contains an error indicator (component returned error without throwing)
+                    const hasResultError = !!result?._error || !!result?.error;
                     const resultStr = JSON.stringify(result || {});
-                    span.addEvent('cmp.call.result', {
-                        'output.size': resultStr.length,
-                        'output.preview': resultStr.substring(0, 200),
-                    });
 
-                    // Add output attributes to span
-                    span.setAttributes({
-                        'output.size': JSON.stringify(result || {}).length,
-                        'output.has_error': !!result?._error,
-                    });
+                    if (hasResultError) {
+                        // Treat as error even though no exception was thrown
+                        const resultError = result._error || result.error;
+                        const errorMessage = resultError?.message || (typeof resultError === 'string' ? resultError : 'Component returned error');
 
-                    // Emit success log with output (formatted safely)
-                    const outputForLog = oTelInstance.formatOutputForLog(result, false);
-                    const logAttributes: Record<string, any> = {
-                        'agent.id': agentId,
-                        'cmp.id': componentId,
-                        'cmp.type': componentType,
-                        'cmp.name': componentName,
-                        'process.id': processId,
-                        'event.id': eventId,
-                        'cmp.output': result,
-                        'team.id': teamId,
-                        'org.slot': orgSlot,
-                        'org.tier': orgTier,
-                        'source.id': sourceId,
-                        'source.name': sourceName,
-                        'session.id': sessionId,
-                        'workflow.id': workflowId,
-                        'workflow.step': workflowStep,
-                        'log.tags': logTags,
-                        'agent.debug': isDebugSession,
-                        'agent.isTest': isTestDomain,
-                    };
+                        span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage });
+                        span.addEvent('cmp.call.error', {
+                            'event.id': eventId,
+                            'cmp.id': componentId,
+                            'cmp.type': componentType,
+                            'cmp.name': componentName,
+                            'error.type': 'result_error',
+                            'error.message': errorMessage,
+                        });
 
-                    // Only include output if formatOutputForLog returns a value
-                    // if (outputForLog !== undefined) {
-                    //     logAttributes['cmp.output'] = outputForLog;
-                    // }
+                        // Add output attributes to span
+                        span.setAttributes({
+                            'output.size': resultStr.length,
+                            'output.has_error': true,
+                        });
 
-                    const spanContext = trace.setSpan(context.active(), span);
-                    context.with(spanContext, () => {
-                        logger.emit({
-                            severityNumber: SeverityNumber.INFO,
-                            severityText: 'INFO',
-                            body: `Component ${componentType} (${componentId}) completed successfully`,
-                            attributes: logAttributes,
+                        // Emit ERROR log for result error
+                        const cmpResultErrorSpanCtx = span.spanContext();
+                        const spanContext = trace.setSpan(context.active(), span);
+                        context.with(spanContext, () => {
+                            logger.emit({
+                                severityNumber: SeverityNumber.ERROR,
+                                severityText: 'ERROR',
+                                body: `Component ${componentType} (${componentId}) failed: ${errorMessage}`,
+                                attributes: {
+                                    // Explicit trace correlation (some backends need these)
+                                    trace_id: cmpResultErrorSpanCtx.traceId,
+                                    span_id: cmpResultErrorSpanCtx.spanId,
+                                    trace_flags: cmpResultErrorSpanCtx.traceFlags,
+
+                                    'agent.id': agentId,
+                                    'agent.name': agentName,
+                                    'process.id': processId,
+                                    'event.id': eventId,
+                                    'cmp.id': componentId,
+                                    'cmp.name': componentName,
+                                    'cmp.type': componentType,
+                                    'error.type': 'result_error',
+                                    'error.message': errorMessage,
+                                    'cmp.output': oTelInstance.redactObject(result),
+                                    'team.id': teamId,
+                                    'org.slot': orgSlot,
+                                    'org.tier': orgTier,
+                                    'source.id': sourceId,
+                                    'source.name': sourceName,
+                                    'session.id': sessionId,
+                                    'workflow.id': workflowId,
+                                    'workflow.step': workflowStep,
+                                    'log.tags': logTags,
+                                    'agent.debug': isDebugSession,
+                                    'agent.isTest': isTestDomain,
+                                },
+                            });
                         });
-                    });
+                    } else {
+                        // True success case
+                        span.setStatus({ code: SpanStatusCode.OK });
+
+                        // Add success event with output summary
+                        span.addEvent('cmp.call.result', {
+                            'output.size': resultStr.length,
+                            'output.preview': oTelInstance.redactString(resultStr.substring(0, 200)),
+                        });
+
+                        // Add output attributes to span
+                        span.setAttributes({
+                            'output.size': resultStr.length,
+                            'output.has_error': false,
+                        });
+
+                        // Emit success log with output (formatted safely)
+                        const cmpSuccessSpanCtx = span.spanContext();
+                        const logAttributes: Record<string, any> = {
+                            // Explicit trace correlation (some backends need these)
+                            trace_id: cmpSuccessSpanCtx.traceId,
+                            span_id: cmpSuccessSpanCtx.spanId,
+                            trace_flags: cmpSuccessSpanCtx.traceFlags,
+
+                            'agent.id': agentId,
+                            'agent.name': agentName,
+                            'cmp.id': componentId,
+                            'cmp.type': componentType,
+                            'cmp.name': componentName,
+                            'process.id': processId,
+                            'event.id': eventId,
+                            'cmp.output': oTelInstance.redactObject(result),
+                            'team.id': teamId,
+                            'org.slot': orgSlot,
+                            'org.tier': orgTier,
+                            'source.id': sourceId,
+                            'source.name': sourceName,
+                            'session.id': sessionId,
+                            'workflow.id': workflowId,
+                            'workflow.step': workflowStep,
+                            'log.tags': logTags,
+                            'agent.debug': isDebugSession,
+                            'agent.isTest': isTestDomain,
+                        };
+
+                        const spanContext = trace.setSpan(context.active(), span);
+                        context.with(spanContext, () => {
+                            logger.emit({
+                                severityNumber: SeverityNumber.INFO,
+                                severityText: 'INFO',
+                                body: `Component ${componentType} (${componentId}) completed successfully`,
+                                attributes: logAttributes,
+                            });
+                        });
+                    }
                 }
 
                 span.end();
             },
-            THook.NonBlocking
+            THook.NonBlocking,
         );
         return Promise.resolve();
     }