@cloudbase/agent-observability 1.0.1-alpha.16 → 1.0.1-alpha.18

package/src/langchain/CallbackHandler.ts

@@ -26,7 +26,7 @@ import {
   type ObservationTool,
   type Observation,
   type ObservationAttributes,
-} from "../index.js";
+} from "../index";
 import type { SpanContext } from "@opentelemetry/api";
 import { type Logger, noopLogger } from "@cloudbase/agent-shared";
 
@@ -104,6 +104,9 @@ export class CallbackHandler extends BaseCallbackHandler {
   // External parent context from AG-UI.Server span
   private externalParentSpanContext?: SpanContext;
 
+  // External metadata from AG-UI.Server (e.g., threadId, runId)
+  private externalMetadata?: Record<string, string>;
+
   // Adapter name for ROOT span prefix
   private adapterName?: string;
 
@@ -130,10 +133,12 @@ export class CallbackHandler extends BaseCallbackHandler {
    * to the server-level span, creating a unified trace hierarchy.
    *
    * @param spanContext - SpanContext from the AG-UI.Server span
+   * @param metadata - Optional metadata from server (e.g., threadId, runId)
    * @public
    */
-  setExternalParentContext(spanContext: SpanContext): void {
+  setExternalParentContext(spanContext: SpanContext, metadata?: Record<string, string>): void {
     this.externalParentSpanContext = spanContext;
+    this.externalMetadata = metadata;
   }
 
   async handleLLMNewToken(
@@ -199,7 +204,7 @@ export class CallbackHandler extends BaseCallbackHandler {
         parentRunId,
         runId,
         tags,
-        metadata,
+        metadata: this.joinTagsAndMetaData(tags, metadata),
         attributes: {
           input: finalInput,
         },
@@ -328,6 +333,8 @@ export class CallbackHandler extends BaseCallbackHandler {
     }
 
     let extractedModelName: string | undefined;
+    let extractedSystem: string | undefined;
+    let extractedProvider: string | undefined;
     if (extraParams) {
       const invocationParamsModelName = (
         extraParams.invocation_params as InvocationParams
@@ -338,6 +345,11 @@ export class CallbackHandler extends BaseCallbackHandler {
           : undefined;
 
       extractedModelName = invocationParamsModelName ?? metadataModelName;
+      
+      // Extract provider from metadata (e.g., "ls_provider" from LangChain)
+      if (metadata && "ls_provider" in metadata) {
+        extractedProvider = metadata["ls_provider"] as string;
+      }
     }
 
     const registeredPrompt = this.promptToParentRunMap.get(
@@ -479,7 +491,7 @@ export class CallbackHandler extends BaseCallbackHandler {
         attributes: {
           input,
         },
-        metadata,
+        metadata: this.joinTagsAndMetaData(tags, metadata),
         tags,
         asType: "tool",
       });
@@ -507,7 +519,7 @@ export class CallbackHandler extends BaseCallbackHandler {
           input: query,
         },
         tags,
-        metadata,
+        metadata: this.joinTagsAndMetaData(tags, metadata),
         asType: "span",
       });
     } catch (e) {
@@ -736,12 +748,19 @@ export class CallbackHandler extends BaseCallbackHandler {
       finalRunName = `Adapter.${this.adapterName}`;
     }
 
+    // Add agui.thread_id and agui.run_id to ALL spans if external metadata available
+    // This ensures consistent tagging across the entire trace hierarchy
+    const serverMetadata = this.externalMetadata
+      ? { "agui.thread_id": this.externalMetadata.threadId, "agui.run_id": this.externalMetadata.runId }
+      : {};
+
     const observation = startObservation(
       finalRunName,
       {
         version: this.version,
         metadata: this.joinTagsAndMetaData(tags, metadata),
         ...attributes,
+        ...serverMetadata,
       },
       {
         asType: asType ?? "span",
@@ -765,6 +784,21 @@ export class CallbackHandler extends BaseCallbackHandler {
       return;
     }
 
+    // Check if this is an error and set span status accordingly
+    const level = attributes.level as string | undefined;
+    const statusMessage = attributes.statusMessage as string | undefined;
+    if (level === "ERROR") {
+      try {
+        const { SpanStatusCode } = require("@opentelemetry/api");
+        observation.otelSpan.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: statusMessage || "Unknown error",
+        });
+      } catch (e) {
+        this.logger.debug?.("Failed to set span error status:", e);
+      }
+    }
+
     // Type-safe update: cast to ObservationAttributes which is the union of all observation attribute types
     observation.update(attributes as ObservationAttributes).end();
 
@@ -797,7 +831,10 @@ export class CallbackHandler extends BaseCallbackHandler {
       return;
     }
 
-    const reservedKeys = ["promptInfo", "userId", "sessionId"];
+    // Filter out keys that should not be in metadata:
+    // - thread_id, run_id: Become agui.thread_id, agui.run_id at span level
+    // - promptInfo, userId, sessionId: Reserved for internal use
+    const reservedKeys = ["promptInfo", "userId", "sessionId", "thread_id", "runId", "run_id"];
 
     return Object.fromEntries(
       Object.entries(metadata).filter(([key, _]) => !reservedKeys.includes(key))
@@ -893,4 +930,5 @@ export class CallbackHandler extends BaseCallbackHandler {
 
     return response;
   }
+
 }

package/src/langchain/index.ts

@@ -4,4 +4,4 @@
  * @module
  */
 
-export { CallbackHandler } from "./CallbackHandler.js";
+export { CallbackHandler } from "./CallbackHandler";

package/src/server/SingleLineConsoleSpanExporter.ts

@@ -0,0 +1,131 @@
+/**
+ * Custom console exporter that outputs single-line JSON.
+ *
+ * This exporter outputs spans as single-line JSON for easier parsing
+ * with line-based tools (grep, jq, etc.).
+ *
+ * To switch back to standard multi-line output, use ConsoleSpanExporter instead.
+ *
+ * @example
+ * ```typescript
+ * // Single-line (current default)
+ * const exporter = new SingleLineConsoleSpanExporter();
+ *
+ * // Multi-line (standard OTel)
+ * import { ConsoleSpanExporter } from '@opentelemetry/sdk-trace-base';
+ * const exporter = new ConsoleSpanExporter();
+ * ```
+ *
+ * @packageDocumentation
+ */
+
+import type { SpanExporter, ReadableSpan } from '@opentelemetry/sdk-trace-base';
+
+/**
+ * Export result codes.
+ */
+export enum ExportResultCode {
+  SUCCESS = 0,
+  FAILED = 1,
+}
+
+/**
+ * Export result interface.
+ */
+export interface ExportResult {
+  code: ExportResultCode;
+  error?: Error;
+}
+
+/**
+ * Custom console exporter that outputs single-line JSON.
+ *
+ * This exporter outputs spans as single-line JSON for easier parsing
+ * with line-based tools (grep, jq, etc.).
+ */
+export class SingleLineConsoleSpanExporter implements SpanExporter {
+  /**
+   * Export spans as single-line JSON.
+   */
+  export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void {
+    try {
+      for (const span of spans) {
+        const spanDict = this.spanToDict(span);
+        // Single-line JSON output
+        const jsonLine = JSON.stringify(spanDict);
+        console.log(jsonLine);
+      }
+      resultCallback({ code: ExportResultCode.SUCCESS });
+    } catch (error) {
+      resultCallback({ code: ExportResultCode.FAILED, error: error as Error });
+    }
+  }
+
+  /**
+   * Shutdown the exporter.
+   */
+  shutdown(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  /**
+   * Force flush the exporter.
+   */
+  forceFlush?(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  /**
+   * Convert a ReadableSpan to a dictionary for JSON serialization.
+   */
+  private spanToDict(span: ReadableSpan): Record<string, unknown> {
+    const context = span.spanContext();
+
+    // Get parent span ID from parentSpanContext if available
+    // Some versions use parentSpanId directly, others use parentSpanContext.spanId
+    const parentId = (span as any).parentSpanId || span.parentSpanContext?.spanId;
+
+    return {
+      name: span.name,
+      context: {
+        trace_id: context.traceId,
+        span_id: context.spanId,
+        trace_flags: context.traceFlags,
+      },
+      kind: span.kind,
+      parent_id: parentId,
+      start_time: span.startTime,
+      end_time: span.endTime,
+      status: {
+        status_code: span.status.code,
+        description: span.status.message,
+      },
+      attributes: span.attributes,
+      events: span.events.map((event) => ({
+        name: event.name,
+        timestamp: event.time,
+        attributes: event.attributes,
+      })),
+      links: span.links.map((link) => ({
+        context: {
+          trace_id: link.context.traceId,
+          span_id: link.context.spanId,
+        },
+        attributes: link.attributes,
+      })),
+      resource: {
+        attributes: span.resource.attributes,
+      },
+    };
+  }
+}
+
+/**
+ * Check if single-line console exporter should be used.
+ *
+ * Set CONSOLE_EXPORTER_SINGLE_LINE=false to use standard multi-line output.
+ */
+export function isSingleLineConsoleExporterEnabled(): boolean {
+  const value = process.env.CONSOLE_EXPORTER_SINGLE_LINE?.toLowerCase() || 'true';
+  return ['true', '1', 'yes', 'on'].includes(value);
+}

package/src/server/index.ts

@@ -15,7 +15,13 @@ export {
   type OTLPTraceConfig,
   type CustomTraceConfig,
   type BatchConfig,
-} from './setup.js';
+} from './setup';
 
 // Exporter type constants
-export { ExporterType } from './config.js';
+export { ExporterType } from './config';
+
+// Single-line console exporter for easier parsing
+export {
+  SingleLineConsoleSpanExporter,
+  isSingleLineConsoleExporterEnabled,
+} from './SingleLineConsoleSpanExporter';

package/src/server/setup.ts

@@ -14,6 +14,10 @@ import type {
   OTLPTraceConfig,
   CustomTraceConfig,
 } from './config';
+import {
+  SingleLineConsoleSpanExporter,
+  isSingleLineConsoleExporterEnabled,
+} from './SingleLineConsoleSpanExporter';
 
 export type {
   BatchConfig,
@@ -22,6 +26,7 @@ export type {
   OTLPTraceConfig,
   CustomTraceConfig,
 } from './config';
+export { SingleLineConsoleSpanExporter } from './SingleLineConsoleSpanExporter';
 
 /**
  * Environment variable truthy values.
@@ -122,6 +127,9 @@ async function safeSetup(
 
 /**
  * Setup console exporter.
+ *
+ * Uses SingleLineConsoleSpanExporter by default for easier parsing with line-based tools.
+ * Set CONSOLE_EXPORTER_SINGLE_LINE=false to use standard multi-line output.
  */
 async function setupConsoleExporter(config: ConsoleTraceConfig): Promise<void> {
   const { trace } = await import('@opentelemetry/api');
@@ -133,18 +141,26 @@ async function setupConsoleExporter(config: ConsoleTraceConfig): Promise<void> {
 
   const batchConfig = resolveBatchConfig(config.batch);
 
+  // Choose exporter type based on CONSOLE_EXPORTER_SINGLE_LINE env var
+  // Single-line: easier for parsing with line-based tools (grep, jq, etc.)
+  // Multi-line: more human-readable for debugging
+  const useSingleLine = isSingleLineConsoleExporterEnabled();
+  const exporter = useSingleLine
+    ? new SingleLineConsoleSpanExporter()
+    : new ConsoleSpanExporter();
+  const exporterType = useSingleLine ? 'single-line' : 'multi-line';
+
   // Check if a real TracerProvider already exists
   let provider = trace.getTracerProvider();
   const isRealProvider = 'addSpanProcessor' in provider;
 
   if (isRealProvider) {
     // Add processor to existing provider
-    const exporter = new ConsoleSpanExporter();
     const processor = new BatchSpanProcessor(exporter, batchConfig);
     (provider as any).addSpanProcessor(processor);
 
     console.info(
-      `[Observability] Console exporter configured (batch=${batchConfig.maxExportBatchSize}, ` +
+      `[Observability] Console exporter configured (${exporterType}, batch=${batchConfig.maxExportBatchSize}, ` +
         `delay=${batchConfig.scheduledDelayMillis}ms)`
     );
   } else {
@@ -154,7 +170,6 @@ async function setupConsoleExporter(config: ConsoleTraceConfig): Promise<void> {
       'service.version': '1.0.0',
     });
 
-    const exporter = new ConsoleSpanExporter();
     const processor = new BatchSpanProcessor(exporter, batchConfig);
 
     const tracerProvider = new NodeTracerProvider({
@@ -165,7 +180,7 @@ async function setupConsoleExporter(config: ConsoleTraceConfig): Promise<void> {
     tracerProvider.register();
 
     console.info(
-      `[Observability] Console exporter configured (batch=${batchConfig.maxExportBatchSize}, ` +
+      `[Observability] Console exporter configured (${exporterType}, batch=${batchConfig.maxExportBatchSize}, ` +
         `delay=${batchConfig.scheduledDelayMillis}ms)`
     );
   }

package/src/types.ts

@@ -55,6 +55,72 @@ export type TokenUsage = {
   totalTokens?: number;
 };
 
+/**
+ * LLM message for input/output tracking.
+ *
+ * Follows OpenInference semantic convention:
+ * - llm.input_messages.{i}.message.role
+ * - llm.input_messages.{i}.message.content
+ * - llm.output_messages.{i}.message.role
+ * - llm.output_messages.{i}.message.content
+ *
+ * @public
+ */
+export type LLMMessage = {
+  /** Message role (e.g., 'user', 'assistant', 'system', 'tool') */
+  role: string;
+  /** Message content */
+  content?: string;
+  /** Tool calls for assistant messages */
+  toolCalls?: ToolCall[];
+  /** Tool call ID for tool messages */
+  toolCallId?: string;
+};
+
+/**
+ * Tool call details.
+ *
+ * Follows OpenInference semantic convention:
+ * - tool_call.id
+ * - tool_call.function.name
+ * - tool_call.function.arguments
+ *
+ * @public
+ */
+export type ToolCall = {
+  /** Tool call ID */
+  id: string;
+  /** Function name */
+  function: {
+    /** Function name */
+    name: string;
+    /** Function arguments as JSON string */
+    arguments: string;
+  };
+};
+
+/**
+ * Document for retrieval tracking.
+ *
+ * Follows OpenInference semantic convention:
+ * - retrieval.documents.{i}.document.id
+ * - retrieval.documents.{i}.document.content
+ * - retrieval.documents.{i}.document.score
+ * - retrieval.documents.{i}.document.metadata
+ *
+ * @public
+ */
+export type Document = {
+  /** Document ID */
+  id?: string;
+  /** Document content */
+  content?: string;
+  /** Relevance score */
+  score?: number;
+  /** Document metadata */
+  metadata?: Record<string, unknown>;
+};
+
 /**
  * Base attributes for all observation types.
  *
@@ -79,6 +145,8 @@ export type BaseSpanAttributes = {
   version?: string;
   /** Environment where the operation is running (e.g., 'production', 'staging') */
   environment?: string;
+  /** Allow additional custom attributes (e.g., http.*, agui.*) */
+  [key: string]: unknown;
 };
 
 /**
@@ -86,7 +154,10 @@ export type BaseSpanAttributes = {
  *
  * Uses attributes like:
  * - llm.model_name
- * - llm.parameters.*
+ * - llm.system
+ * - llm.provider
+ * - llm.input_messages
+ * - llm.output_messages
  * - llm.token_count.*
  * - llm.invocation_parameters
  *
@@ -97,10 +168,22 @@ export type LLMAttributes = BaseSpanAttributes & {
   completionStartTime?: Date;
   /** Name of the language model (e.g., 'gpt-4', 'claude-3-opus') */
   model?: string;
+  /** AI system/vendor identifier (e.g., 'openai', 'anthropic') */
+  system?: string;
+  /** Hosting provider (e.g., 'openai', 'azure', 'aws') */
+  provider?: string;
   /** Parameters passed to the model (temperature, max_tokens, etc.) */
   modelParameters?: Record<string, string | number>;
   /** Token usage metrics */
   usageDetails?: TokenUsage | Record<string, number>;
+  /** Input messages for chat completions */
+  inputMessages?: LLMMessage[];
+  /** Output messages from the model */
+  outputMessages?: LLMMessage[];
+  /** MIME type of input (e.g., 'application/json') */
+  inputMimeType?: string;
+  /** MIME type of output (e.g., 'application/json') */
+  outputMimeType?: string;
 };
 
 /**
@@ -121,10 +204,22 @@ export type EmbeddingAttributes = LLMAttributes;
  * - tool.name
  * - tool.description
  * - tool.parameters
+ * - tool_call.id
+ * - tool_call.function.name
+ * - tool_call.function.arguments
  *
  * @public
  */
-export type ToolAttributes = BaseSpanAttributes;
+export type ToolAttributes = BaseSpanAttributes & {
+  /** Tool name */
+  toolName?: string;
+  /** Tool description */
+  toolDescription?: string;
+  /** Tool parameters schema */
+  toolParameters?: Record<string, unknown>;
+  /** Tool call details */
+  toolCall?: ToolCall;
+};
 
 /**
  * Agent-specific attributes following OpenInference semantic conventions.
@@ -135,7 +230,10 @@ export type ToolAttributes = BaseSpanAttributes;
  *
  * @public
  */
-export type AgentAttributes = BaseSpanAttributes;
+export type AgentAttributes = BaseSpanAttributes & {
+  /** Agent name */
+  agentName?: string;
+};
 
 /**
  * Chain-specific attributes following OpenInference semantic conventions.
@@ -154,11 +252,16 @@ export type ChainAttributes = BaseSpanAttributes;
  * Uses attributes like:
  * - retriever.name
  * - retriever.query
- * - retriever.*
+ * - retrieval.documents
  *
  * @public
  */
-export type RetrieverAttributes = BaseSpanAttributes;
+export type RetrieverAttributes = BaseSpanAttributes & {
+  /** Retrieved documents */
+  documents?: Document[];
+  /** Query used for retrieval */
+  query?: string;
+};
 
 /**
  * Reranker-specific attributes following OpenInference semantic conventions.