@senad-d/observme 0.1.0

package/src/pi/event-handlers/llm.ts

@@ -0,0 +1,115 @@
+import { SpanStatusCode } from "@opentelemetry/api";
+import { recordObsSessionLlmCall } from "../../commands/obs-session.ts";
+import { LOG_EVENT_NAMES } from "../../semconv/metrics.ts";
+import { SPAN_NAMES } from "../../semconv/spans.ts";
+import {
+  buildLlmFinalAttributes,
+  buildLlmRequestAttributes,
+  buildLlmResponseAttributes,
+  deleteCurrentLlmRequest,
+  emitLifecycleLog,
+  endActiveSpan,
+  isAssistantMessage,
+  isBashExecutionMessage,
+  isLlmError,
+  llmMetricLabels,
+  nextLlmRequestId,
+  readMessage,
+  recordLlmSizeMetrics,
+  recordLlmUsageMetrics,
+  recordOptionalLlmContent,
+  recordOptionalPromptContent,
+  recordPromptSizeMetric,
+  recordSpanDurationMs,
+  resolveCurrentLlmSpan,
+  resolveLlmParentSpan,
+  startActiveChildSpan,
+} from "../handler-internals.ts";
+import type { HandlerRegistrar } from "../handler-runtime.ts";
+import type { Handler, HandlerSessionState, ObservMeHandlerContext } from "../handler-types.ts";
+import { recordBashExecution } from "./tool-bash.ts";
+
+export function registerLlmHandlers(registrar: HandlerRegistrar, state: HandlerSessionState): void {
+  registrar.add("before_provider_request", createBeforeProviderRequestHandler(state));
+  registrar.add("after_provider_response", createAfterProviderResponseHandler(state));
+  registrar.add("message_end", createMessageEndHandler(state));
+}
+
+function createBeforeProviderRequestHandler(state: HandlerSessionState): Handler {
+  return handleBeforeProviderRequest.bind(undefined, state);
+}
+
+function handleBeforeProviderRequest(
+  state: HandlerSessionState,
+  event: unknown,
+  ctx: ObservMeHandlerContext,
+): void {
+  const session = state.session;
+  if (!session) return;
+
+  const requestId = nextLlmRequestId(session, event);
+  const parentSpan = resolveLlmParentSpan(session);
+  const attributes = buildLlmRequestAttributes(event, ctx, session, requestId);
+  const span = startActiveChildSpan(session, SPAN_NAMES.PI_LLM_REQUEST, parentSpan, attributes, "llm_request");
+
+  recordPromptSizeMetric(session, event, llmMetricLabels(session, attributes));
+  recordOptionalPromptContent(session, span, event);
+  session.currentLlmRequestId = requestId;
+  session.spans.activeLlmRequests.set(requestId, span);
+  session.metrics.llmRequests.add(1, llmMetricLabels(session, attributes));
+  recordObsSessionLlmCall();
+  emitLifecycleLog(session.logger, LOG_EVENT_NAMES.LLM_REQUEST_STARTED, attributes);
+}
+
+function createAfterProviderResponseHandler(state: HandlerSessionState): Handler {
+  return handleAfterProviderResponse.bind(undefined, state);
+}
+
+function handleAfterProviderResponse(
+  state: HandlerSessionState,
+  event: unknown,
+  _ctx: ObservMeHandlerContext,
+): void {
+  const session = state.session;
+  if (!session) return;
+
+  const span = resolveCurrentLlmSpan(session, event);
+  span?.setAttributes(buildLlmResponseAttributes(event));
+}
+
+function createMessageEndHandler(state: HandlerSessionState): Handler {
+  return handleMessageEnd.bind(undefined, state);
+}
+
+function handleMessageEnd(state: HandlerSessionState, event: unknown, _ctx: ObservMeHandlerContext): void {
+  const session = state.session;
+  if (!session) return;
+
+  const message = readMessage(event);
+  if (isBashExecutionMessage(message)) {
+    recordBashExecution(session, event);
+    return;
+  }
+  if (!isAssistantMessage(message)) return;
+
+  const span = resolveCurrentLlmSpan(session, event);
+  const attributes = buildLlmFinalAttributes(message, session);
+  const labels = llmMetricLabels(session, attributes);
+
+  span?.setAttributes(attributes);
+  recordLlmUsageMetrics(session, message, labels);
+  recordLlmSizeMetrics(session, message, labels);
+  recordOptionalLlmContent(session, span, message);
+
+  if (isLlmError(message)) {
+    span?.setStatus({ code: SpanStatusCode.ERROR, message: "provider_error" });
+    session.metrics.llmErrors.add(1, labels);
+    emitLifecycleLog(session.logger, LOG_EVENT_NAMES.LLM_REQUEST_FAILED, attributes, "ERROR");
+  } else {
+    emitLifecycleLog(session.logger, LOG_EVENT_NAMES.LLM_REQUEST_COMPLETED, attributes);
+  }
+
+  recordSpanDurationMs(span, session.metrics.llmRequestDurationMs, labels);
+  endActiveSpan(session, span);
+  deleteCurrentLlmRequest(session, event);
+}

package/src/pi/event-handlers/session-events.ts

@@ -0,0 +1,159 @@
+import { SpanStatusCode } from "@opentelemetry/api";
+import { COMPACTION_ATTRIBUTES, SESSION_ATTRIBUTES } from "../../semconv/attributes.ts";
+import { LOG_EVENT_NAMES } from "../../semconv/metrics.ts";
+import { SPAN_NAMES } from "../../semconv/spans.ts";
+import {
+  buildBranchAttributes,
+  buildBranchPreparationState,
+  buildCompactionAttributes,
+  buildModelChangeAttributes,
+  buildThinkingLevelChangeAttributes,
+  emitStructuredLog,
+  endActiveSpan,
+  metricLabels,
+  modelChangeMetricLabels,
+  resolveOperationParentSpan,
+  startActiveChildSpan,
+  thinkingLevelChangeMetricLabels,
+  updateCurrentSessionAttributes,
+} from "../handler-internals.ts";
+import type { HandlerRegistrar } from "../handler-runtime.ts";
+import type {
+  AttributeMap,
+  Handler,
+  HandlerSessionState,
+  ObservMeHandlerContext,
+  ObservMeTelemetrySession,
+} from "../handler-types.ts";
+
+export function registerSessionEventHandlers(registrar: HandlerRegistrar, state: HandlerSessionState): void {
+  registrar.add("model_select", createModelChangeHandler(state));
+  registrar.add("model_change", createModelChangeHandler(state));
+  registrar.add("thinking_level_select", createThinkingLevelChangeHandler(state));
+  registrar.add("thinking_level_change", createThinkingLevelChangeHandler(state));
+  registrar.add("session_before_tree", createSessionBeforeTreeHandler(state));
+  registrar.add("session_tree", createBranchHandler(state));
+  registrar.add("session_compact", createCompactionHandler(state));
+}
+
+function createModelChangeHandler(state: HandlerSessionState): Handler {
+  return handleModelChange.bind(undefined, state);
+}
+
+function handleModelChange(state: HandlerSessionState, event: unknown, ctx: ObservMeHandlerContext): void {
+  const session = state.session;
+  if (!session) return;
+
+  const attributes = buildModelChangeAttributes(event, ctx, session);
+
+  updateCurrentSessionAttributes(session, attributes, [
+    SESSION_ATTRIBUTES.PI_MODEL_PROVIDER_CURRENT,
+    SESSION_ATTRIBUTES.PI_MODEL_ID_CURRENT,
+  ]);
+  session.metrics.modelChanges.add(1, modelChangeMetricLabels(session, attributes));
+  session.sessionSpan?.addEvent(LOG_EVENT_NAMES.MODEL_CHANGED, attributes);
+  emitStructuredLog(session.logger, LOG_EVENT_NAMES.MODEL_CHANGED, "model", attributes);
+}
+
+function createThinkingLevelChangeHandler(state: HandlerSessionState): Handler {
+  return handleThinkingLevelChange.bind(undefined, state);
+}
+
+function handleThinkingLevelChange(
+  state: HandlerSessionState,
+  event: unknown,
+  ctx: ObservMeHandlerContext,
+): void {
+  const session = state.session;
+  if (!session) return;
+
+  const attributes = buildThinkingLevelChangeAttributes(event, ctx, session);
+
+  updateCurrentSessionAttributes(session, attributes, [SESSION_ATTRIBUTES.PI_THINKING_LEVEL_CURRENT]);
+  session.metrics.thinkingLevelChanges.add(1, thinkingLevelChangeMetricLabels(session));
+  session.sessionSpan?.addEvent(LOG_EVENT_NAMES.THINKING_CHANGED, attributes);
+  emitStructuredLog(session.logger, LOG_EVENT_NAMES.THINKING_CHANGED, "thinking", attributes);
+}
+
+function createSessionBeforeTreeHandler(state: HandlerSessionState): Handler {
+  return handleSessionBeforeTree.bind(undefined, state);
+}
+
+function handleSessionBeforeTree(
+  state: HandlerSessionState,
+  event: unknown,
+  _ctx: ObservMeHandlerContext,
+): void {
+  const session = state.session;
+  if (!session) return;
+
+  session.currentBranchPreparation = buildBranchPreparationState(event, session.config);
+}
+
+function createBranchHandler(state: HandlerSessionState): Handler {
+  return handleBranch.bind(undefined, state);
+}
+
+function handleBranch(state: HandlerSessionState, event: unknown, _ctx: ObservMeHandlerContext): void {
+  const session = state.session;
+  if (!session) return;
+
+  try {
+    recordBranch(session, event);
+  } finally {
+    session.currentBranchPreparation = undefined;
+  }
+}
+
+function recordBranch(session: ObservMeTelemetrySession, event: unknown): void {
+  const attributes = buildBranchAttributes(event, session);
+  const labels = metricLabels(session.config, session.lineage);
+  const span = startActiveChildSpan(session, SPAN_NAMES.PI_BRANCH, resolveOperationParentSpan(session), attributes, "branch");
+
+  span.addEvent(LOG_EVENT_NAMES.BRANCH_CREATED, attributes);
+  span.setStatus({ code: SpanStatusCode.OK });
+  endActiveSpan(session, span);
+  session.metrics.branches.add(1, labels);
+  emitStructuredLog(session.logger, LOG_EVENT_NAMES.BRANCH_CREATED, "branch", attributes);
+}
+
+function createCompactionHandler(state: HandlerSessionState): Handler {
+  return handleCompaction.bind(undefined, state);
+}
+
+function handleCompaction(state: HandlerSessionState, event: unknown, _ctx: ObservMeHandlerContext): void {
+  const session = state.session;
+  if (!session) return;
+
+  recordCompaction(session, event);
+}
+
+function recordCompaction(session: ObservMeTelemetrySession, event: unknown): void {
+  const attributes = buildCompactionAttributes(event, session);
+  const labels = metricLabels(session.config, session.lineage);
+  const span = startActiveChildSpan(
+    session,
+    SPAN_NAMES.PI_COMPACTION,
+    resolveOperationParentSpan(session),
+    attributes,
+    "compaction",
+  );
+
+  span.addEvent(LOG_EVENT_NAMES.COMPACTION_CREATED, attributes);
+  span.setStatus({ code: SpanStatusCode.OK });
+  endActiveSpan(session, span);
+  session.metrics.compactions.add(1, labels);
+  recordCompactionTokensBefore(session, attributes, labels);
+  emitStructuredLog(session.logger, LOG_EVENT_NAMES.COMPACTION_CREATED, "compaction", attributes);
+}
+
+function recordCompactionTokensBefore(
+  session: ObservMeTelemetrySession,
+  attributes: AttributeMap,
+  labels: Record<string, string>,
+): void {
+  const tokensBefore = attributes[COMPACTION_ATTRIBUTES.PI_COMPACTION_TOKENS_BEFORE];
+  if (typeof tokensBefore !== "number") return;
+
+  session.metrics.compactionTokensBefore.record(tokensBefore, labels);
+}

package/src/pi/event-handlers/tool-bash.ts

@@ -0,0 +1,275 @@
+import { SpanStatusCode } from "@opentelemetry/api";
+import { TOOL_ATTRIBUTES } from "../../semconv/attributes.ts";
+import { LOG_EVENT_NAMES } from "../../semconv/metrics.ts";
+import { SPAN_NAMES } from "../../semconv/spans.ts";
+import {
+  bashErrorClass,
+  bashExecutionFailed,
+  bashExecutionMetricLabels,
+  bashFailureMetricLabels,
+  buildBashExecutionAttributes,
+  buildBashPreExecutionAttributes,
+  buildToolCallInputAttributes,
+  buildToolCompletionLogAttributes,
+  buildToolFinalAttributes,
+  buildToolResultAttributes,
+  closePendingUserBashOperation,
+  deleteCurrentToolCall,
+  emitCapturedToolErrorLog,
+  emitLifecycleLog,
+  endActiveSpan,
+  hasBashCompletionResult,
+  mergeToolStateLabels,
+  nextToolCallId,
+  readBashPayload,
+  readBashPreExecutionTimestampMs,
+  recordMissingToolCallIdDrop,
+  recordOptionalBashContent,
+  recordOptionalToolArguments,
+  recordOptionalToolResult,
+  recordSpanDurationMs,
+  recordTelemetryDrop,
+  resolveBashEventDurationMs,
+  resolveCurrentToolCallId,
+  resolveStandaloneBashEventDurationMs,
+  resolveToolCallState,
+  resolveToolParentSpan,
+  safeElapsedDurationMs,
+  startActiveChildSpan,
+  startToolCallState,
+  toolEventHasAmbiguousMissingToolCallId,
+} from "../handler-internals.ts";
+import { monotonicNowMs } from "../handler-runtime.ts";
+import type { HandlerRegistrar } from "../handler-runtime.ts";
+import type {
+  Handler,
+  HandlerSessionState,
+  ObservMeHandlerContext,
+  ObservMeTelemetrySession,
+  PendingBashOperationState,
+  ToolCallState,
+} from "../handler-types.ts";
+
+export function registerToolBashHandlers(registrar: HandlerRegistrar, state: HandlerSessionState): void {
+  registrar.add("tool_execution_start", createToolExecutionStartHandler(state));
+  registrar.add("tool_call", createToolCallHandler(state));
+  registrar.add("tool_result", createToolResultHandler(state));
+  registrar.add("tool_execution_end", createToolExecutionEndHandler(state));
+  registrar.add("user_bash", createUserBashPreExecutionHandler(state));
+  registrar.add("bashExecution", createBashExecutionHandler(state));
+}
+
+export function recordBashExecution(session: ObservMeTelemetrySession, event: unknown): void {
+  const payload = readBashPayload(event);
+  if (!hasBashCompletionResult(payload)) {
+    recordTelemetryDrop(session, "bash_completion_incomplete", { operation: "bash_execution" });
+    return;
+  }
+
+  const pending = session.pendingUserBash;
+  const attributes = buildBashExecutionAttributes(payload, session);
+  const span = pending?.span ?? startActiveChildSpan(
+    session,
+    SPAN_NAMES.PI_BASH_EXECUTION,
+    resolveToolParentSpan(session),
+    attributes,
+    "bash_execution",
+  );
+  const failed = bashExecutionFailed(payload);
+  const labels = bashExecutionMetricLabels(session, payload, failed);
+  const durationMs = resolveBashDurationMs(session, pending, payload);
+
+  span.setAttributes(attributes);
+  recordOptionalBashContent(session, span, payload);
+  session.metrics.bashExecutions.add(1, labels);
+
+  if (failed) {
+    span.setStatus({ code: SpanStatusCode.ERROR, message: bashErrorClass(payload) });
+    session.metrics.bashFailures.add(1, bashFailureMetricLabels(session, payload));
+  } else {
+    span.setStatus({ code: SpanStatusCode.OK });
+  }
+
+  span.addEvent(LOG_EVENT_NAMES.BASH_COMPLETED, attributes);
+  if (durationMs !== undefined) session.metrics.bashDurationMs.record(durationMs, labels);
+  endActiveSpan(session, span);
+  if (session.pendingUserBash === pending) session.pendingUserBash = undefined;
+  emitLifecycleLog(session.logger, LOG_EVENT_NAMES.BASH_COMPLETED, attributes, failed ? "ERROR" : "INFO");
+}
+
+function createToolExecutionStartHandler(state: HandlerSessionState): Handler {
+  return handleToolExecutionStart.bind(undefined, state);
+}
+
+function handleToolExecutionStart(
+  state: HandlerSessionState,
+  event: unknown,
+  _ctx: ObservMeHandlerContext,
+): void {
+  const session = state.session;
+  if (!session) return;
+
+  const toolCallId = nextToolCallId(session, event);
+  const toolState = startToolCallState(session, event, toolCallId);
+  recordOptionalToolArguments(session, toolState.span, event);
+}
+
+function createToolCallHandler(state: HandlerSessionState): Handler {
+  return handleToolCall.bind(undefined, state);
+}
+
+function handleToolCall(state: HandlerSessionState, event: unknown, _ctx: ObservMeHandlerContext): void {
+  const session = state.session;
+  if (!session) return;
+  if (dropAmbiguousToolLifecycleEvent(session, event, "tool_call")) return;
+
+  const toolCallId = resolveCurrentToolCallId(session, event) ?? nextToolCallId(session, event);
+  const toolState = resolveToolCallState(session, event) ?? startToolCallState(session, event, toolCallId);
+  const attributes = buildToolCallInputAttributes(event, session.config);
+
+  toolState.span.setAttributes(attributes);
+  mergeToolStateLabels(toolState, attributes);
+  recordOptionalToolArguments(session, toolState.span, event);
+}
+
+function createToolResultHandler(state: HandlerSessionState): Handler {
+  return handleToolResult.bind(undefined, state);
+}
+
+function handleToolResult(state: HandlerSessionState, event: unknown, _ctx: ObservMeHandlerContext): void {
+  const session = state.session;
+  if (!session) return;
+  if (dropAmbiguousToolLifecycleEvent(session, event, "tool_result")) return;
+
+  const toolState = resolveToolCallState(session, event);
+  if (!toolState) return;
+
+  const attributes = buildToolResultAttributes(event, session.config);
+  toolState.span.setAttributes(attributes);
+  mergeToolStateLabels(toolState, attributes);
+  captureToolResult(session, toolState, event);
+}
+
+function captureToolResult(session: ObservMeTelemetrySession, toolState: ToolCallState, event: unknown): void {
+  const capturedResult = recordOptionalToolResult(session, toolState.span, event);
+  if (capturedResult) toolState.capturedResult = capturedResult;
+}
+
+function createToolExecutionEndHandler(state: HandlerSessionState): Handler {
+  return handleToolExecutionEnd.bind(undefined, state);
+}
+
+function handleToolExecutionEnd(
+  state: HandlerSessionState,
+  event: unknown,
+  _ctx: ObservMeHandlerContext,
+): void {
+  const session = state.session;
+  if (!session) return;
+  if (dropAmbiguousToolLifecycleEvent(session, event, "tool_execution_end")) return;
+
+  const toolCallId = resolveCurrentToolCallId(session, event);
+  const toolState = toolCallId ? resolveToolCallState(session, event) : undefined;
+  if (!toolCallId || !toolState) {
+    recordTelemetryDrop(session, "tool_call_missing_end", { operation: "tool_execution_end" });
+    return;
+  }
+
+  const resultAttributes = buildToolResultAttributes(event, session.config);
+  const finalAttributes = buildToolFinalAttributes(event);
+  const failed = finalAttributes[TOOL_ATTRIBUTES.PI_TOOL_SUCCESS] === false;
+
+  toolState.span.setAttributes({ ...resultAttributes, ...finalAttributes });
+  mergeToolStateLabels(toolState, resultAttributes);
+  captureToolResult(session, toolState, event);
+  const resultSizeChars = resultAttributes[TOOL_ATTRIBUTES.PI_TOOL_RESULT_SIZE];
+  if (typeof resultSizeChars === "number") session.metrics.toolResultSizeChars.record(resultSizeChars, toolState.labels);
+  const completionLogAttributes = buildToolCompletionLogAttributes(toolState, finalAttributes);
+
+  if (failed) {
+    const errorClass = String(finalAttributes[TOOL_ATTRIBUTES.PI_TOOL_ERROR_CLASS] ?? "tool_error");
+    toolState.span.setStatus({ code: SpanStatusCode.ERROR, message: errorClass });
+    session.metrics.toolFailures.add(1, toolState.labels);
+    emitLifecycleLog(session.logger, LOG_EVENT_NAMES.TOOL_CALL_FAILED, completionLogAttributes, "ERROR");
+    if (toolState.capturedResult) emitCapturedToolErrorLog(session, completionLogAttributes, toolState.capturedResult);
+  } else {
+    toolState.span.setStatus({ code: SpanStatusCode.OK });
+    emitLifecycleLog(session.logger, LOG_EVENT_NAMES.TOOL_CALL_COMPLETED, completionLogAttributes);
+  }
+
+  recordSpanDurationMs(toolState.span, session.metrics.toolDurationMs, toolState.labels);
+  endActiveSpan(session, toolState.span);
+  deleteCurrentToolCall(session, toolCallId);
+}
+
+function createUserBashPreExecutionHandler(state: HandlerSessionState): Handler {
+  return handleUserBashPreExecution.bind(undefined, state);
+}
+
+function handleUserBashPreExecution(
+  state: HandlerSessionState,
+  event: unknown,
+  _ctx: ObservMeHandlerContext,
+): void {
+  const session = state.session;
+  if (!session) return;
+
+  startPendingUserBashOperation(session, event);
+}
+
+function createBashExecutionHandler(state: HandlerSessionState): Handler {
+  return handleBashExecution.bind(undefined, state);
+}
+
+function handleBashExecution(state: HandlerSessionState, event: unknown, _ctx: ObservMeHandlerContext): void {
+  const session = state.session;
+  if (!session) return;
+
+  recordBashExecution(session, event);
+}
+
+function dropAmbiguousToolLifecycleEvent(
+  session: ObservMeTelemetrySession,
+  event: unknown,
+  operation: string,
+): boolean {
+  if (!toolEventHasAmbiguousMissingToolCallId(session, event)) return false;
+
+  recordMissingToolCallIdDrop(session, operation);
+  return true;
+}
+
+function startPendingUserBashOperation(session: ObservMeTelemetrySession, event: unknown): void {
+  if (session.pendingUserBash) {
+    closePendingUserBashOperation(session, "bash_overlap_ambiguous", true);
+    return;
+  }
+
+  const attributes = buildBashPreExecutionAttributes(event, session);
+  const span = startActiveChildSpan(
+    session,
+    SPAN_NAMES.PI_BASH_EXECUTION,
+    resolveToolParentSpan(session),
+    attributes,
+    "bash_execution",
+  );
+  session.pendingUserBash = {
+    span,
+    startedAtMs: session.now?.() ?? monotonicNowMs(),
+    eventStartedAtMs: readBashPreExecutionTimestampMs(event),
+  };
+  recordOptionalBashContent(session, span, event);
+}
+
+function resolveBashDurationMs(
+  session: ObservMeTelemetrySession,
+  pending: PendingBashOperationState | undefined,
+  completionEvent: unknown,
+): number | undefined {
+  if (!pending) return resolveStandaloneBashEventDurationMs(completionEvent);
+
+  const eventDurationMs = resolveBashEventDurationMs(pending.eventStartedAtMs, completionEvent);
+  if (eventDurationMs !== undefined) return eventDurationMs;
+
+  return safeElapsedDurationMs(pending.startedAtMs, session.now?.() ?? monotonicNowMs());
+}