@superblocksteam/telemetry 2.0.83-next.1

package/dist/llmobs/tier2-summarizer.d.ts

@@ -0,0 +1,268 @@
+/**
+ * Tier 2 LLMObs Summarizer
+ *
+ * Extracts operational metrics from LLMObs spans for Tier 2 export.
+ * CRITICAL: This emits metrics WITHOUT any sensitive content.
+ *
+ * Tier 2 metrics include:
+ * - LLM latency by provider/model
+ * - Token usage (input, output, cached, reasoning)
+ * - Success/failure rates by error category
+ * - Cost aggregates
+ *
+ * EXPLICITLY EXCLUDED:
+ * - Prompts, code, tool IO
+ * - File paths, stack traces
+ * - User identifiers
+ */
+import type { ReadableSpan } from "@opentelemetry/sdk-trace-base";
+/**
+ * Error categories aligned with tier2-metrics.v0.1.0.json contract.
+ */
+export type LLMErrorCategory = "none" | "rate_limit" | "context_length" | "content_filter" | "quota" | "network" | "timeout" | "auth_error" | "api_error" | "validation_error" | "aborted" | "unknown";
+/**
+ * Finish reason aligned with tier2-metrics.v0.2.0.json contract.
+ */
+export type LLMFinishReason = "stop" | "length" | "content_filter" | "tool_calls" | "error";
+/**
+ * Operation name aligned with OTEL gen_ai.operation.name semantic convention.
+ * @see https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/
+ */
+export type GenAIOperationName = "chat" | "text_completion" | "embeddings" | "execute_tool" | "invoke_agent" | "generate_content";
+/**
+ * Tool error categories aligned with tier2-metrics.v0.2.0.json contract.
+ */
+export type ToolErrorCategory = "none" | "timeout" | "execution_error" | "validation_error" | "permission_denied" | "not_found" | "unknown";
+/**
+ * Provider names aligned with tier2-metrics.v0.1.0.json contract.
+ */
+export type LLMProvider = "openai" | "anthropic" | "azure" | "bedrock" | "google" | "cohere" | "mistral";
+/**
+ * Metric labels for LLM requests.
+ * These are the ONLY labels allowed per tier2-metrics.v0.1.0.json.
+ */
+export interface LLMRequestMetricLabels {
+    provider: LLMProvider;
+    model: string;
+    status: "success" | "error";
+    error_category?: LLMErrorCategory;
+    finish_reason?: LLMFinishReason;
+}
+/**
+ * Token metric labels.
+ */
+export interface TokenMetricLabels {
+    provider: LLMProvider;
+    model: string;
+    token_type: "input" | "output" | "reasoning" | "cached";
+}
+/**
+ * Tool call metric labels.
+ */
+export interface ToolCallMetricLabels {
+    tool_name: string;
+    status: "success" | "error";
+}
+/**
+ * Summary of an LLM span extracted for Tier 2 metrics.
+ * Contains NO sensitive data.
+ */
+export interface LLMSpanSummary {
+    provider: LLMProvider;
+    /** OTEL canonical provider name (e.g., aws.bedrock, azure.ai.openai) */
+    providerOtel: string;
+    model: string;
+    /** Response model if different from request model */
+    responseModel?: string;
+    status: "success" | "error";
+    durationMs: number;
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+    reasoningTokens: number;
+    cachedTokens: number;
+    errorCategory: LLMErrorCategory;
+    finishReason?: LLMFinishReason;
+    estimatedCost: number;
+    /** OTEL operation name (chat, text_completion, etc.) */
+    operationName: GenAIOperationName;
+    /** Time to first token in milliseconds (streaming only) */
+    timeToFirstTokenMs?: number;
+}
+/**
+ * Tool span summary for metrics.
+ */
+export interface ToolSpanSummary {
+    toolName: string;
+    status: "success" | "error";
+    durationMs: number;
+    /** Error category for tool failures (added in v0.2.0) */
+    errorCategory: ToolErrorCategory;
+}
+/**
+ * Metrics client interface for emitting Tier 2 metrics.
+ * Matches OTEL Metrics API patterns.
+ */
+export interface Tier2MetricsClient {
+    recordHistogram(name: string, value: number, labels: Record<string, string>): void;
+    incrementCounter(name: string, value: number, labels: Record<string, string>): void;
+}
+/**
+ * LLMObs Tier 2 Summarizer.
+ *
+ * Processes Tier 1 LLMObs spans and emits Tier 2 metrics.
+ * CRITICAL: Ensures NO prompts, code, tool IO appear in metrics.
+ *
+ * @example
+ * ```typescript
+ * const summarizer = new LLMObsTier2Summarizer(metricsClient);
+ *
+ * // Process each LLMObs span
+ * for (const span of llmobsSpans) {
+ *   summarizer.summarize(span);
+ * }
+ * ```
+ */
+export declare class LLMObsTier2Summarizer {
+    private readonly metricsClient;
+    private readonly pricing;
+    /**
+     * Create a Tier 2 summarizer.
+     *
+     * @param metricsClient - Client for emitting metrics
+     * @param customPricing - Optional custom pricing per model
+     */
+    constructor(metricsClient: Tier2MetricsClient, customPricing?: Map<string, {
+        input: number;
+        output: number;
+    }>);
+    /**
+     * Process a Tier 1 LLMObs span and emit Tier 2 metrics.
+     *
+     * @param span - The span to summarize
+     */
+    summarize(span: ReadableSpan): void;
+    /**
+     * Extract a summary from an LLM span (for testing/debugging).
+     * Contains NO sensitive data.
+     *
+     * @param span - The span to extract from
+     * @returns Summary with safe metrics only
+     */
+    extractLLMSummary(span: ReadableSpan): LLMSpanSummary;
+    /**
+     * Extract a summary from a tool span.
+     *
+     * @param span - The span to extract from
+     * @returns Tool summary with safe metrics only
+     */
+    extractToolSummary(span: ReadableSpan): ToolSpanSummary;
+    /**
+     * Emit LLM span metrics.
+     * Emits BOTH legacy (llm_*) and OTEL-aligned (gen_ai.*) metrics for migration.
+     */
+    private summarizeLLMSpan;
+    /**
+     * Emit tool span metrics.
+     * Uses OTEL-aligned naming with gen_ai.operation.name = "execute_tool".
+     */
+    private summarizeToolSpan;
+    /**
+     * Check if span is an LLM span by name pattern.
+     */
+    private isLLMSpan;
+    /**
+     * Check if span is a tool span by name pattern.
+     */
+    private isToolSpan;
+    /**
+     * Extract span status.
+     */
+    private extractStatus;
+    /**
+     * Categorize error from span attributes.
+     * CRITICAL: Does NOT expose error messages or stack traces.
+     */
+    private categorizeError;
+    /**
+     * Extract finish reason.
+     * Handles gen_ai.response.finish_reasons as array per OTEL semantic conventions.
+     */
+    private extractFinishReason;
+    /**
+     * Normalize provider name to allowed values.
+     * Defensive: handles non-string inputs gracefully.
+     */
+    private normalizeProvider;
+    /**
+     * Normalize provider to OTEL canonical names.
+     * Maps legacy names to OTEL gen_ai.provider.name values.
+     * @see https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-metrics/
+     */
+    private normalizeProviderOtel;
+    /**
+     * Extract operation name from span attributes.
+     * Maps span kind and attributes to OTEL gen_ai.operation.name values.
+     */
+    private extractOperationName;
+    /**
+     * Extract time to first token from span attributes.
+     * Checks multiple attribute names for compatibility.
+     * Returns value in MILLISECONDS for consistent internal handling.
+     *
+     * IMPORTANT: Different attributes use different units:
+     * - gen_ai.client.time_to_first_token: SECONDS (OTEL spec)
+     * - llm.time_to_first_token_ms: MILLISECONDS
+     * - llmobs.metrics.time_to_first_token_ms: MILLISECONDS
+     * - timeToFirstTokenMs: MILLISECONDS
+     */
+    private extractTimeToFirstToken;
+    /**
+     * Categorize tool errors.
+     * CRITICAL: Does NOT expose error messages or stack traces.
+     */
+    private categorizeToolError;
+    /**
+     * Validate operation name is in the allowed enum.
+     */
+    private isValidOperationName;
+    /**
+     * Validate tool error category is in the allowed enum.
+     */
+    private isValidToolErrorCategory;
+    /**
+     * Normalize model name with strict sanitization.
+     * SECURITY: Only allows alphanumeric, dots, hyphens to prevent data injection.
+     * Attacker-controlled model names could otherwise leak sensitive data to Tier 2.
+     */
+    private normalizeModel;
+    /**
+     * Normalize tool name with strict sanitization.
+     * SECURITY: Only allows alphanumeric, underscores, hyphens to prevent data injection.
+     * Attacker-controlled tool names could otherwise leak sensitive data to Tier 2.
+     */
+    private normalizeToolName;
+    /**
+     * Get span duration in milliseconds, clamped to non-negative.
+     * Handles clock skew or corrupted data gracefully.
+     */
+    private getDurationMs;
+    /**
+     * Validate error category is in the allowed enum.
+     */
+    private isValidErrorCategory;
+    /**
+     * Validate finish reason is in the allowed enum.
+     */
+    private isValidFinishReason;
+    /**
+     * Estimate cost based on model and token counts.
+     * Uses rough estimates - actual costs should be calculated server-side.
+     */
+    private estimateCost;
+    /**
+     * Get default pricing per 1000 tokens (in USD).
+     */
+    private getDefaultPricing;
+}
+//# sourceMappingURL=tier2-summarizer.d.ts.map

package/dist/llmobs/tier2-summarizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tier2-summarizer.d.ts","sourceRoot":"","sources":["../../src/llmobs/tier2-summarizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAElE;;GAEG;AACH,MAAM,MAAM,gBAAgB,GACxB,MAAM,GACN,YAAY,GACZ,gBAAgB,GAChB,gBAAgB,GAChB,OAAO,GACP,SAAS,GACT,SAAS,GACT,YAAY,GACZ,WAAW,GACX,kBAAkB,GAClB,SAAS,GACT,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB,MAAM,GACN,QAAQ,GACR,gBAAgB,GAChB,YAAY,GACZ,OAAO,CAAC;AAEZ;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAC1B,MAAM,GACN,iBAAiB,GACjB,YAAY,GACZ,cAAc,GACd,cAAc,GACd,kBAAkB,CAAC;AAEvB;;GAEG;AACH,MAAM,MAAM,iBAAiB,GACzB,MAAM,GACN,SAAS,GACT,iBAAiB,GACjB,kBAAkB,GAClB,mBAAmB,GACnB,WAAW,GACX,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,MAAM,WAAW,GACnB,QAAQ,GACR,WAAW,GACX,OAAO,GACP,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,SAAS,CAAC;AAEd;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,WAAW,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC;IAC5B,cAAc,CAAC,EAAE,gBAAgB,CAAC;IAClC,aAAa,CAAC,EAAE,eAAe,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,OAAO,GAAG,QAAQ,GAAG,WAAW,GAAG,QAAQ,CAAC;CACzD;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,wEAAwE;IACxE,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,gBAAgB,CAAC;IAChC,YAAY,CAAC,EAAE,eAAe,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,wDAAwD;IACxD,aAAa,EAAE,kBAAkB,CAAC;IAClC,2DAA2D;IAC3D,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,yDAAyD;IACzD,aAAa,EAAE,iBAAiB,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,eAAe,CACb,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B,IAAI,CAAC;IACR,gBAAgB,CACd,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B,IAAI,CAAC;CACT;AAED;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqB;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiD;IAEzE;;;;;OAKG;gBAED,aAAa,EAAE,kBAAkB,EACjC,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAMhE;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,YAAY,GAAG,IAAI;IAyBnC;;;;;;OAMG;IACH,iBAAiB,CAAC,IAAI,EAAE,YAAY,GAAG,cAAc;IAyGrD;;;;;OAKG;IACH,kBAAkB,CAAC,IAAI,EAAE,YAAY,GAAG,eAAe;IA6BvD;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAyExB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAkBzB;;OAEG;IACH,OAAO,CAAC,SAAS;IAUjB;;OAEG;IACH,OAAO,CAAC,UAAU;IAOlB;;OAEG;IACH,OAAO,CAAC,aAAa;IAYrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAkDvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAuB3B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAwBzB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAgC7B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IA0B5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,uBAAuB;IAsB/B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAsC3B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAY5B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAahC;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAetB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAezB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAQrB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAW3B;;;OAGG;IACH,OAAO,CAAC,YAAY;IAWpB;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAoB1B"}