agentic-qe 1.8.4 → 1.9.1

package/dist/telemetry/metrics/collectors/cost.d.ts

@@ -0,0 +1,246 @@
+/**
+ * Token and Cost Tracking Collector (Phase 2 - Action A5)
+ *
+ * Implements middleware for tracking LLM token usage and costs across agents,
+ * tasks, and fleet operations. Supports multiple LLM providers with accurate
+ * cost calculations based on pricing tables.
+ *
+ * Features:
+ * - Token counting for input/output from all LLM API responses
+ * - Per-agent cost tracking with attribution
+ * - Fleet-wide cost aggregation
+ * - Support for Anthropic, OpenRouter, ONNX (local), OpenAI
+ * - Prometheus-compatible metric export
+ * - Cache-aware cost calculation (write premium, read discount)
+ *
+ * @module telemetry/metrics/collectors/cost
+ */
+/**
+ * LLM provider pricing configuration
+ *
+ * Prices are per million tokens (input/output).
+ * Updated: January 2025
+ */
+export interface ProviderPricing {
+    /** Provider name */
+    provider: 'anthropic' | 'openrouter' | 'openai' | 'onnx';
+    /** Model identifier */
+    model: string;
+    /** Input token cost per million */
+    inputCostPerMillion: number;
+    /** Output token cost per million */
+    outputCostPerMillion: number;
+    /** Cache write cost per million (if supported) */
+    cacheWriteCostPerMillion?: number;
+    /** Cache read cost per million (if supported) */
+    cacheReadCostPerMillion?: number;
+}
+/**
+ * Pricing table for supported LLM providers
+ *
+ * Anthropic pricing (January 2025):
+ * - Claude Sonnet 4.5: $3.00 input / $15.00 output per 1M tokens
+ * - Cache write: 25% premium ($3.75/M)
+ * - Cache read: 90% discount ($0.30/M)
+ *
+ * OpenRouter: ~99% cost savings vs Anthropic
+ * ONNX: $0 (local inference)
+ */
+export declare const PRICING_TABLE: ProviderPricing[];
+/**
+ * Token usage from LLM API response
+ */
+export interface TokenUsage {
+    /** Input tokens (prompt) */
+    inputTokens: number;
+    /** Output tokens (completion) */
+    outputTokens: number;
+    /** Cache creation tokens (Anthropic only) */
+    cacheCreationTokens?: number;
+    /** Cache read tokens (Anthropic only) */
+    cacheReadTokens?: number;
+    /** Total tokens */
+    totalTokens: number;
+}
+/**
+ * Cost breakdown for an LLM call
+ */
+export interface CostBreakdown {
+    /** Input cost in USD */
+    inputCost: number;
+    /** Output cost in USD */
+    outputCost: number;
+    /** Cache write cost in USD (if applicable) */
+    cacheWriteCost?: number;
+    /** Cache read cost in USD (if applicable) */
+    cacheReadCost?: number;
+    /** Total cost in USD */
+    totalCost: number;
+    /** Cost savings from caching in USD (if applicable) */
+    cacheSavings?: number;
+}
+/**
+ * Token and cost metrics for an agent or task
+ */
+export interface TokenMetrics {
+    /** Agent or task identifier */
+    id: string;
+    /** Type (agent or task) */
+    type: 'agent' | 'task' | 'fleet';
+    /** Token usage */
+    tokens: TokenUsage;
+    /** Cost breakdown */
+    cost: CostBreakdown;
+    /** Provider and model used */
+    provider: string;
+    model: string;
+    /** Timestamp */
+    timestamp: number;
+}
+/**
+ * CostTracker
+ *
+ * Middleware for tracking LLM token usage and costs with OpenTelemetry metrics.
+ *
+ * Usage:
+ * 1. Wrap LLM calls with trackTokens()
+ * 2. Extract usage from API response
+ * 3. Metrics automatically exported to OTEL collector
+ *
+ * @example
+ * ```typescript
+ * const costTracker = new CostTracker();
+ *
+ * // Track Anthropic call
+ * const response = await anthropic.messages.create({...});
+ * costTracker.trackTokens({
+ *   agentId: 'test-gen-001',
+ *   taskId: 'task-123',
+ *   provider: 'anthropic',
+ *   model: 'claude-sonnet-4',
+ *   usage: {
+ *     inputTokens: response.usage.input_tokens,
+ *     outputTokens: response.usage.output_tokens,
+ *     cacheCreationTokens: response.usage.cache_creation_input_tokens,
+ *     cacheReadTokens: response.usage.cache_read_input_tokens,
+ *     totalTokens: response.usage.input_tokens + response.usage.output_tokens,
+ *   },
+ * });
+ * ```
+ */
+export declare class CostTracker {
+    private tokenCounter;
+    private costCounter;
+    private tokenHistogram;
+    private costHistogram;
+    private activeAgentsGauge;
+    private agentMetrics;
+    private taskMetrics;
+    private fleetMetrics;
+    constructor();
+    /**
+     * Track token usage and costs for an LLM call
+     *
+     * Extracts token counts from API response, calculates costs using pricing table,
+     * and records metrics to OpenTelemetry.
+     *
+     * @param params - Tracking parameters
+     */
+    trackTokens(params: {
+        agentId: string;
+        taskId?: string;
+        provider: string;
+        model: string;
+        usage: TokenUsage;
+        attributes?: Record<string, string | number>;
+    }): void;
+    /**
+     * Get pricing configuration for a provider/model
+     */
+    private getPricing;
+    /**
+     * Calculate cost breakdown from token usage
+     */
+    private calculateCost;
+    /**
+     * Update agent-level metrics (accumulate)
+     */
+    private updateAgentMetrics;
+    /**
+     * Update task-level metrics
+     */
+    private updateTaskMetrics;
+    /**
+     * Update fleet-wide metrics
+     */
+    private updateFleetMetrics;
+    /**
+     * Initialize empty fleet metrics
+     */
+    private initializeFleetMetrics;
+    /**
+     * Record metrics to OpenTelemetry
+     */
+    private recordMetrics;
+    /**
+     * Get metrics for a specific agent
+     */
+    getAgentMetrics(agentId: string): TokenMetrics | null;
+    /**
+     * Get metrics for a specific task
+     */
+    getTaskMetrics(taskId: string): TokenMetrics | null;
+    /**
+     * Get fleet-wide metrics
+     */
+    getFleetMetrics(): TokenMetrics | null;
+    /**
+     * Get all agent metrics
+     */
+    getAllAgentMetrics(): Map<string, TokenMetrics>;
+    /**
+     * Reset all metrics (for testing)
+     */
+    reset(): void;
+    /**
+     * Export metrics in Prometheus format
+     */
+    exportPrometheusMetrics(): string;
+}
+/**
+ * Get or create the global CostTracker instance
+ */
+export declare function getCostTracker(): CostTracker;
+/**
+ * Middleware wrapper for LLM calls with automatic token tracking
+ *
+ * @example
+ * ```typescript
+ * const callLLM = withTokenTracking(
+ *   async (prompt: string) => {
+ *     const response = await anthropic.messages.create({
+ *       model: 'claude-sonnet-4',
+ *       messages: [{ role: 'user', content: prompt }],
+ *     });
+ *     return response;
+ *   },
+ *   {
+ *     agentId: 'test-gen-001',
+ *     taskId: 'task-123',
+ *     provider: 'anthropic',
+ *     model: 'claude-sonnet-4',
+ *   }
+ * );
+ *
+ * const result = await callLLM('Generate tests for...');
+ * ```
+ */
+export declare function withTokenTracking<T extends {
+    usage: any;
+}>(fn: (...args: any[]) => Promise<T>, context: {
+    agentId: string;
+    taskId?: string;
+    provider: string;
+    model: string;
+}): (...args: any[]) => Promise<T>;
+//# sourceMappingURL=cost.d.ts.map

package/dist/telemetry/metrics/collectors/cost.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.d.ts","sourceRoot":"","sources":["../../../../src/telemetry/metrics/collectors/cost.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,oBAAoB;IACpB,QAAQ,EAAE,WAAW,GAAG,YAAY,GAAG,QAAQ,GAAG,MAAM,CAAC;IACzD,uBAAuB;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,oCAAoC;IACpC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,kDAAkD;IAClD,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,iDAAiD;IACjD,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,aAAa,EAAE,eAAe,EA8E1C,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,4BAA4B;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,6CAA6C;IAC7C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,yCAAyC;IACzC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,wBAAwB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,yBAAyB;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6CAA6C;IAC7C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,wBAAwB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,+BAA+B;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,2BAA2B;IAC3B,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,CAAC;IACjC,kBAAkB;IAClB,MAAM,EAAE,UAAU,CAAC;IACnB,qBAAqB;IACrB,IAAI,EAAE,aAAa,CAAC;IACpB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,WAAW;IAEtB,OAAO,CAAC,YAAY,CAAU;IAC9B,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,cAAc,CAAY;IAClC,OAAO,CAAC,aAAa,CAAY;IACjC,OAAO,CAAC,iBAAiB,CAAgB;IAGzC,OAAO,CAAC,YAAY,CAAwC;IAC5D,OAAO,CAAC,WAAW,CAAwC;IAC3D,OAAO,CAAC,YAAY,CAA6B;;IAsCjD;;;;;;;OAOG;IACH,WAAW,CAAC,MAAM,EAAE;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,UAAU,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;KAC9C,GAAG,IAAI;IAuCR;;OAEG;IACH,OAAO,CAAC,UAAU;IASlB;;OAEG;IACH,OAAO,CAAC,aAAa;IAoCrB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAsC1B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAKzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAoC1B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAyB9B;;OAEG;IACH,OAAO,CAAC,aAAa;IAsDrB;;OAEG;IACH,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI;IAIrD;;OAEG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI;IAInD;;OAEG;IACH,eAAe,IAAI,YAAY,GAAG,IAAI;IAItC;;OAEG;IACH,kBAAkB,IAAI,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC;IAI/C;;OAEG;IACH,KAAK,IAAI,IAAI;IAMb;;OAEG;IACH,uBAAuB,IAAI,MAAM;CAmClC;AAOD;;GAEG;AACH,wBAAgB,cAAc,IAAI,WAAW,CAK5C;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,SAAS;IAAE,KAAK,EAAE,GAAG,CAAA;CAAE,EACxD,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC,EAClC,OAAO,EAAE;IACP,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf,GACA,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC,CAsBhC"}