palaryn 0.1.0 → 0.3.2

package/src/metrics/collector.ts

@@ -0,0 +1,357 @@
+import * as promClient from 'prom-client';
+
+export class GatewayMetrics {
+  private registry: promClient.Registry;
+
+  // Counters
+  private requestsTotal: promClient.Counter;
+  private policyDecisionsTotal: promClient.Counter;
+  private dlpDetectionsTotal: promClient.Counter;
+  private budgetBlockedTotal: promClient.Counter;
+  private rateLimitBlockedTotal: promClient.Counter;
+  private idempotencyHitsTotal: promClient.Counter;
+  private executorErrorsTotal: promClient.Counter;
+
+  // Cost & usage counters
+  private costUsdTotal: promClient.Counter;
+  private tokenUsageTotal: promClient.Counter;
+
+  // LLM-specific counters/histograms
+  private llmCostUsdTotal: promClient.Counter;
+  private llmTokensTotal: promClient.Counter;
+  private llmRequestsTotal: promClient.Counter;
+  private llmLatencySeconds: promClient.Histogram;
+
+  // Observability: cache and storage health
+  private cacheHitsTotal: promClient.Counter;
+  private cacheMissesTotal: promClient.Counter;
+  private storageWriteFailuresTotal: promClient.Counter;
+  private dlpTruncatedTotal: promClient.Counter;
+
+  // Histograms
+  private requestDuration: promClient.Histogram;
+  private costPerRequest: promClient.Histogram;
+
+  // Gauge
+  private activeApprovals: promClient.Gauge;
+
+  constructor() {
+    this.registry = new promClient.Registry();
+
+    // Enable default metrics (process CPU, memory, etc.) with palaryn_ prefix
+    promClient.collectDefaultMetrics({
+      register: this.registry,
+      prefix: 'palaryn_',
+    });
+
+    // palaryn_requests_total
+    this.requestsTotal = new promClient.Counter({
+      name: 'palaryn_requests_total',
+      help: 'Total number of tool call requests processed by the gateway',
+      labelNames: ['status', 'tool_name', 'capability'],
+      registers: [this.registry],
+    });
+
+    // palaryn_request_duration_seconds
+    this.requestDuration = new promClient.Histogram({
+      name: 'palaryn_request_duration_seconds',
+      help: 'Duration of tool call request processing in seconds',
+      labelNames: ['status', 'tool_name'],
+      buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+      registers: [this.registry],
+    });
+
+    // palaryn_policy_decisions_total
+    this.policyDecisionsTotal = new promClient.Counter({
+      name: 'palaryn_policy_decisions_total',
+      help: 'Total number of policy decisions made',
+      labelNames: ['decision', 'rule_id'],
+      registers: [this.registry],
+    });
+
+    // palaryn_dlp_detections_total
+    this.dlpDetectionsTotal = new promClient.Counter({
+      name: 'palaryn_dlp_detections_total',
+      help: 'Total number of DLP detections',
+      labelNames: ['detection_type', 'severity'],
+      registers: [this.registry],
+    });
+
+    // palaryn_budget_blocked_total
+    this.budgetBlockedTotal = new promClient.Counter({
+      name: 'palaryn_budget_blocked_total',
+      help: 'Total number of requests blocked due to budget limits',
+      labelNames: ['reason_type'],
+      registers: [this.registry],
+    });
+
+    // palaryn_rate_limit_blocked_total
+    this.rateLimitBlockedTotal = new promClient.Counter({
+      name: 'palaryn_rate_limit_blocked_total',
+      help: 'Total number of requests blocked due to rate limiting',
+      labelNames: ['blocked_by'],
+      registers: [this.registry],
+    });
+
+    // palaryn_idempotency_hits_total
+    this.idempotencyHitsTotal = new promClient.Counter({
+      name: 'palaryn_idempotency_hits_total',
+      help: 'Total number of idempotency cache hits (deduplicated requests)',
+      registers: [this.registry],
+    });
+
+    // palaryn_active_approvals
+    this.activeApprovals = new promClient.Gauge({
+      name: 'palaryn_active_approvals',
+      help: 'Current number of pending approval requests',
+      registers: [this.registry],
+    });
+
+    // palaryn_executor_errors_total
+    this.executorErrorsTotal = new promClient.Counter({
+      name: 'palaryn_executor_errors_total',
+      help: 'Total number of executor errors during tool execution',
+      labelNames: ['tool_name', 'error_type'],
+      registers: [this.registry],
+    });
+
+    // palaryn_cost_usd_total
+    this.costUsdTotal = new promClient.Counter({
+      name: 'palaryn_cost_usd_total',
+      help: 'Total cost in USD tracked by the gateway',
+      labelNames: ['source', 'tool_name'],
+      registers: [this.registry],
+    });
+
+    // palaryn_token_usage_total
+    this.tokenUsageTotal = new promClient.Counter({
+      name: 'palaryn_token_usage_total',
+      help: 'Total token usage tracked by the gateway',
+      labelNames: ['type', 'tool_name'],
+      registers: [this.registry],
+    });
+
+    // palaryn_cost_per_request_usd
+    this.costPerRequest = new promClient.Histogram({
+      name: 'palaryn_cost_per_request_usd',
+      help: 'Cost per request in USD',
+      labelNames: ['tool_name'],
+      buckets: [0.0001, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10],
+      registers: [this.registry],
+    });
+
+    // palaryn_llm_cost_usd_total
+    this.llmCostUsdTotal = new promClient.Counter({
+      name: 'palaryn_llm_cost_usd_total',
+      help: 'Total LLM cost in USD by model and provider',
+      labelNames: ['model', 'provider'],
+      registers: [this.registry],
+    });
+
+    // palaryn_llm_tokens_total
+    this.llmTokensTotal = new promClient.Counter({
+      name: 'palaryn_llm_tokens_total',
+      help: 'Total LLM tokens by type, model, and provider',
+      labelNames: ['type', 'model', 'provider'],
+      registers: [this.registry],
+    });
+
+    // palaryn_llm_requests_total
+    this.llmRequestsTotal = new promClient.Counter({
+      name: 'palaryn_llm_requests_total',
+      help: 'Total LLM requests by model, provider, and status',
+      labelNames: ['model', 'provider', 'status'],
+      registers: [this.registry],
+    });
+
+    // palaryn_llm_latency_seconds
+    this.llmLatencySeconds = new promClient.Histogram({
+      name: 'palaryn_llm_latency_seconds',
+      help: 'LLM request latency in seconds by model and provider',
+      labelNames: ['model', 'provider'],
+      buckets: [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],
+      registers: [this.registry],
+    });
+
+    // palaryn_cache_hits_total
+    this.cacheHitsTotal = new promClient.Counter({
+      name: 'palaryn_cache_hits_total',
+      help: 'Total cache hits by store type',
+      labelNames: ['store'],
+      registers: [this.registry],
+    });
+
+    // palaryn_cache_misses_total
+    this.cacheMissesTotal = new promClient.Counter({
+      name: 'palaryn_cache_misses_total',
+      help: 'Total cache misses by store type',
+      labelNames: ['store'],
+      registers: [this.registry],
+    });
+
+    // palaryn_storage_write_failures_total
+    this.storageWriteFailuresTotal = new promClient.Counter({
+      name: 'palaryn_storage_write_failures_total',
+      help: 'Total storage write failures by store and backend',
+      labelNames: ['store', 'backend'],
+      registers: [this.registry],
+    });
+
+    // palaryn_dlp_truncated_total
+    this.dlpTruncatedTotal = new promClient.Counter({
+      name: 'palaryn_dlp_truncated_total',
+      help: 'Total number of DLP scans that were truncated due to response size limits',
+      registers: [this.registry],
+    });
+  }
+
+  /**
+   * Record a completed request with its status, tool info, and duration.
+   */
+  recordRequest(status: string, toolName: string, capability: string, durationSeconds: number): void {
+    this.requestsTotal.inc({ status, tool_name: toolName, capability });
+    this.requestDuration.observe({ status, tool_name: toolName }, durationSeconds);
+  }
+
+  /**
+   * Record a policy engine decision.
+   */
+  recordPolicyDecision(decision: string, ruleId: string): void {
+    this.policyDecisionsTotal.inc({ decision, rule_id: ruleId });
+  }
+
+  /**
+   * Record a DLP detection event.
+   */
+  recordDLPDetection(detectionType: string, severity: string): void {
+    this.dlpDetectionsTotal.inc({ detection_type: detectionType, severity });
+  }
+
+  /**
+   * Record a budget block event.
+   */
+  recordBudgetBlock(reasonType: string): void {
+    this.budgetBlockedTotal.inc({ reason_type: reasonType });
+  }
+
+  /**
+   * Record a rate limit block event.
+   */
+  recordRateLimitBlock(blockedBy: string): void {
+    this.rateLimitBlockedTotal.inc({ blocked_by: blockedBy });
+  }
+
+  /**
+   * Record an idempotency cache hit.
+   */
+  recordIdempotencyHit(): void {
+    this.idempotencyHitsTotal.inc();
+  }
+
+  /**
+   * Record an executor error.
+   */
+  recordExecutorError(toolName: string, errorType: string): void {
+    this.executorErrorsTotal.inc({ tool_name: toolName, error_type: errorType });
+  }
+
+  /**
+   * Set the current count of active (pending) approvals.
+   */
+  setActiveApprovals(count: number): void {
+    this.activeApprovals.set(count);
+  }
+
+  /**
+   * Record a cost observation.
+   */
+  recordCost(costUsd: number, source: string, toolName: string): void {
+    this.costUsdTotal.inc({ source, tool_name: toolName }, costUsd);
+    this.costPerRequest.observe({ tool_name: toolName }, costUsd);
+  }
+
+  /**
+   * Record token usage.
+   */
+  recordTokenUsage(type: string, toolName: string, tokens: number): void {
+    this.tokenUsageTotal.inc({ type, tool_name: toolName }, tokens);
+  }
+
+  /**
+   * Record LLM-specific usage metrics (model, provider, tokens, cost, latency).
+   */
+  recordLLMUsage(params: {
+    model: string;
+    provider: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    costUsd?: number;
+    durationSeconds: number;
+    status: string;
+  }): void {
+    const { model, provider, inputTokens, outputTokens, costUsd, durationSeconds, status } = params;
+
+    this.llmRequestsTotal.inc({ model, provider, status });
+    this.llmLatencySeconds.observe({ model, provider }, durationSeconds);
+
+    if (costUsd !== undefined && costUsd > 0) {
+      this.llmCostUsdTotal.inc({ model, provider }, costUsd);
+    }
+    if (inputTokens !== undefined) {
+      this.llmTokensTotal.inc({ type: 'input', model, provider }, inputTokens);
+    }
+    if (outputTokens !== undefined) {
+      this.llmTokensTotal.inc({ type: 'output', model, provider }, outputTokens);
+    }
+  }
+
+  /**
+   * Record a cache hit for a specific store.
+   */
+  recordCacheHit(store: string): void {
+    this.cacheHitsTotal.inc({ store });
+  }
+
+  /**
+   * Record a cache miss for a specific store.
+   */
+  recordCacheMiss(store: string): void {
+    this.cacheMissesTotal.inc({ store });
+  }
+
+  /**
+   * Record a storage write failure.
+   */
+  recordStorageWriteFailure(store: string, backend: string): void {
+    this.storageWriteFailuresTotal.inc({ store, backend });
+  }
+
+  /**
+   * Record a DLP scan truncation event.
+   */
+  recordDLPTruncation(): void {
+    this.dlpTruncatedTotal.inc();
+  }
+
+  /**
+   * Get serialized Prometheus metrics for the /metrics endpoint.
+   */
+  async getMetrics(): Promise<string> {
+    return this.registry.metrics();
+  }
+
+  /**
+   * Get the content type header for Prometheus metrics responses.
+   */
+  getContentType(): string {
+    return this.registry.contentType;
+  }
+
+  /**
+   * Reset all metrics in the registry. Used during graceful shutdown
+   * to flush metric state before the process exits.
+   */
+  reset(): void {
+    this.registry.resetMetrics();
+  }
+}

package/src/metrics/index.ts

@@ -0,0 +1 @@
+export { GatewayMetrics } from './collector';