claude-code-workflow 6.3.18 → 6.3.20

package/ccw/src/core/services/api-key-tester.ts

@@ -0,0 +1,160 @@
+/**
+ * API Key Tester Service
+ * Shared module for testing API key connectivity across different provider types.
+ * Used by both manual testing (litellm-api-routes.ts) and health check service.
+ */
+
+import type { ProviderType } from '../../types/litellm-api-config.js';
+
+/**
+ * Validate API base URL format
+ * Note: This is a local development tool, so we allow localhost and internal networks
+ * for users who run local API gateways or proxies.
+ * @param url - The URL to validate
+ * @returns Object with valid flag and optional error message
+ */
+export function validateApiBaseUrl(url: string): { valid: boolean; error?: string } {
+  try {
+    const parsed = new URL(url);
+
+    // Must be HTTP or HTTPS
+    if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
+      return { valid: false, error: 'URL must use HTTP or HTTPS protocol' };
+    }
+
+    return { valid: true };
+  } catch {
+    return { valid: false, error: 'Invalid URL format' };
+  }
+}
+
+/**
+ * Result of an API key connection test
+ */
+export interface TestResult {
+  /** Whether the API key is valid */
+  valid: boolean;
+  /** Error message if validation failed */
+  error?: string;
+  /** Latency in milliseconds (only if valid) */
+  latencyMs?: number;
+}
+
+/**
+ * Get default API base URL for a provider type
+ */
+export function getDefaultApiBase(providerType: ProviderType): string {
+  const defaults: Record<string, string> = {
+    openai: 'https://api.openai.com/v1',
+    anthropic: 'https://api.anthropic.com/v1',
+    custom: 'https://api.openai.com/v1', // Assume OpenAI-compatible by default
+  };
+  return defaults[providerType] || defaults.openai;
+}
+
+/**
+ * Test API key connection by making a minimal API request
+ * @param providerType - The type of provider (openai, anthropic, custom)
+ * @param apiBase - The base URL for the API
+ * @param apiKey - The API key to test
+ * @param timeout - Timeout in milliseconds (default: 10000)
+ * @returns TestResult indicating if the key is valid
+ */
+export async function testApiKeyConnection(
+  providerType: ProviderType,
+  apiBase: string,
+  apiKey: string,
+  timeout: number = 10000
+): Promise<TestResult> {
+  // Validate URL to prevent SSRF
+  const urlValidation = validateApiBaseUrl(apiBase);
+  if (!urlValidation.valid) {
+    return { valid: false, error: urlValidation.error };
+  }
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  const startTime = Date.now();
+
+  try {
+    if (providerType === 'anthropic') {
+      // Anthropic format: Use /v1/models endpoint (no cost, no model dependency)
+      // This validates the API key without making a billable request
+      const response = await fetch(`${apiBase}/models`, {
+        method: 'GET',
+        headers: {
+          'x-api-key': apiKey,
+          'anthropic-version': '2023-06-01',
+        },
+        signal: controller.signal,
+      });
+
+      clearTimeout(timeoutId);
+      const latencyMs = Date.now() - startTime;
+
+      if (response.ok) {
+        return { valid: true, latencyMs };
+      }
+
+      // Parse error response
+      const errorBody = await response.json().catch(() => ({}));
+      const errorMessage = (errorBody as any)?.error?.message || response.statusText;
+
+      // 401 = invalid API key
+      if (response.status === 401) {
+        return { valid: false, error: 'Invalid API key' };
+      }
+      if (response.status === 403) {
+        return { valid: false, error: 'Access denied - check API key permissions' };
+      }
+      if (response.status === 429) {
+        // Rate limited means the key is valid but being throttled
+        return { valid: true, latencyMs };
+      }
+
+      return { valid: false, error: errorMessage };
+    } else {
+      // OpenAI-compatible format: GET /v1/models
+      const modelsUrl = apiBase.endsWith('/v1') ? `${apiBase}/models` : `${apiBase}/v1/models`;
+      const response = await fetch(modelsUrl, {
+        method: 'GET',
+        headers: {
+          'Authorization': `Bearer ${apiKey}`,
+        },
+        signal: controller.signal,
+      });
+
+      clearTimeout(timeoutId);
+      const latencyMs = Date.now() - startTime;
+
+      if (response.ok) {
+        return { valid: true, latencyMs };
+      }
+
+      // Parse error response
+      const errorBody = await response.json().catch(() => ({}));
+      const errorMessage = (errorBody as any)?.error?.message || response.statusText;
+
+      if (response.status === 401) {
+        return { valid: false, error: 'Invalid API key' };
+      }
+      if (response.status === 403) {
+        return { valid: false, error: 'Access denied - check API key permissions' };
+      }
+      if (response.status === 429) {
+        // Rate limited means the key is valid but being throttled
+        return { valid: true, latencyMs };
+      }
+
+      return { valid: false, error: errorMessage };
+    }
+  } catch (err) {
+    clearTimeout(timeoutId);
+
+    if ((err as Error).name === 'AbortError') {
+      return { valid: false, error: 'Connection timed out' };
+    }
+
+    return { valid: false, error: `Connection failed: ${(err as Error).message}` };
+  }
+}

package/ccw/src/core/services/health-check-service.ts

@@ -0,0 +1,366 @@
+/**
+ * Health Check Service
+ * Singleton service that periodically checks API key health for providers
+ * with health check enabled. Updates key health status and filters unhealthy
+ * keys from rotation endpoints.
+ */
+
+import {
+  getAllProviders,
+  getProvider,
+  updateProvider,
+  resolveEnvVar,
+} from '../../config/litellm-api-config-manager.js';
+import { testApiKeyConnection, getDefaultApiBase } from './api-key-tester.js';
+import type { ProviderCredential, ApiKeyEntry, HealthCheckConfig } from '../../types/litellm-api-config.js';
+
+/**
+ * Internal state for tracking consecutive failures per key
+ */
+interface KeyHealthState {
+  consecutiveFailures: number;
+  cooldownUntil?: Date;
+}
+
+/**
+ * Health Check Service - Singleton
+ * Manages periodic health checks for API keys across all providers
+ */
+export class HealthCheckService {
+  private static instance: HealthCheckService;
+
+  /** Timer handles for each provider's health check interval */
+  private timers: Map<string, NodeJS.Timeout> = new Map();
+
+  /** Track consecutive failures and cooldown per key (providerId:keyId -> state) */
+  private keyStates: Map<string, KeyHealthState> = new Map();
+
+  /** Base directory for config operations */
+  private baseDir: string = '';
+
+  /** Lock to prevent concurrent checks on same provider */
+  private checkingProviders: Set<string> = new Set();
+
+  private constructor() {}
+
+  /**
+   * Get singleton instance
+   */
+  static getInstance(): HealthCheckService {
+    if (!HealthCheckService.instance) {
+      HealthCheckService.instance = new HealthCheckService();
+    }
+    return HealthCheckService.instance;
+  }
+
+  /**
+   * Start health check for a specific provider
+   * @param providerId - The provider ID to start health checks for
+   */
+  startHealthCheck(providerId: string): void {
+    // Stop existing timer if any
+    this.stopHealthCheck(providerId);
+
+    const provider = getProvider(this.baseDir, providerId);
+    if (!provider) {
+      console.warn(`[HealthCheck] Provider not found: ${providerId}`);
+      return;
+    }
+
+    if (!provider.enabled) {
+      console.log(`[HealthCheck] Provider ${providerId} is disabled, skipping`);
+      return;
+    }
+
+    const healthConfig = provider.healthCheck;
+    if (!healthConfig || !healthConfig.enabled) {
+      console.log(`[HealthCheck] Health check not enabled for provider ${providerId}`);
+      return;
+    }
+
+    const intervalMs = (healthConfig.intervalSeconds || 300) * 1000;
+
+    console.log(`[HealthCheck] Starting health check for ${provider.name} (${providerId}), interval: ${healthConfig.intervalSeconds}s`);
+
+    // Run initial check immediately
+    void this.checkProviderNow(providerId);
+
+    // Set up interval
+    const timer = setInterval(() => {
+      void this.checkProviderNow(providerId);
+    }, intervalMs);
+
+    this.timers.set(providerId, timer);
+  }
+
+  /**
+   * Stop health check for a specific provider
+   * @param providerId - The provider ID to stop health checks for
+   */
+  stopHealthCheck(providerId: string): void {
+    const timer = this.timers.get(providerId);
+    if (timer) {
+      clearInterval(timer);
+      this.timers.delete(providerId);
+      console.log(`[HealthCheck] Stopped health check for provider ${providerId}`);
+    }
+  }
+
+  /**
+   * Start health checks for all providers that have health check enabled
+   * @param baseDir - Base directory for config operations
+   */
+  startAllHealthChecks(baseDir: string): void {
+    this.baseDir = baseDir;
+
+    const providers = getAllProviders(baseDir);
+    let startedCount = 0;
+
+    for (const provider of providers) {
+      if (provider.enabled && provider.healthCheck?.enabled) {
+        this.startHealthCheck(provider.id);
+        startedCount++;
+      }
+    }
+
+    if (startedCount > 0) {
+      console.log(`[HealthCheck] Started health checks for ${startedCount} provider(s)`);
+    } else {
+      console.log('[HealthCheck] No providers with health check enabled');
+    }
+  }
+
+  /**
+   * Stop all active health checks
+   */
+  stopAllHealthChecks(): void {
+    const providerIds = Array.from(this.timers.keys());
+    for (const providerId of providerIds) {
+      this.stopHealthCheck(providerId);
+    }
+    this.keyStates.clear();
+    console.log('[HealthCheck] Stopped all health checks');
+  }
+
+  /**
+   * Manually trigger a health check for a provider
+   * @param providerId - The provider ID to check
+   */
+  async checkProviderNow(providerId: string): Promise<void> {
+    // Prevent concurrent checks on same provider
+    if (this.checkingProviders.has(providerId)) {
+      console.log(`[HealthCheck] Already checking provider ${providerId}, skipping`);
+      return;
+    }
+
+    this.checkingProviders.add(providerId);
+
+    try {
+      const provider = getProvider(this.baseDir, providerId);
+      if (!provider) {
+        console.warn(`[HealthCheck] Provider not found: ${providerId}`);
+        return;
+      }
+
+      if (!provider.enabled) {
+        return;
+      }
+
+      const healthConfig = provider.healthCheck;
+      if (!healthConfig) {
+        return;
+      }
+
+      const apiBase = provider.apiBase || getDefaultApiBase(provider.type);
+      const apiKeys = provider.apiKeys || [];
+
+      if (apiKeys.length === 0 && provider.apiKey) {
+        // Single key mode - create virtual key entry
+        await this.checkSingleKey(provider, 'default', provider.apiKey, apiBase, healthConfig);
+      } else {
+        // Multi-key mode
+        for (const keyEntry of apiKeys) {
+          if (!keyEntry.enabled) continue;
+          await this.checkSingleKey(provider, keyEntry.id, keyEntry.key, apiBase, healthConfig);
+        }
+      }
+
+      // Persist updated health statuses
+      const updatedApiKeys = provider.apiKeys;
+      if (updatedApiKeys && updatedApiKeys.length > 0) {
+        try {
+          updateProvider(this.baseDir, providerId, { apiKeys: updatedApiKeys });
+        } catch (err) {
+          console.error(`[HealthCheck] Failed to persist health status for ${providerId}:`, err);
+        }
+      }
+    } finally {
+      this.checkingProviders.delete(providerId);
+    }
+  }
+
+  /**
+   * Check a single API key's health
+   */
+  private async checkSingleKey(
+    provider: ProviderCredential,
+    keyId: string,
+    keyValue: string,
+    apiBase: string,
+    healthConfig: HealthCheckConfig
+  ): Promise<void> {
+    const stateKey = `${provider.id}:${keyId}`;
+    let state = this.keyStates.get(stateKey);
+
+    if (!state) {
+      state = { consecutiveFailures: 0 };
+      this.keyStates.set(stateKey, state);
+    }
+
+    // Check if in cooldown
+    if (state.cooldownUntil && new Date() < state.cooldownUntil) {
+      console.log(`[HealthCheck] Key ${keyId} for ${provider.name} is in cooldown until ${state.cooldownUntil.toISOString()}`);
+      return;
+    }
+
+    // Resolve environment variables
+    const resolvedKey = resolveEnvVar(keyValue);
+    if (!resolvedKey) {
+      console.warn(`[HealthCheck] Key ${keyId} for ${provider.name} has empty value (env var not set?)`);
+      this.updateKeyHealth(provider, keyId, 'unhealthy', 'API key is empty or environment variable not set');
+      return;
+    }
+
+    // Test the key
+    const result = await testApiKeyConnection(provider.type, apiBase, resolvedKey);
+    const now = new Date().toISOString();
+
+    if (result.valid) {
+      // Reset failure count on success
+      state.consecutiveFailures = 0;
+      state.cooldownUntil = undefined;
+      this.updateKeyHealth(provider, keyId, 'healthy', undefined, now, result.latencyMs);
+      console.log(`[HealthCheck] Key ${keyId} for ${provider.name}: healthy (${result.latencyMs}ms)`);
+    } else {
+      // Increment failure count
+      state.consecutiveFailures++;
+
+      if (state.consecutiveFailures >= healthConfig.failureThreshold) {
+        // Mark as unhealthy and enter cooldown
+        const cooldownMs = (healthConfig.cooldownSeconds || 60) * 1000;
+        state.cooldownUntil = new Date(Date.now() + cooldownMs);
+        this.updateKeyHealth(provider, keyId, 'unhealthy', result.error, now);
+        console.warn(`[HealthCheck] Key ${keyId} for ${provider.name}: UNHEALTHY after ${state.consecutiveFailures} failures. Cooldown until ${state.cooldownUntil.toISOString()}`);
+      } else {
+        // Still unknown/degraded, not yet unhealthy
+        this.updateKeyHealth(provider, keyId, 'unknown', result.error, now);
+        console.log(`[HealthCheck] Key ${keyId} for ${provider.name}: failed (${state.consecutiveFailures}/${healthConfig.failureThreshold}): ${result.error}`);
+      }
+    }
+  }
+
+  /**
+   * Update the health status of a key in the provider's apiKeys array
+   */
+  private updateKeyHealth(
+    provider: ProviderCredential,
+    keyId: string,
+    status: 'healthy' | 'unhealthy' | 'unknown',
+    error?: string,
+    timestamp?: string,
+    latencyMs?: number
+  ): void {
+    if (!provider.apiKeys) return;
+
+    const keyEntry = provider.apiKeys.find(k => k.id === keyId);
+    if (keyEntry) {
+      keyEntry.healthStatus = status;
+      keyEntry.lastHealthCheck = timestamp || new Date().toISOString();
+      if (error) {
+        keyEntry.lastError = error;
+      } else {
+        delete keyEntry.lastError;
+      }
+      // Save latency if provided (only on successful checks)
+      if (latencyMs !== undefined) {
+        keyEntry.lastLatencyMs = latencyMs;
+      }
+    }
+  }
+
+  /**
+   * Get the current health status of all keys for a provider
+   */
+  getProviderHealthStatus(providerId: string): Array<{
+    keyId: string;
+    status: 'healthy' | 'unhealthy' | 'unknown';
+    lastCheck?: string;
+    lastError?: string;
+    consecutiveFailures: number;
+    inCooldown: boolean;
+  }> {
+    const provider = getProvider(this.baseDir, providerId);
+    if (!provider || !provider.apiKeys) return [];
+
+    return provider.apiKeys.map(key => {
+      const stateKey = `${providerId}:${key.id}`;
+      const state = this.keyStates.get(stateKey) || { consecutiveFailures: 0 };
+
+      return {
+        keyId: key.id,
+        status: key.healthStatus || 'unknown',
+        lastCheck: key.lastHealthCheck,
+        lastError: key.lastError,
+        consecutiveFailures: state.consecutiveFailures,
+        inCooldown: state.cooldownUntil ? new Date() < state.cooldownUntil : false,
+      };
+    });
+  }
+
+  /**
+   * Check if the service is running health checks for any provider
+   */
+  isRunning(): boolean {
+    return this.timers.size > 0;
+  }
+
+  /**
+   * Get list of provider IDs currently being monitored
+   */
+  getMonitoredProviders(): string[] {
+    return Array.from(this.timers.keys());
+  }
+
+  /**
+   * Clean up all state for a deleted provider
+   * Call this when a provider is deleted to prevent memory leaks
+   * @param providerId - The provider ID to clean up
+   */
+  cleanupProvider(providerId: string): void {
+    // Stop health check timer
+    this.stopHealthCheck(providerId);
+
+    // Remove all key states for this provider
+    const keysToRemove: string[] = [];
+    for (const key of this.keyStates.keys()) {
+      if (key.startsWith(`${providerId}:`)) {
+        keysToRemove.push(key);
+      }
+    }
+
+    for (const key of keysToRemove) {
+      this.keyStates.delete(key);
+    }
+
+    if (keysToRemove.length > 0) {
+      console.log(`[HealthCheck] Cleaned up ${keysToRemove.length} key state(s) for deleted provider ${providerId}`);
+    }
+  }
+}
+
+/**
+ * Get the singleton health check service instance
+ */
+export function getHealthCheckService(): HealthCheckService {
+  return HealthCheckService.getInstance();
+}

package/ccw/src/core/websocket.ts

@@ -1,11 +1,17 @@
-// @ts-nocheck
 import { createHash } from 'crypto';
+import type { IncomingMessage } from 'http';
+import type { Duplex } from 'stream';
 
 // WebSocket clients for real-time notifications
-export const wsClients = new Set();
-
-export function handleWebSocketUpgrade(req, socket, head) {
-  const key = req.headers['sec-websocket-key'];
+export const wsClients = new Set<Duplex>();
+
+export function handleWebSocketUpgrade(req: IncomingMessage, socket: Duplex, _head: Buffer): void {
+  const header = req.headers['sec-websocket-key'];
+  const key = Array.isArray(header) ? header[0] : header;
+  if (!key) {
+    socket.end();
+    return;
+  }
   const acceptKey = createHash('sha1')
     .update(key + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11')
     .digest('base64');
@@ -26,7 +32,7 @@ export function handleWebSocketUpgrade(req, socket, head) {
   console.log(`[WS] Client connected (${wsClients.size} total)`);
 
   // Handle incoming messages
-  socket.on('data', (buffer) => {
+  socket.on('data', (buffer: Buffer) => {
     try {
       const frame = parseWebSocketFrame(buffer);
       if (!frame) return;
@@ -74,7 +80,7 @@ export function handleWebSocketUpgrade(req, socket, head) {
  * Parse WebSocket frame (simplified)
  * Returns { opcode, payload } or null
  */
-export function parseWebSocketFrame(buffer) {
+export function parseWebSocketFrame(buffer: Buffer): { opcode: number; payload: string } | null {
   if (buffer.length < 2) return null;
 
   const firstByte = buffer[0];
@@ -97,7 +103,7 @@ export function parseWebSocketFrame(buffer) {
     offset = 10;
   }
 
-  let mask = null;
+  let mask: Buffer | null = null;
   if (isMasked) {
     mask = buffer.slice(offset, offset + 4);
     offset += 4;
@@ -117,7 +123,7 @@ export function parseWebSocketFrame(buffer) {
 /**
  * Create WebSocket frame
  */
-export function createWebSocketFrame(data) {
+export function createWebSocketFrame(data: unknown): Buffer {
   const payload = Buffer.from(JSON.stringify(data), 'utf8');
   const length = payload.length;
 
@@ -147,7 +153,7 @@ export function createWebSocketFrame(data) {
 /**
  * Broadcast message to all connected WebSocket clients
  */
-export function broadcastToClients(data) {
+export function broadcastToClients(data: unknown): void {
   const frame = createWebSocketFrame(data);
 
   for (const client of wsClients) {
@@ -158,13 +164,15 @@ export function broadcastToClients(data) {
     }
   }
 
-  console.log(`[WS] Broadcast to ${wsClients.size} clients:`, data.type);
+  const eventType =
+    typeof data === 'object' && data !== null && 'type' in data ? (data as { type?: unknown }).type : undefined;
+  console.log(`[WS] Broadcast to ${wsClients.size} clients:`, eventType);
 }
 
 /**
  * Extract session ID from file path
  */
-export function extractSessionIdFromPath(filePath) {
+export function extractSessionIdFromPath(filePath: string): string | null {
   // Normalize path
   const normalized = filePath.replace(/\\/g, '/');