ai-inference-stepper 1.0.0

package/src/stepper/README.md

@@ -0,0 +1,43 @@
+# 🎼 Inference Orchestrator
+
+The Orchestrator is the "Head Chef" of the package. It coordinates the various AI providers, handles retries, and ensures that the user always receives a report, even if multiple providers fail.
+
+## 🎯 Purpose
+
+- **Reliability**: Implements a rotation of providers. If one fails, it tries the next.
+- **Resilience**: Uses **Circuit Breakers** and **Exponential Backoff**.
+- **Efficiency**: Respects rate limits via **Bottleneck** (e.g., max 5 requests per minute).
+
+## 🛡️ Resilience Strategies
+
+### 1. Circuit Breaker
+
+If a provider fails more than 50% of the time, the "circuit flips open." The Orchestrator will stop sending requests to that provider for 5 minutes to give it time to recover.
+
+### 2. Smart Retries
+
+When a provider fails with a temporary error (like a network blip), the Orchestrator waits before trying again:
+
+- **Base Delay**: 40 seconds.
+- **Exponential Backoff**: Each retry waits longer than the last.
+- **Jitter**: Adds randomness to avoid "thundering herd" problems.
+
+### 3. Rate Limiting
+
+Controls the flow of requests.
+
+- **Requests Per Minute (RPM)**: Default is 5.
+- **Concurrency**: Default is 2 simultaneous requests.
+
+### 4. Fail-safe Fallback
+
+If _all_ AI providers are down or timeout (after 1 minute), the Orchestrator generates a generic, high-quality template report based on the commit message. This ensures the user is never left with an empty result.
+
+## 📋 Core Functions
+
+| Function                | Description                                                             |
+| ----------------------- | ----------------------------------------------------------------------- |
+| `generateReportNow()`   | The high-level entry point that manages the entire multi-provider flow. |
+| `initializeProviders()` | Sets up the rate limiters and circuit breakers for each service.        |
+| `callWithRetries()`     | Handles the low-level retry logic for a single provider.                |
+| `getProviderHealth()`   | Returns the current status (Healthy/Broken) of all AI services.         |

package/src/stepper/orchestrator.ts

@@ -0,0 +1,437 @@
+
+//packages/stepper/src/stepper/orchestrator.ts`
+
+import Bottleneck from 'bottleneck';
+import CircuitBreaker from 'opossum';
+import { ProviderAdapter, ProviderError, AuthError, RateLimitError } from '../providers/provider.interface.js';
+import { createProviderAdapter } from '../providers/factory.js';
+import { PromptInput, ReportOutput, ProviderResult, ProviderAttemptMeta, StepperCallbacks, ProviderConfig, WebhookCallback } from '../types.js';
+import { config } from '../config.js';
+import { logger, createChildLogger } from '../logging.js';
+import { generateTemplateFallback } from '../fallback/templateFallback.js';
+import { recordProviderAttempt, recordProviderSuccess, recordProviderFailure } from '../metrics/metrics.js';
+import { isRetryableError } from '../utils/safeRequest.js';
+import { alertProviderFailure, alertCircuitOpen } from '../alerts/discord.js';
+
+interface ProviderWithLimiter {
+  adapter: ProviderAdapter;
+  limiter: Bottleneck;
+  circuit: CircuitBreaker;
+  config: ProviderConfig;
+  consecutiveErrors: number;
+}
+
+let providers: ProviderWithLimiter[] = [];
+let callbacks: StepperCallbacks = {};
+
+/**
+ * Initialize providers with rate limiters and circuit breakers
+ */
+export function initializeProviders(providerConfigs: ProviderConfig[] = config.providers): void {
+  providers = providerConfigs
+    .filter((pc) => pc.enabled)
+    .map((pc) => {
+      // Create adapter using factory
+      const adapter = createProviderAdapter(pc);
+      if (!adapter) {
+        throw new Error(`Failed to create adapter for provider ${pc.name}`);
+      }
+
+      // Create Bottleneck limiter
+      // Convert RPM (Requests Per Minute) to ms between requests
+      // Example: 5 RPM = 60000ms / 5 = 12000ms (12 seconds) between each request
+      // const minTime = 60000 / pc.rateLimitRPM;
+
+      // Convert RPM (Requests Per Minute) or RPS (Requests Per Second) to ms between requests
+      const minTime = pc.rateLimitRPS
+        ? 1000 / pc.rateLimitRPS
+        : 60000 / (pc.rateLimitRPM || 5);
+
+      const limiter = new Bottleneck({
+        maxConcurrent: pc.concurrency,
+        minTime: Math.ceil(minTime),
+      });
+
+      // Create circuit breaker
+      const circuit = new CircuitBreaker(async (input: PromptInput) => adapter.call(input), {
+        timeout: pc.timeout || 15000,
+        errorThresholdPercentage: 50,
+        resetTimeout: config.circuit.cooldownSeconds * 1000,
+        volumeThreshold: config.circuit.failureThreshold,
+        rollingCountTimeout: config.circuit.windowSeconds * 1000,
+      });
+
+      circuit.on('open', () => {
+        logger.warn({ provider: pc.name }, 'Circuit breaker opened');
+        void alertCircuitOpen(pc.name);
+      });
+
+      circuit.on('halfOpen', () => {
+        logger.info({ provider: pc.name }, 'Circuit breaker half-open, trying probe');
+      });
+
+      circuit.on('close', () => {
+        logger.info({ provider: pc.name }, 'Circuit breaker closed');
+      });
+
+      return { adapter, limiter, circuit, config: pc, consecutiveErrors: 0 };
+    });
+
+  logger.info({ providerCount: providers.length, names: providers.map((p) => p.config.name) }, 'Providers initialized');
+}
+
+/**
+ * Register lifecycle callbacks
+ */
+export function registerCallbacks(cbs: StepperCallbacks): void {
+  callbacks = { ...callbacks, ...cbs };
+}
+
+/**
+ * Get backoff delay with jitter
+ */
+function getBackoffDelay(attempt: number): number {
+  const base = config.retry.baseDelayMs;
+  const jitter = Math.floor(Math.random() * config.retry.maxJitterMs);
+  return base * Math.pow(2, attempt) + jitter;
+}
+
+/**
+ * Sleep helper
+ */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Generic callback result interface
+ */
+interface CallbackResult {
+  url: string;
+  success: boolean;
+  statusCode?: number;
+  error?: string;
+}
+
+/**
+ * Send a single callback with retry support
+ * Stepper remains agnostic - just sends raw JSON to the URL
+ */
+async function sendCallback(
+  callback: WebhookCallback,
+  payload: unknown
+): Promise<CallbackResult> {
+  const maxAttempts = callback.retry?.maxAttempts ?? 3;
+  const backoffMs = callback.retry?.backoffMs ?? 1000;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      const response = await fetch(callback.url, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'User-Agent': 'Stepper/1.0',
+          'X-Stepper-Timestamp': Date.now().toString(),
+          ...callback.headers,
+        },
+        body: JSON.stringify(payload),
+        signal: AbortSignal.timeout(10000),
+      });
+
+      if (response.ok) {
+        logger.info({ url: callback.url, attempt }, 'Callback succeeded');
+        return { url: callback.url, success: true, statusCode: response.status };
+      }
+
+      // Retry on server errors or rate limits
+      if ((response.status >= 500 || response.status === 429) && attempt < maxAttempts) {
+        const delay = backoffMs * Math.pow(2, attempt - 1);
+        logger.warn({ url: callback.url, status: response.status, delay }, 'Retrying callback');
+        await sleep(delay);
+        continue;
+      }
+
+      logger.error({ url: callback.url, status: response.status }, 'Callback failed');
+      return { url: callback.url, success: false, statusCode: response.status };
+    } catch (error) {
+      if (attempt < maxAttempts) {
+        const delay = backoffMs * Math.pow(2, attempt - 1);
+        logger.warn({ url: callback.url, error: error instanceof Error ? error.message : String(error), delay }, 'Callback error, retrying');
+        await sleep(delay);
+        continue;
+      }
+      return {
+        url: callback.url,
+        success: false,
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+
+  return { url: callback.url, success: false, error: 'Max attempts exceeded' };
+}
+
+/**
+ * Execute all configured callbacks with raw result payload
+ * Stepper remains agnostic - callers decide what to do with the result
+ */
+async function executeCallbacks(
+  callbacks: WebhookCallback[],
+  payload: {
+    success: boolean;
+    result?: ReportOutput;
+    error?: string;
+    metadata: {
+      jobId: string;
+      userId: string;
+      commitSha: string;
+      repo: string;
+      provider?: string;
+      generationTimeMs?: number;
+      timestamp: string;
+    };
+  }
+): Promise<CallbackResult[]> {
+  const results: CallbackResult[] = [];
+
+  for (const callback of callbacks) {
+    const result = await sendCallback(callback, payload);
+    results.push(result);
+
+    if (!result.success && !callback.continueOnFailure) {
+      logger.warn({ url: callback.url }, 'Callback failed, stopping chain');
+      break;
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Call provider with retries
+ */
+async function callWithRetries(
+  provider: ProviderWithLimiter,
+  input: PromptInput,
+  jobId: string
+): Promise<{ result: ReportOutput; durationMs: number }> {
+  const maxAttempts = config.retry.maxAttemptsPerProvider;
+  const log = createChildLogger({ provider: provider.config.name, jobId });
+
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    const startTime = Date.now();
+
+    try {
+      // Use circuit breaker - the result type from Opossum is unknown, but we know
+      // it returns ReportOutput since the circuit wraps adapter.call(input)
+      const result = await provider.circuit.fire(input) as ReportOutput;
+      const durationMs = Date.now() - startTime;
+
+      log.debug({ attempt, durationMs }, 'Provider call succeeded');
+      return { result, durationMs };
+    } catch (error) {
+      const durationMs = Date.now() - startTime;
+      log.warn({ attempt, error: error instanceof Error ? error.message : String(error), durationMs }, 'Provider call failed');
+
+      // Don't retry auth errors
+      if (error instanceof AuthError) {
+        log.error({ error: error.message }, 'Auth error - stopping retries');
+        throw error;
+      }
+
+      // Handle rate limits with Retry-After
+      if (error instanceof RateLimitError) {
+        // Use the AI service's requested wait time, or fallback to config (90 minutes default)
+        const retryAfter = error.retryAfter || config.retry.rateLimitFallbackSeconds;
+        log.info({ retryAfterSeconds: retryAfter, attempt }, 'Rate limited, backing off');
+
+        if (attempt < maxAttempts - 1) {
+          await sleep(retryAfter * 1000);
+          continue;
+        }
+        throw error;
+      }
+
+      // Retry on retryable errors
+      if (isRetryableError(error) && attempt < maxAttempts - 1) {
+        const delay = getBackoffDelay(attempt);
+        log.debug({ delay, attempt }, 'Retrying after backoff');
+        await sleep(delay);
+        continue;
+      }
+
+      throw error;
+    }
+  }
+
+  throw new Error('Max retries exceeded');
+}
+
+/**
+ * Safe callback invocation
+ */
+async function invokeCallback<T extends keyof StepperCallbacks>(
+  name: T,
+  ...args: Parameters<NonNullable<StepperCallbacks[T]>>
+): Promise<void> {
+  const callback = callbacks[name];
+  if (!callback) return;
+
+  try {
+    await (callback as (...args: unknown[]) => void | Promise<void>)(...args);
+  } catch (error) {
+    logger.error({ callback: name, error }, 'Callback threw error');
+  }
+}
+
+/**
+ * Generate report using provider orchestration
+ */
+export async function generateReportNow(input: PromptInput, jobId: string = 'immediate'): Promise<ProviderResult> {
+  const log = createChildLogger({ jobId, userId: input.userId, commitSha: input.commitSha });
+  const startTime = Date.now();
+  const providersAttempted: ProviderAttemptMeta[] = [];
+
+  await invokeCallback('onStart', jobId, input);
+
+  // Ensure providers are initialized
+  if (providers.length === 0) {
+    initializeProviders();
+  }
+
+  // Try each provider in order
+  for (const provider of providers) {
+    const providerName = provider.config.name;
+
+    // Check circuit breaker state
+    if (provider.circuit.opened) {
+      log.info({ provider: providerName }, 'Skipping provider - circuit open');
+      providersAttempted.push({
+        provider: providerName,
+        attemptNumber: 0,
+        skipped: 'circuit_open',
+      });
+      continue;
+    }
+
+    let attemptNumber = 0;
+
+    try {
+      attemptNumber++;
+      log.info({ provider: providerName, attempt: attemptNumber }, 'Attempting provider');
+
+      await invokeCallback('onProviderAttempt', jobId, providerName, attemptNumber, {
+        provider: providerName,
+        attemptNumber,
+      });
+
+      recordProviderAttempt(providerName);
+
+      // Schedule with rate limiter and call with retries
+      const { result, durationMs } = await provider.limiter.schedule(() =>
+        callWithRetries(provider, input, jobId)
+      );
+
+      // Success!
+      provider.consecutiveErrors = 0;
+      const totalMs = Date.now() - startTime;
+      log.info({ provider: providerName, totalMs, providerMs: durationMs }, 'Report generated successfully');
+
+      recordProviderSuccess(providerName, durationMs);
+      providersAttempted.push({
+        provider: providerName,
+        attemptNumber,
+        durationMs,
+      });
+
+      await invokeCallback('onSuccess', jobId, providerName, result, {
+        timings: { totalMs, providerMs: durationMs },
+      });
+
+      // Execute configured callbacks immediately after success
+      // This ensures delivery even if subsequent DB operations fail
+      if (input.callbacks && input.callbacks.length > 0) {
+        const callbackPayload = {
+          success: true,
+          result,
+          metadata: {
+            jobId,
+            userId: input.userId,
+            commitSha: input.commitSha,
+            repo: input.repo,
+            provider: providerName,
+            generationTimeMs: totalMs,
+            timestamp: new Date().toISOString(),
+          },
+        };
+
+        executeCallbacks(input.callbacks, callbackPayload)
+          .then((callbackResults: CallbackResult[]) => {
+            log.info({ callbackResults: callbackResults.map(r => ({ url: r.url, success: r.success })) }, 'Callbacks executed');
+          })
+          .catch((err: unknown) => {
+            log.error({ error: err instanceof Error ? err.message : String(err) }, 'Callbacks execution error');
+          });
+      }
+
+      return {
+        result,
+        usedProvider: providerName,
+        providersAttempted,
+        fallback: false,
+        timings: { totalMs, providerMs: durationMs },
+      };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      const errorCode = error instanceof ProviderError ? error.type : 'UNKNOWN';
+
+      log.warn({ provider: providerName, error: errorMessage, errorCode }, 'Provider failed');
+
+      // Update consecutive errors
+      provider.consecutiveErrors = (provider.consecutiveErrors || 0) + 1;
+
+      recordProviderFailure(providerName, errorCode);
+      providersAttempted.push({
+        provider: providerName,
+        attemptNumber,
+        error: errorMessage,
+        errorCode,
+      });
+
+      // Alert on failure
+      void alertProviderFailure(providerName, provider.consecutiveErrors, error);
+
+      // Continue to next provider
+      continue;
+    }
+  }
+
+  // All providers failed
+  const totalMs = Date.now() - startTime;
+  log.error({ totalMs, providersAttempted: providersAttempted.length }, 'All providers failed, job will be retried');
+
+  if (!config.fallback.enabled) {
+    throw new Error(`All ${providersAttempted.length} provider(s) failed. Job will retry.`);
+  }
+
+  const fallbackResult = generateTemplateFallback(input);
+  await invokeCallback('onFallback', jobId, fallbackResult, { providersAttempted });
+  return {
+    result: fallbackResult,
+    usedProvider: 'fallback',
+    providersAttempted,
+    fallback: true,
+    timings: { totalMs },
+  };
+}
+
+/**
+ * Get provider health status
+ */
+export function getProviderHealth(): Array<{ name: string; circuitOpen: boolean; healthy: boolean }> {
+  return providers.map((p) => ({
+    name: p.config.name,
+    circuitOpen: p.circuit.opened,
+    healthy: !p.circuit.opened,
+  }));
+}

package/src/types.ts

@@ -0,0 +1,238 @@
+// types.ts - Stepper Type Definitions
+
+/**
+ * Generic webhook callback configuration
+ * Stepper sends raw results to these URLs - callers handle transformation
+ */
+export interface WebhookCallback {
+  /** Callback URL to send results */
+  url: string;
+  /** Custom headers (auth tokens, content-type, etc.) */
+  headers?: Record<string, string>;
+  /** Continue to next callback even if this one fails */
+  continueOnFailure?: boolean;
+  /** Retry configuration */
+  retry?: {
+    maxAttempts: number;
+    backoffMs: number;
+  };
+}
+
+/**
+ * Input to generate a commit report
+ */
+export interface PromptInput {
+  userId: string;
+  commitSha: string;
+  repo: string;
+  message: string;
+  files: string[];
+  components: string[];
+  diffSummary: string;
+  template?: string;
+  
+  /**
+   * Multiple webhook callbacks for resilience
+   * Stepper will call each in order, sending the raw result
+   * Use continueOnFailure: true to ensure all callbacks are attempted
+   */
+  callbacks?: WebhookCallback[];
+}
+
+/**
+ * Structured report output from AI providers
+ */
+export interface ReportOutput {
+  title: string;
+  summary: string;
+  changes: string[];
+  rationale: string;
+  impact_and_tests: string;
+  next_steps: string[];
+  tags: string;
+}
+
+/**
+ * Provider attempt result
+ */
+export interface ProviderResult {
+  result: ReportOutput;
+  usedProvider: string;
+  providersAttempted: ProviderAttemptMeta[];
+  fallback: boolean;
+  timings: {
+    totalMs: number;
+    providerMs?: number;
+  };
+}
+
+/**
+ * Metadata for each provider attempt
+ */
+export interface ProviderAttemptMeta {
+  provider: string;
+  attemptNumber: number;
+  error?: string;
+  errorCode?: string;
+  durationMs?: number;
+  skipped?: string;
+}
+
+/**
+ * Cache entry structure
+ */
+export interface CacheEntry {
+  status: 'hydrated' | 'dehydrated' | 'failed';
+  result?: ReportOutput;
+  jobId?: string;
+  providersAttempted?: ProviderAttemptMeta[];
+  timestamps: {
+    created: string;
+    updated: string;
+  };
+  ttl?: number;
+  etag?: string;
+  fallback?: boolean;
+  error?: string;
+}
+
+/**
+ * Job data for BullMQ
+ */
+export interface ReportJobData {
+  jobId: string;
+  input: PromptInput;
+  cacheKey: string;
+  priority?: number;
+  callbackUrl?: string;
+}
+
+/**
+ * Provider error types
+ */
+export enum ProviderErrorType {
+  RateLimit = 'RATE_LIMIT',
+  Auth = 'AUTH_ERROR',
+  Timeout = 'TIMEOUT',
+  Unavailable = 'UNAVAILABLE',
+  InvalidResponse = 'INVALID_RESPONSE',
+  Unknown = 'UNKNOWN',
+}
+
+/**
+ * Lifecycle callbacks for stepper events
+ */
+export interface StepperCallbacks {
+  onEnqueue?: (jobId: string, meta: { input: PromptInput; cacheKey: string }) => void | Promise<void>;
+  onStart?: (jobId: string, input: PromptInput) => void | Promise<void>;
+  onProviderAttempt?: (
+    jobId: string,
+    providerName: string,
+    attemptNumber: number,
+    meta: ProviderAttemptMeta
+  ) => void | Promise<void>;
+  onSuccess?: (
+    jobId: string,
+    providerName: string,
+    result: ReportOutput,
+    meta: { timings: { totalMs: number; providerMs?: number } }
+  ) => void | Promise<void>;
+  onFallback?: (
+    jobId: string,
+    result: ReportOutput,
+    meta: { providersAttempted: ProviderAttemptMeta[] }
+  ) => void | Promise<void>;
+  onFailure?: (
+    jobId: string,
+    errors: ProviderAttemptMeta[],
+    meta: { lastError?: string }
+  ) => void | Promise<void>;
+}
+
+/**
+ * Provider configuration
+ */
+export interface ProviderConfig {
+  name: string;
+  enabled: boolean;
+  baseUrl?: string;
+  modelName?: string;
+  apiKey?: string;
+  apiKeyEnvVar?: string;
+  rateLimitRPM?: number; // Requests Per Minute
+  rateLimitRPS?: number; // Requests Per Second
+  concurrency: number;
+  timeout?: number;
+}
+
+/**
+ * Stepper configuration
+ */
+export interface StepperConfig {
+  providers: ProviderConfig[];
+  providerConfigs?: ProviderConfig[];
+  fallback: {
+    enabled: boolean;
+  };
+  redis: {
+    url: string;
+    keyPrefix: string;
+  };
+  cache: {
+    ttlSeconds: number;
+    staleThresholdSeconds: number;
+    enableStaleWhileRevalidate: boolean;
+  };
+  queue: {
+    name: string;
+    concurrency: number;
+  };
+  webhook: {
+    enabled: boolean;
+    secret: string;
+    maxRetries: number;
+    retryDelayMs: number;
+  };
+  retry: {
+    maxAttemptsPerProvider: number;
+    baseDelayMs: number;
+    maxJitterMs: number;
+    rateLimitFallbackSeconds: number;
+  };
+  circuit: {
+    failureThreshold: number;
+    windowSeconds: number;
+    cooldownSeconds: number;
+  };
+  security: {
+    redactBeforeSend: boolean;
+    // CORS configuration
+    cors: {
+      enabled: boolean;
+      allowedOrigins: string[];
+      allowCredentials: boolean;
+    };
+    // Rate limiting configuration
+    rateLimit: {
+      enabled: boolean;
+      windowMs: number; // Time window in milliseconds
+      maxRequests: number; // Max requests per window per IP
+      maxRequestsPerUser: number; // Max requests per window per userId
+      skipHealthEndpoints: boolean; // Skip rate limiting for /health and /metrics
+    };
+    // Helmet security headers
+    helmet: {
+      enabled: boolean;
+    };
+    // API Key authentication
+    apiKey: {
+      enabled: boolean;
+      headerName: string; // e.g., 'x-api-key'
+      skipHealthEndpoints: boolean; // Skip auth for /health and /metrics
+    };
+  };
+  server: {
+    port: number;
+    metricsPort?: number;
+  };
+}