@doclo/core 0.1.10 → 0.1.11

package/dist/index.d.ts

@@ -928,4 +928,164 @@ declare function getAllModels(): ResolvedModelMetadata[];
  */
 declare function clearModelRegistry(): void;
 
-export { type AcceptedMimeType, AccessMethod, type AllAutoVariables, type AutoVariablesForNode, type BaseProviderConfig, type CategorizeAutoVariables, type DocumentMimeType, type ExtractAutoVariables, type FeatureName, FlowInputValidationError, type InputRequirements, type ModelMetadata, type ModelQueryFilter, type NormalizedCapabilities, type NormalizedFeatures, type NormalizedProviderMetadata, type OCRProviderConfig, type OutputFormatSupport, type ParseAutoVariables, type PromptVariables, type ProviderConfig, type ProviderInputType, type ProviderInstance, type ProviderMetadataWithModels, type ProviderQueryFilter, type ProviderRegistry, type ProviderSecrets, ProviderVendor, type ResolvedModelMetadata, type VLMProviderConfig, bufferToBase64, bufferToDataUri, buildProviderFromConfig, buildProvidersFromConfigs, clearModelRegistry, clearProviderRegistry, defineMarkerProvider, defineSuryaProvider, defineVLMProvider, detectDocumentType, detectMimeTypeFromBase64, detectMimeTypeFromBase64Async, detectMimeTypeFromBytes, extractBase64, getAllModels, getAllProviders, getCheapestProviderFor, getModelsForNode, getProviderById, getProvidersBySource, getProvidersForLargeFiles, getProvidersForMimeType, isPDFDocument, queryModels, queryProviders, registerProviderMetadata, registerProviderWithModels, resolveDocument, resolveModelMetadata, validateFlowInputFormat, validateMimeType, validateMimeTypeAsync };
+/**
+ * @doclo/core - Retry Utilities
+ *
+ * Shared retry infrastructure for LLM and OCR providers.
+ * Includes exponential backoff, circuit breaker pattern, and error classification.
+ */
+/**
+ * Configuration for retry behavior
+ */
+interface RetryConfig {
+    /** Maximum number of retry attempts (default: 2) */
+    maxRetries?: number;
+    /** Base delay in milliseconds between retries (default: 1000) */
+    retryDelay?: number;
+    /** Enable exponential backoff (default: true) */
+    useExponentialBackoff?: boolean;
+    /** Maximum delay cap in milliseconds (default: 30000) */
+    maxDelay?: number;
+}
+/**
+ * Configuration for circuit breaker behavior
+ */
+interface CircuitBreakerConfig {
+    /** Number of consecutive failures before opening circuit (default: 3) */
+    threshold?: number;
+    /** Time in milliseconds before trying again after circuit opens (default: 30000) */
+    resetTimeout?: number;
+}
+/**
+ * Internal state for a circuit breaker
+ */
+interface CircuitBreakerState {
+    consecutiveFailures: number;
+    lastFailureTime?: number;
+    isOpen: boolean;
+}
+/**
+ * Circuit breaker interface
+ */
+interface CircuitBreaker {
+    /** Check if circuit is currently open (should skip requests) */
+    isOpen(): boolean;
+    /** Record a successful request (resets failure count) */
+    recordSuccess(): void;
+    /** Record a failed request (may open circuit) */
+    recordFailure(): void;
+    /** Get current state for inspection */
+    getState(): CircuitBreakerState;
+}
+/**
+ * Options for the withRetry wrapper
+ */
+interface WithRetryOptions<T> extends RetryConfig {
+    /** Called before each retry attempt (for logging/observability) */
+    onRetry?: (attempt: number, error: Error, delay: number) => void | Promise<void>;
+    /** Override to parse Retry-After header from response errors */
+    getRetryAfter?: (error: Error) => number | undefined;
+    /** Circuit breaker to use (optional) */
+    circuitBreaker?: CircuitBreaker;
+}
+/** Default retry configuration */
+declare const DEFAULT_RETRY_CONFIG: Required<RetryConfig>;
+/** Default circuit breaker configuration */
+declare const DEFAULT_CIRCUIT_BREAKER_CONFIG: Required<CircuitBreakerConfig>;
+/**
+ * Determines if an error is retryable based on its message content.
+ *
+ * Retryable errors include:
+ * - HTTP 408, 429, 500, 502, 503, 504
+ * - Timeout errors
+ * - Rate limit errors
+ * - Network errors (ECONNRESET, ETIMEDOUT, etc.)
+ *
+ * Non-retryable errors include:
+ * - HTTP 400, 401, 403, 404 (client errors)
+ * - Business logic failures
+ *
+ * @param error - The error to classify
+ * @returns true if the error is retryable
+ */
+declare function isRetryableError(error: Error): boolean;
+/**
+ * Extracts HTTP status code from an error message if present.
+ *
+ * @param error - The error to extract status from
+ * @returns The HTTP status code or undefined
+ */
+declare function extractStatusCode(error: Error): number | undefined;
+/**
+ * Parses Retry-After header value from error message or response.
+ * Supports both seconds (integer) and HTTP-date formats.
+ *
+ * @param error - Error that may contain Retry-After information
+ * @returns Delay in milliseconds, or undefined if not found
+ */
+declare function parseRetryAfter(error: Error): number | undefined;
+/**
+ * Calculates the delay before the next retry attempt.
+ *
+ * With exponential backoff enabled (default):
+ * - Attempt 1: baseDelay * 2^0 = 1x baseDelay
+ * - Attempt 2: baseDelay * 2^1 = 2x baseDelay
+ * - Attempt 3: baseDelay * 2^2 = 4x baseDelay
+ * Plus random jitter (0-1000ms) to prevent thundering herd.
+ *
+ * @param attempt - Current attempt number (1-indexed)
+ * @param config - Retry configuration
+ * @returns Delay in milliseconds
+ */
+declare function calculateRetryDelay(attempt: number, config?: RetryConfig): number;
+/**
+ * Creates or retrieves a circuit breaker for a given key.
+ *
+ * Circuit breakers prevent cascading failures by:
+ * 1. Tracking consecutive failures per provider/endpoint
+ * 2. "Opening" the circuit after threshold failures (skipping requests)
+ * 3. Allowing a retry after resetTimeout (half-open state)
+ * 4. Closing the circuit on success
+ *
+ * @param key - Unique identifier (e.g., "datalab:surya" or "openai:gpt-4")
+ * @param config - Circuit breaker configuration
+ * @returns CircuitBreaker instance
+ */
+declare function createCircuitBreaker(key: string, config?: CircuitBreakerConfig): CircuitBreaker;
+/**
+ * Clears all circuit breakers. Useful for testing.
+ */
+declare function clearCircuitBreakers(): void;
+/**
+ * Gets the circuit breaker for a key without creating one.
+ *
+ * @param key - Unique identifier
+ * @returns CircuitBreaker or undefined if not found
+ */
+declare function getCircuitBreaker(key: string): CircuitBreaker | undefined;
+/**
+ * Wraps an async function with retry logic.
+ *
+ * @example
+ * ```typescript
+ * const result = await withRetry(
+ *   () => fetchWithTimeout(url, options),
+ *   {
+ *     maxRetries: 3,
+ *     retryDelay: 1000,
+ *     useExponentialBackoff: true,
+ *     onRetry: (attempt, error, delay) => {
+ *       console.log(`Retry ${attempt} after ${delay}ms: ${error.message}`);
+ *     }
+ *   }
+ * );
+ * ```
+ *
+ * @param fn - Async function to retry
+ * @param options - Retry options
+ * @returns Result of the function
+ * @throws Last error if all retries fail
+ */
+declare function withRetry<T>(fn: () => Promise<T>, options?: WithRetryOptions<T>): Promise<T>;
+
+export { type AcceptedMimeType, AccessMethod, type AllAutoVariables, type AutoVariablesForNode, type BaseProviderConfig, type CategorizeAutoVariables, type CircuitBreaker, type CircuitBreakerConfig, type CircuitBreakerState, DEFAULT_CIRCUIT_BREAKER_CONFIG, DEFAULT_RETRY_CONFIG, type DocumentMimeType, type ExtractAutoVariables, type FeatureName, FlowInputValidationError, type InputRequirements, type ModelMetadata, type ModelQueryFilter, type NormalizedCapabilities, type NormalizedFeatures, type NormalizedProviderMetadata, type OCRProviderConfig, type OutputFormatSupport, type ParseAutoVariables, type PromptVariables, type ProviderConfig, type ProviderInputType, type ProviderInstance, type ProviderMetadataWithModels, type ProviderQueryFilter, type ProviderRegistry, type ProviderSecrets, ProviderVendor, type ResolvedModelMetadata, type RetryConfig, type VLMProviderConfig, type WithRetryOptions, bufferToBase64, bufferToDataUri, buildProviderFromConfig, buildProvidersFromConfigs, calculateRetryDelay, clearCircuitBreakers, clearModelRegistry, clearProviderRegistry, createCircuitBreaker, defineMarkerProvider, defineSuryaProvider, defineVLMProvider, detectDocumentType, detectMimeTypeFromBase64, detectMimeTypeFromBase64Async, detectMimeTypeFromBytes, extractBase64, extractStatusCode, getAllModels, getAllProviders, getCheapestProviderFor, getCircuitBreaker, getModelsForNode, getProviderById, getProvidersBySource, getProvidersForLargeFiles, getProvidersForMimeType, isPDFDocument, isRetryableError, parseRetryAfter, queryModels, queryProviders, registerProviderMetadata, registerProviderWithModels, resolveDocument, resolveModelMetadata, validateFlowInputFormat, validateMimeType, validateMimeTypeAsync, withRetry };

package/dist/index.js

@@ -2288,7 +2288,187 @@ function getAllModels() {
 function clearModelRegistry() {
   modelRegistry.clear();
 }
+
+// src/retry.ts
+var DEFAULT_RETRY_CONFIG = {
+  maxRetries: 2,
+  retryDelay: 1e3,
+  useExponentialBackoff: true,
+  maxDelay: 3e4
+};
+var DEFAULT_CIRCUIT_BREAKER_CONFIG = {
+  threshold: 3,
+  resetTimeout: 3e4
+};
+var RETRYABLE_STATUS_CODES = ["408", "429", "500", "502", "503", "504"];
+var RETRYABLE_ERROR_PATTERNS = [
+  "timeout",
+  "rate limit",
+  "overloaded",
+  "econnreset",
+  "etimedout",
+  "enotfound",
+  "econnrefused",
+  "socket hang up",
+  "network error"
+];
+function isRetryableError(error) {
+  const message = error.message.toLowerCase();
+  for (const code of RETRYABLE_STATUS_CODES) {
+    if (message.includes(code)) {
+      return true;
+    }
+  }
+  for (const pattern of RETRYABLE_ERROR_PATTERNS) {
+    if (message.includes(pattern)) {
+      return true;
+    }
+  }
+  return false;
+}
+function extractStatusCode(error) {
+  const patterns = [
+    /\b(\d{3})\b/,
+    // Just the status code
+    /status[:\s]+(\d{3})/i,
+    /http[:\s]+(\d{3})/i,
+    /failed[:\s]+(\d{3})/i
+  ];
+  for (const pattern of patterns) {
+    const match = error.message.match(pattern);
+    if (match && match[1]) {
+      const code = parseInt(match[1], 10);
+      if (code >= 100 && code < 600) {
+        return code;
+      }
+    }
+  }
+  return void 0;
+}
+function parseRetryAfter(error) {
+  const message = error.message;
+  const match = message.match(/retry-after[:\s]+(\d+)/i);
+  if (match && match[1]) {
+    const seconds = parseInt(match[1], 10);
+    if (!isNaN(seconds) && seconds > 0 && seconds < 3600) {
+      return seconds * 1e3;
+    }
+  }
+  return void 0;
+}
+function calculateRetryDelay(attempt, config = {}) {
+  const {
+    retryDelay = DEFAULT_RETRY_CONFIG.retryDelay,
+    useExponentialBackoff = DEFAULT_RETRY_CONFIG.useExponentialBackoff,
+    maxDelay = DEFAULT_RETRY_CONFIG.maxDelay
+  } = config;
+  if (!useExponentialBackoff) {
+    return retryDelay;
+  }
+  const exponentialDelay = retryDelay * Math.pow(2, attempt - 1);
+  const jitter = Math.random() * 1e3;
+  return Math.min(exponentialDelay + jitter, maxDelay);
+}
+var circuitBreakerRegistry = /* @__PURE__ */ new Map();
+function createCircuitBreaker(key, config = {}) {
+  const existing = circuitBreakerRegistry.get(key);
+  if (existing) {
+    return existing;
+  }
+  const {
+    threshold = DEFAULT_CIRCUIT_BREAKER_CONFIG.threshold,
+    resetTimeout = DEFAULT_CIRCUIT_BREAKER_CONFIG.resetTimeout
+  } = config;
+  let state = {
+    consecutiveFailures: 0,
+    isOpen: false
+  };
+  const circuitBreaker = {
+    isOpen() {
+      if (!state.isOpen) return false;
+      if (state.lastFailureTime && Date.now() - state.lastFailureTime > resetTimeout) {
+        state = {
+          consecutiveFailures: 0,
+          isOpen: false
+        };
+        return false;
+      }
+      return true;
+    },
+    recordSuccess() {
+      state = {
+        consecutiveFailures: 0,
+        isOpen: false
+      };
+    },
+    recordFailure() {
+      state.consecutiveFailures++;
+      state.lastFailureTime = Date.now();
+      if (state.consecutiveFailures >= threshold) {
+        state.isOpen = true;
+        console.warn(`Circuit breaker opened for ${key} after ${state.consecutiveFailures} consecutive failures`);
+      }
+    },
+    getState() {
+      return { ...state };
+    }
+  };
+  circuitBreakerRegistry.set(key, circuitBreaker);
+  return circuitBreaker;
+}
+function clearCircuitBreakers() {
+  circuitBreakerRegistry.clear();
+}
+function getCircuitBreaker(key) {
+  return circuitBreakerRegistry.get(key);
+}
+async function withRetry(fn, options = {}) {
+  const {
+    maxRetries = DEFAULT_RETRY_CONFIG.maxRetries,
+    retryDelay = DEFAULT_RETRY_CONFIG.retryDelay,
+    useExponentialBackoff = DEFAULT_RETRY_CONFIG.useExponentialBackoff,
+    maxDelay = DEFAULT_RETRY_CONFIG.maxDelay,
+    onRetry,
+    getRetryAfter,
+    circuitBreaker
+  } = options;
+  if (circuitBreaker?.isOpen()) {
+    throw new Error("Circuit breaker is open");
+  }
+  let lastError = null;
+  const totalAttempts = maxRetries + 1;
+  for (let attempt = 1; attempt <= totalAttempts; attempt++) {
+    try {
+      const result = await fn();
+      circuitBreaker?.recordSuccess();
+      return result;
+    } catch (error) {
+      lastError = error;
+      const isLastAttempt = attempt === totalAttempts;
+      const canRetry = !isLastAttempt && isRetryableError(lastError);
+      if (!canRetry) {
+        break;
+      }
+      let delay = calculateRetryDelay(attempt, { retryDelay, useExponentialBackoff, maxDelay });
+      const retryAfter = getRetryAfter?.(lastError) ?? parseRetryAfter(lastError);
+      if (retryAfter !== void 0 && retryAfter > 0) {
+        delay = Math.min(retryAfter, maxDelay);
+      }
+      if (onRetry) {
+        await onRetry(attempt, lastError, delay);
+      }
+      await sleep(delay);
+    }
+  }
+  circuitBreaker?.recordFailure();
+  throw lastError;
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
 export {
+  DEFAULT_CIRCUIT_BREAKER_CONFIG,
+  DEFAULT_RETRY_CONFIG,
   FlowExecutionError,
   FlowInputValidationError,
   FlowValidationError,
@@ -2299,9 +2479,12 @@ export {
   bufferToDataUri,
   buildProviderFromConfig,
   buildProvidersFromConfigs,
+  calculateRetryDelay,
   canStartForEachItemFlow,
+  clearCircuitBreakers,
   clearModelRegistry,
   clearProviderRegistry,
+  createCircuitBreaker,
   createIdentity,
   defineMarkerProvider,
   defineSuryaProvider,
@@ -2312,9 +2495,11 @@ export {
   detectMimeTypeFromBytes,
   extractBase64,
   extractErrorMessage,
+  extractStatusCode,
   getAllModels,
   getAllProviders,
   getCheapestProviderFor,
+  getCircuitBreaker,
   getCompatibleTargets,
   getDocumentPageCount,
   getModelsForNode,
@@ -2331,8 +2516,10 @@ export {
   getValidForEachStarters,
   isLocalEndpoint,
   isPDFDocument,
+  isRetryableError,
   node,
   parseProviderString,
+  parseRetryAfter,
   protectReservedVariables,
   queryModels,
   queryProviders,
@@ -2347,6 +2534,7 @@ export {
   validateJson,
   validateMimeType,
   validateMimeTypeAsync,
-  validateNodeConnection
+  validateNodeConnection,
+  withRetry
 };
 //# sourceMappingURL=index.js.map