@chappibunny/repolens 0.4.3 → 0.6.2

package/src/utils/rate-limit.js

@@ -0,0 +1,289 @@
+/**
+ * Rate Limiting for External APIs
+ * 
+ * Implements rate limiting and request throttling for Notion API and other services
+ * to prevent abuse and respect API limits.
+ */
+
+/**
+ * Rate limiter class using token bucket algorithm
+ */
+class RateLimiter {
+  constructor(options = {}) {
+    this.maxRequests = options.maxRequests || 3; // Notion: 3 requests per second
+    this.timeWindow = options.timeWindow || 1000; // 1 second
+    this.tokens = this.maxRequests;
+    this.lastRefill = Date.now();
+    this.queue = [];
+  }
+  
+  /**
+   * Refill tokens based on elapsed time
+   */
+  refill() {
+    const now = Date.now();
+    const elapsed = now - this.lastRefill;
+    
+    if (elapsed >= this.timeWindow) {
+      const tokensToAdd = Math.floor(elapsed / this.timeWindow) * this.maxRequests;
+      this.tokens = Math.min(this.maxRequests, this.tokens + tokensToAdd);
+      this.lastRefill = now;
+    }
+  }
+  
+  /**
+   * Attempt to consume a token
+   * @returns {boolean} True if token available, false otherwise
+   */
+  tryConsume() {
+    this.refill();
+    
+    if (this.tokens > 0) {
+      this.tokens--;
+      return true;
+    }
+    
+    return false;
+  }
+  
+  /**
+   * Wait for a token to become available
+   * @returns {Promise<void>}
+   */
+  async waitForToken() {
+    return new Promise((resolve) => {
+      const attemptConsume = () => {
+        if (this.tryConsume()) {
+          resolve();
+        } else {
+          // Wait a bit and try again
+          const waitTime = Math.max(50, (this.timeWindow / this.maxRequests) / 2);
+          setTimeout(attemptConsume, waitTime);
+        }
+      };
+      
+      attemptConsume();
+    });
+  }
+  
+  /**
+   * Execute a function with rate limiting
+   * @param {Function} fn - Async function to execute
+   * @returns {Promise<any>}
+   */
+  async execute(fn) {
+    await this.waitForToken();
+    return await fn();
+  }
+}
+
+/**
+ * Retry logic with exponential backoff
+ */
+class RetryPolicy {
+  constructor(options = {}) {
+    this.maxRetries = options.maxRetries || 3;
+    this.initialDelay = options.initialDelay || 1000; // 1 second
+    this.maxDelay = options.maxDelay || 30000; // 30 seconds
+    this.backoffMultiplier = options.backoffMultiplier || 2;
+    this.retryableStatuses = options.retryableStatuses || [429, 500, 502, 503, 504];
+  }
+  
+  /**
+   * Execute function with retry logic
+   * @param {Function} fn - Async function to execute
+   * @param {object} context - Context for logging
+   * @returns {Promise<any>}
+   */
+  async execute(fn, context = {}) {
+    let lastError;
+    let delay = this.initialDelay;
+    
+    for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
+      try {
+        return await fn();
+      } catch (error) {
+        lastError = error;
+        
+        // Check if error is retryable
+        if (!this.shouldRetry(error, attempt)) {
+          throw error;
+        }
+        
+        // Log retry attempt
+        if (context.onRetry) {
+          context.onRetry(attempt + 1, this.maxRetries, delay, error);
+        }
+        
+        // Wait before retrying
+        await this.sleep(delay);
+        
+        // Increase delay for next retry (exponential backoff)
+        delay = Math.min(delay * this.backoffMultiplier, this.maxDelay);
+        
+        // Add jitter to prevent thundering herd
+        delay += Math.random() * 1000;
+      }
+    }
+    
+    throw lastError;
+  }
+  
+  /**
+   * Check if error should trigger a retry
+   */
+  shouldRetry(error, attempt) {
+    // Don't retry if max attempts reached
+    if (attempt >= this.maxRetries) {
+      return false;
+    }
+    
+    // Retry on rate limit errors
+    if (error.status === 429 || error.code === "rate_limited") {
+      return true;
+    }
+    
+    // Retry on retryable HTTP statuses
+    if (error.status && this.retryableStatuses.includes(error.status)) {
+      return true;
+    }
+    
+    // Retry on network errors
+    if (error.code === "ECONNRESET" || error.code === "ETIMEDOUT") {
+      return true;
+    }
+    
+    return false;
+  }
+  
+  /**
+   * Sleep for specified milliseconds
+   */
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+// Global rate limiters for different services
+const notionRateLimiter = new RateLimiter({
+  maxRequests: 3, // Notion: 3 requests per second
+  timeWindow: 1000,
+});
+
+const openAIRateLimiter = new RateLimiter({
+  maxRequests: 3, // Conservative: 3 requests per second
+  timeWindow: 1000,
+});
+
+/**
+ * Execute Notion API request with rate limiting and retries
+ * @param {Function} fn - Async function that makes Notion API call
+ * @param {object} options - Options for rate limiting and retries
+ * @returns {Promise<any>}
+ */
+export async function executeNotionRequest(fn, options = {}) {
+  const retryPolicy = new RetryPolicy({
+    maxRetries: options.maxRetries || 3,
+    initialDelay: options.initialDelay || 1000,
+  });
+  
+  return await notionRateLimiter.execute(async () => {
+    return await retryPolicy.execute(fn, {
+      onRetry: (attempt, maxRetries, delay, error) => {
+        if (options.onRetry) {
+          options.onRetry(attempt, maxRetries, delay, error);
+        } else {
+          console.warn(
+            `Notion API retry ${attempt}/${maxRetries} after ${Math.round(delay)}ms (${error.message})`
+          );
+        }
+      },
+    });
+  });
+}
+
+/**
+ * Execute OpenAI/AI API request with rate limiting and retries
+ * @param {Function} fn - Async function that makes AI API call
+ * @param {object} options - Options for rate limiting and retries
+ * @returns {Promise<any>}
+ */
+export async function executeAIRequest(fn, options = {}) {
+  const retryPolicy = new RetryPolicy({
+    maxRetries: options.maxRetries || 2, // Fewer retries for AI (can be expensive)
+    initialDelay: options.initialDelay || 2000,
+    maxDelay: options.maxDelay || 60000, // AI can take longer
+  });
+  
+  return await openAIRateLimiter.execute(async () => {
+    return await retryPolicy.execute(fn, {
+      onRetry: (attempt, maxRetries, delay, error) => {
+        if (options.onRetry) {
+          options.onRetry(attempt, maxRetries, delay, error);
+        } else {
+          console.warn(
+            `AI API retry ${attempt}/${maxRetries} after ${Math.round(delay)}ms (${error.message})`
+          );
+        }
+      },
+    });
+  });
+}
+
+/**
+ * Batch API requests to respect rate limits
+ * @param {Array<Function>} requests - Array of async functions
+ * @param {object} options - Batching options
+ * @returns {Promise<Array<any>>}
+ */
+export async function batchRequests(requests, options = {}) {
+  const batchSize = options.batchSize || 3;
+  const results = [];
+  
+  for (let i = 0; i < requests.length; i += batchSize) {
+    const batch = requests.slice(i, i + batchSize);
+    const batchResults = await Promise.all(
+      batch.map(fn => fn().catch(err => ({ error: err })))
+    );
+    results.push(...batchResults);
+    
+    // Progress callback
+    if (options.onProgress) {
+      options.onProgress(Math.min(i + batchSize, requests.length), requests.length);
+    }
+  }
+  
+  return results;
+}
+
+/**
+ * Create a rate-limited version of a function
+ * @param {Function} fn - Function to rate limit
+ * @param {object} options - Rate limiter options
+ * @returns {Function} Rate-limited function
+ */
+export function rateLimit(fn, options = {}) {
+  const limiter = new RateLimiter(options);
+  
+  return async function(...args) {
+    return await limiter.execute(() => fn(...args));
+  };
+}
+
+/**
+ * Get rate limiter stats for monitoring
+ */
+export function getRateLimiterStats() {
+  return {
+    notion: {
+      availableTokens: notionRateLimiter.tokens,
+      maxTokens: notionRateLimiter.maxRequests,
+      queueLength: notionRateLimiter.queue.length,
+    },
+    openai: {
+      availableTokens: openAIRateLimiter.tokens,
+      maxTokens: openAIRateLimiter.maxRequests,
+      queueLength: openAIRateLimiter.queue.length,
+    },
+  };
+}

package/src/utils/secrets.js

@@ -0,0 +1,240 @@
+/**
+ * Secrets Detection & Sanitization
+ * 
+ * Prevents accidental exposure of sensitive information in documentation,
+ * logs, and telemetry.
+ */
+
+/**
+ * Patterns for detecting common secret formats
+ */
+const SECRET_PATTERNS = [
+  // API Keys
+  { name: "OpenAI API Key", pattern: /sk-[a-zA-Z0-9]{20,}/, severity: "high" },
+  { name: "Anthropic API Key", pattern: /sk-ant-[a-zA-Z0-9_-]{95,}/, severity: "high" },
+  { name: "GitHub Token", pattern: /gh[ps]_[a-zA-Z0-9]{36,}/, severity: "critical" },
+  { name: "Generic API Key", pattern: /api[_-]?key[_-]?[a-zA-Z0-9]{20,}/i, severity: "high" },
+  
+  // OAuth Tokens
+  { name: "Slack Token", pattern: /xox[baprs]-[a-zA-Z0-9-]{10,}/, severity: "critical" },
+  { name: "Bearer Token", pattern: /Bearer\s+[a-zA-Z0-9_-]{20,}/, severity: "high" },
+  
+  // Cloud Provider Keys
+  { name: "AWS Access Key", pattern: /AKIA[0-9A-Z]{16}/, severity: "critical" },
+  { name: "Google API Key", pattern: /AIzaSy[a-zA-Z0-9_-]{33}/, severity: "critical" },
+  
+  // Database Connection Strings
+  { name: "MongoDB URI", pattern: /mongodb(\+srv)?:\/\/[^\s]+/, severity: "high" },
+  { name: "PostgreSQL URI", pattern: /postgres(ql)?:\/\/[^\s]+/, severity: "high" },
+  
+  // Generic Secrets
+  { name: "Private Key", pattern: /-----BEGIN (RSA|OPENSSH|DSA|EC|PGP) PRIVATE KEY-----/, severity: "critical" },
+  { name: "Password in URL", pattern: /[a-zA-Z]{3,10}:\/\/[^:\/]+:[^@\/]+@[^\s]+/, severity: "high" },
+  
+  // Notion Specific
+  { name: "Notion Token", pattern: /secret_[a-zA-Z0-9]{30,}/, severity: "high" },
+  { name: "Notion Integration Token", pattern: /ntn_[a-zA-Z0-9]{50,}/, severity: "high" },
+];
+
+/**
+ * Scan text for potential secrets
+ * @param {string} text - Text to scan
+ * @returns {Array<{type: string, severity: string, match: string, position: number}>}
+ */
+export function detectSecrets(text) {
+  if (!text || typeof text !== "string") return [];
+  
+  const findings = [];
+  
+  for (const { name, pattern, severity } of SECRET_PATTERNS) {
+    const matches = text.matchAll(new RegExp(pattern, "g"));
+    
+    for (const match of matches) {
+      findings.push({
+        type: name,
+        severity,
+        match: match[0],
+        position: match.index,
+        // Redacted version for safe logging
+        redacted: redactSecret(match[0]),
+      });
+    }
+  }
+  
+  return findings;
+}
+
+/**
+ * Redact a secret for safe display
+ * @param {string} secret - Secret to redact
+ * @returns {string} Redacted version
+ */
+function redactSecret(secret) {
+  if (secret.length <= 8) {
+    return "***";
+  }
+  
+  const visibleChars = 4;
+  const start = secret.substring(0, visibleChars);
+  const end = secret.substring(secret.length - visibleChars);
+  
+  return `${start}***${end}`;
+}
+
+/**
+ * Sanitize text by replacing secrets with redacted versions
+ * @param {string} text - Text to sanitize
+ * @returns {string} Sanitized text
+ */
+export function sanitizeSecrets(text) {
+  if (!text || typeof text !== "string") return text;
+  
+  let sanitized = text;
+  
+  for (const { pattern } of SECRET_PATTERNS) {
+    sanitized = sanitized.replace(new RegExp(pattern, "g"), (match) => {
+      return redactSecret(match);
+    });
+  }
+  
+  return sanitized;
+}
+
+/**
+ * Check if a string looks like a secret
+ * @param {string} value - Value to check
+ * @returns {boolean}
+ */
+export function isLikelySecret(value) {
+  if (!value || typeof value !== "string") return false;
+  
+  // Check against known patterns
+  for (const { pattern } of SECRET_PATTERNS) {
+    if (pattern.test(value)) {
+      return true;
+    }
+  }
+  
+  // Heuristic: high entropy strings might be secrets
+  const entropy = calculateEntropy(value);
+  const hasHighEntropy = entropy > 4.5 && value.length >= 20;
+  
+  // Heuristic: looks like base64-encoded data
+  const isBase64Like = /^[A-Za-z0-9+/]{20,}={0,2}$/.test(value);
+  
+  return hasHighEntropy || isBase64Like;
+}
+
+/**
+ * Calculate Shannon entropy of a string
+ * @param {string} str - String to analyze
+ * @returns {number} Entropy value
+ */
+function calculateEntropy(str) {
+  const freq = {};
+  for (const char of str) {
+    freq[char] = (freq[char] || 0) + 1;
+  }
+  
+  let entropy = 0;
+  const len = str.length;
+  
+  for (const count of Object.values(freq)) {
+    const p = count / len;
+    entropy -= p * Math.log2(p);
+  }
+  
+  return entropy;
+}
+
+/**
+ * Validate environment variables don't contain unexpected secrets
+ * @param {object} env - Environment variables object
+ * @returns {Array<{key: string, issue: string}>}
+ */
+export function validateEnvironment(env = process.env) {
+  const issues = [];
+  
+  // List of keys that SHOULD contain secrets (expected)
+  const expectedSecretKeys = [
+    "NOTION_TOKEN",
+    "REPOLENS_AI_API_KEY",
+    "OPENAI_API_KEY",
+    "ANTHROPIC_API_KEY",
+    "AZURE_OPENAI_API_KEY",
+  ];
+  
+  // Check each environment variable
+  for (const [key, value] of Object.entries(env)) {
+    if (!value || typeof value !== "string") continue;
+    
+    // Skip expected secret keys
+    if (expectedSecretKeys.includes(key)) continue;
+    
+    // Check if value looks like a secret but key doesn't indicate it should be
+    if (isLikelySecret(value)) {
+      issues.push({
+        key,
+        issue: `Environment variable "${key}" contains what looks like a secret`,
+        severity: "warning",
+      });
+    }
+    
+    // Check for secrets in the value
+    const findings = detectSecrets(value);
+    if (findings.length > 0) {
+      issues.push({
+        key,
+        issue: `Environment variable "${key}" contains detected secret: ${findings[0].type}`,
+        severity: findings[0].severity,
+        redacted: findings[0].redacted,
+      });
+    }
+  }
+  
+  return issues;
+}
+
+/**
+ * Sanitize an object for logging/telemetry
+ * Recursively redacts any values that look like secrets
+ * @param {any} obj - Object to sanitize
+ * @param {number} depth - Current recursion depth
+ * @returns {any} Sanitized copy
+ */
+export function sanitizeObject(obj, depth = 0) {
+  if (depth > 10) return "[Max depth exceeded]";
+  if (obj === null || obj === undefined) return obj;
+  
+  // Handle strings
+  if (typeof obj === "string") {
+    return sanitizeSecrets(obj);
+  }
+  
+  // Handle arrays
+  if (Array.isArray(obj)) {
+    return obj.map(item => sanitizeObject(item, depth + 1));
+  }
+  
+  // Handle objects
+  if (typeof obj === "object") {
+    const sanitized = {};
+    
+    for (const [key, value] of Object.entries(obj)) {
+      // Known secret keys - always redact
+      const secretKeys = ["token", "key", "secret", "password", "auth", "apikey"];
+      const isSecretKey = secretKeys.some(sk => key.toLowerCase().includes(sk));
+      
+      if (isSecretKey && typeof value === "string") {
+        sanitized[key] = redactSecret(value);
+      } else {
+        sanitized[key] = sanitizeObject(value, depth + 1);
+      }
+    }
+    
+    return sanitized;
+  }
+  
+  // Return primitives as-is
+  return obj;
+}