llmjs2 1.1.0 → 1.3.0

package/docs/GUARDRAILS_GUIDE.md

@@ -0,0 +1,734 @@
+# llmjs2 Guardrails Usage Guide
+
+Guardrails provide a powerful mechanism to add custom logic before and after LLM calls, enabling content filtering, logging, transformation, and safety checks.
+
+## Overview
+
+Guardrails are simple function configurations that run before or after LLM calls, allowing you to:
+
+- **Content Filtering**: Block or modify inappropriate content
+- **Logging & Monitoring**: Track requests and responses
+- **Transformation**: Modify messages before/after processing
+- **Safety Checks**: Validate inputs and outputs
+- **Rate Limiting**: Implement custom throttling logic
+- **Caching**: Add intelligent response caching
+
+## Basic Usage
+
+### Router with Guardrails
+
+```javascript
+import { router } from 'llmjs2';
+
+const modelList = [
+  {
+    "model_name": "gpt-3.5-turbo",
+    "llm_params": {
+      "model": "openai/gpt-3.5-turbo",
+      "api_key": process.env.OPENAI_API_KEY
+    }
+  }
+];
+
+// Create router with guardrails
+const route = router({
+  model_list: modelList,
+  routing: 'random',
+  guardrails: [
+    {
+      name: "content_filter",
+      mode: "pre_call",
+      code: (processId, input) => {
+        // Filter inappropriate content before LLM call
+        const { model, messages } = input;
+        const filteredMessages = messages.map(msg => ({
+          ...msg,
+          content: msg.content.replace(/badword/gi, '****')
+        }));
+        return { model, messages: filteredMessages };
+      }
+    },
+    {
+      name: "response_logger",
+      mode: "post_call",
+      code: (processId, result) => {
+        // Log responses after LLM call
+        console.log(`[${processId}] Response:`, result);
+        return result;
+      }
+    }
+  ]
+});
+
+// Use router
+const response = await route.completion({
+  messages: [{"role": "user", "content": "Hello!"}]
+});
+```
+
+## Guardrail Configuration
+
+Guardrails are defined as objects with three properties:
+
+```javascript
+{
+  name: "string",     // Unique identifier for the guardrail
+  mode: "pre_call" | "post_call",  // When to execute
+  code: function     // The guardrail function
+}
+```
+
+### Pre-call Guardrails
+
+Executed before sending to LLM provider:
+
+```javascript
+{
+  name: "content_filter",
+  mode: "pre_call",
+  code: (processId, input) => {
+    /**
+     * @param {string} processId - Unique identifier for this request
+     * @param {object} input - Input object containing model and messages
+     * @param {string} input.model - Final model selected by router
+     * @param {Array} input.messages - Final message array to send to LLM
+     * @returns {object} Modified input object or throw error to block
+     */
+    const { model, messages } = input;
+
+    // Your pre-processing logic here
+    return { model, messages }; // Return modified input
+  }
+}
+```
+
+### Post-call Guardrails
+
+Executed after receiving response from LLM provider:
+
+```javascript
+{
+  name: "response_logger",
+  mode: "post_call",
+  code: (processId, result) => {
+    /**
+     * @param {string} processId - Unique identifier for this request
+     * @param {string|object} result - Normalized LLM response (llmjs2 format)
+     * @returns {string|object} Modified result or throw error to block
+     */
+
+    // Your post-processing logic here
+    return result; // Return modified result
+  }
+}
+```
+
+## Built-in Guardrail Examples
+
+### Content Filter Guardrail
+
+```javascript
+const contentFilterGuardrail = {
+  name: "content_filter",
+  mode: "pre_call",
+  code: (processId, input) => {
+    const { model, messages } = input;
+
+    // Log which model was selected
+    console.log(`[${processId}] Selected model: ${model}`);
+
+    for (const message of messages) {
+      if (message.role === 'user' && message.content) {
+        // Check for inappropriate content
+        if (containsProfanity(message.content)) {
+          throw new Error('Content violates usage policy');
+        }
+
+        // Apply content transformations
+        message.content = sanitizeContent(message.content);
+      }
+    }
+
+    return { model, messages };
+  }
+};
+
+const responseFilterGuardrail = {
+  name: "response_filter",
+  mode: "post_call",
+  code: (processId, result) => {
+    if (typeof result === 'string') {
+      // Filter response content
+      return filterResponse(result);
+    }
+    return result;
+  }
+};
+
+// Helper functions
+function containsProfanity(text) {
+  const badWords = ['badword1', 'badword2'];
+  return badWords.some(word => text.toLowerCase().includes(word));
+}
+
+function sanitizeContent(text) {
+  // Implement content sanitization logic
+  return text;
+}
+
+function filterResponse(text) {
+  // Implement response filtering logic
+  return text;
+}
+```
+
+### Logging Guardrail
+
+```javascript
+const requestLoggerGuardrail = {
+  name: "request_logger",
+  mode: "pre_call",
+  code: (processId, input) => {
+    const { model, messages } = input;
+
+    console.log(`[${new Date().toISOString()}] [${processId}] REQUEST:`, {
+      model,
+      messageCount: messages.length,
+      totalChars: messages.reduce((sum, m) => sum + (m.content?.length || 0), 0),
+      userMessages: messages.filter(m => m.role === 'user').length
+    });
+
+    return input;
+  }
+};
+
+const responseLoggerGuardrail = {
+  name: "response_logger",
+  mode: "post_call",
+  code: (processId, result) => {
+    const responseInfo = {
+      processId,
+      timestamp: new Date().toISOString(),
+      responseType: typeof result,
+      responseLength: typeof result === 'string' ? result.length : 'object'
+    };
+
+    console.log(`[${responseInfo.timestamp}] [${processId}] RESPONSE:`, responseInfo);
+
+    // Could also log to external service
+    logToExternalService(responseInfo);
+
+    return result;
+  }
+};
+
+// Helper function
+function logToExternalService(data) {
+  // Implement external logging (e.g., to database, monitoring service)
+}
+```
+
+### Rate Limiting Guardrail
+
+```javascript
+// Store for rate limiting (in production, use Redis or similar)
+const rateLimitStore = new Map();
+
+const rateLimitGuardrail = {
+  name: "rate_limiter",
+  mode: "pre_call",
+  code: (processId, input) => {
+    const { model, messages } = input;
+    const now = Date.now();
+    const userId = getUserFromMessages(messages);
+
+    if (!userId) {
+      return input; // No user context, allow
+    }
+
+    // Clean old requests
+    cleanOldRequests(userId, now);
+
+    // Check rate limit
+    const userRequests = rateLimitStore.get(userId) || [];
+    if (userRequests.length >= 60) { // 60 requests per minute
+      throw new Error(`Rate limit exceeded. Maximum 60 requests per minute.`);
+    }
+
+    // Record this request
+    userRequests.push(now);
+    rateLimitStore.set(userId, userRequests);
+
+    return input;
+  }
+};
+
+// Helper functions
+function getUserFromMessages(messages) {
+  // Extract user identifier from messages (e.g., from metadata)
+  // This is application-specific
+  return null; // Implement based on your needs
+}
+
+function cleanOldRequests(userId, now) {
+  const userRequests = rateLimitStore.get(userId) || [];
+  const oneMinuteAgo = now - 60000;
+
+  const recentRequests = userRequests.filter(time => time > oneMinuteAgo);
+  if (recentRequests.length === 0) {
+    rateLimitStore.delete(userId);
+  } else {
+    rateLimitStore.set(userId, recentRequests);
+  }
+}
+```
+
+### Response Caching Guardrail
+
+```javascript
+// Cache store (in production, use Redis or similar)
+const cacheStore = new Map();
+const CACHE_TTL = 300000; // 5 minutes in milliseconds
+
+const cachingGuardrail = {
+  name: "response_cache",
+  mode: "pre_call",
+  code: (processId, input) => {
+    const { model, messages } = input;
+
+    // Create cache key from model and messages
+    const cacheKey = createCacheKey(model, messages);
+
+    // Check cache
+    const cached = cacheStore.get(cacheKey);
+    if (cached && (Date.now() - cached.timestamp) < CACHE_TTL) {
+      console.log(`[${processId}] Cache hit for key: ${cacheKey}`);
+      // Return cached result instead of making API call
+      // Note: This would need special handling in the router
+    }
+
+    return input;
+  }
+};
+
+const cacheWriterGuardrail = {
+  name: "cache_writer",
+  mode: "post_call",
+  code: (processId, result) => {
+    // Cache the result (this would need the original input to generate key)
+    // For now, just return the result
+    return result;
+  }
+};
+
+// Helper functions
+function createCacheKey(model, messages) {
+  // Create deterministic key from model and messages
+  const content = `${model}|${messages.map(m => `${m.role}:${m.content}`).join('|')}`;
+  return hashString(content);
+}
+
+function hashString(str) {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash = ((hash << 5) - hash) + char;
+    hash = hash & hash; // Convert to 32-bit integer
+  }
+  return hash.toString();
+}
+
+function cleanCache() {
+  const now = Date.now();
+  for (const [key, value] of cacheStore.entries()) {
+    if ((now - value.timestamp) > CACHE_TTL) {
+      cacheStore.delete(key);
+    }
+  }
+}
+```
+
+## Advanced Guardrail Patterns
+
+### Chained Transformations
+
+```javascript
+const sanitizationGuardrail = {
+  name: "content_sanitizer",
+  mode: "pre_call",
+  code: (processId, input) => {
+    const { model, messages } = input;
+
+    const sanitizedMessages = messages.map(message => ({
+      ...message,
+      content: sanitize(message.content)
+    }));
+
+    return { model, messages: sanitizedMessages };
+  }
+};
+
+const responseSanitizationGuardrail = {
+  name: "response_sanitizer",
+  mode: "post_call",
+  code: (processId, result) => {
+    if (typeof result === 'string') {
+      return finalSanitize(result);
+    }
+    return result;
+  }
+};
+
+// Helper functions
+function sanitize(text) {
+  // Basic sanitization
+  return text.replace(/<script>/gi, '').replace(/javascript:/gi, '');
+}
+
+function finalSanitize(text) {
+  // Additional sanitization for responses
+  return text;
+}
+```
+
+### Conditional Processing
+
+```javascript
+const conditionalGuardrail = {
+  name: "conditional_processor",
+  mode: "pre_call",
+  code: (processId, input) => {
+    const { model, messages } = input;
+
+    if (shouldProcess(messages)) {
+      const processedMessages = processMessages(messages);
+      return { model, messages: processedMessages };
+    }
+    return input;
+  }
+};
+
+const conditionalResponseGuardrail = {
+  name: "conditional_response_processor",
+  mode: "post_call",
+  code: (processId, result) => {
+    if (shouldProcessResult(result)) {
+      return processResult(result);
+    }
+    return result;
+  }
+};
+
+// Helper functions
+function shouldProcess(messages) {
+  // Check conditions for processing
+  return messages.some(m => m.content?.length > 100);
+}
+
+function shouldProcessResult(result) {
+  // Check conditions for result processing
+  return typeof result === 'string' && result.length > 50;
+}
+
+function processMessages(messages) {
+  // Apply processing logic
+  return messages;
+}
+
+function processResult(result) {
+  // Apply result processing
+  return result;
+}
+```
+
+### Async Guardrails
+
+```javascript
+const asyncValidationGuardrail = {
+  name: "async_validator",
+  mode: "pre_call",
+  code: async (processId, input) => {
+    const { model, messages } = input;
+
+    // Async validation
+    await validateContent(messages);
+
+    // Async transformation
+    const transformedMessages = await transformMessages(messages);
+
+    return { model, messages: transformedMessages };
+  }
+};
+
+const asyncProcessingGuardrail = {
+  name: "async_processor",
+  mode: "post_call",
+  code: async (processId, result) => {
+    // Async result processing
+    const processed = await processResultAsync(result);
+
+    // Async logging
+    await logResult(processId, processed);
+
+    return processed;
+  }
+};
+
+// Helper functions
+async function validateContent(messages) {
+  // Simulate async validation
+  await new Promise(resolve => setTimeout(resolve, 10));
+  // Throw error if validation fails
+}
+
+async function transformMessages(messages) {
+  // Simulate async transformation
+  await new Promise(resolve => setTimeout(resolve, 5));
+  return messages;
+}
+
+async function processResultAsync(result) {
+  // Simulate async processing
+  await new Promise(resolve => setTimeout(resolve, 5));
+  return result;
+}
+
+async function logResult(processId, result) {
+  // Simulate async logging
+  await new Promise(resolve => setTimeout(resolve, 1));
+}
+```
+
+## Configuration Examples
+
+### Production Setup with Multiple Guardrails
+
+```javascript
+import { router } from 'llmjs2';
+
+const route = router({
+  model_list: [
+    {
+      "model_name": "gpt-4",
+      "llm_params": {
+        "model": "openai/gpt-4",
+        "api_key": process.env.OPENAI_API_KEY
+      }
+    }
+  ],
+  routing: 'random',
+  guardrails: [
+    new ContentFilterGuardrail(),
+    new LoggingGuardrail(),
+    new RateLimitGuardrail(100), // 100 requests per minute
+    new CachingGuardrail(600)    // 10 minute cache TTL
+  ]
+});
+
+// Use with comprehensive protection
+const response = await route.completion({
+  messages: [{"role": "user", "content": "Hello!"}]
+});
+```
+
+### Development Setup with Minimal Guardrails
+
+```javascript
+const devRoute = router({
+  model_list: [
+    {
+      "model_name": "test-model",
+      "llm_params": {
+        "model": "ollama/llama2",
+        "api_key": process.env.OLLAMA_API_KEY
+      }
+    }
+  ],
+  routing: 'sequential',
+  guardrails: [
+    {
+      name: "dev_logger",
+      mode: "post_call",
+      code: (processId, result) => {
+        console.log(`[DEV] ${processId}:`, result);
+        return result;
+      }
+    }
+  ]
+});
+```
+
+## Error Handling
+
+Guardrails can throw errors to block requests:
+
+```javascript
+const strictContentGuardrail = {
+  name: "strict_content_filter",
+  mode: "pre_call",
+  code: (processId, input) => {
+    const { model, messages } = input;
+
+    for (const message of messages) {
+      if (message.content?.includes('blocked_word')) {
+        throw new Error('Content blocked by guardrail policy');
+      }
+    }
+    return input;
+  }
+};
+
+  post_call(processId, result) {
+    if (typeof result === 'string' && result.includes('inappropriate')) {
+      throw new Error('Response blocked by guardrail policy');
+    }
+    return result;
+  }
+}
+
+// Usage
+try {
+  const response = await route.completion({
+    messages: [{"role": "user", "content": "This contains blocked_word"}]
+  });
+} catch (error) {
+  if (error.message.includes('blocked by guardrail')) {
+    console.log('Request blocked by guardrail');
+  }
+}
+```
+
+## Best Practices
+
+### Guardrail Ordering
+
+```javascript
+const route = router({
+  model_list: modelList,
+  routing: 'random',
+  guardrails: [
+    // 1. Rate limiting first (fast rejection)
+    new RateLimitGuardrail(60),
+
+    // 2. Content validation
+    new ContentFilterGuardrail(),
+
+    // 3. Logging (should be reliable)
+    new LoggingGuardrail(),
+
+    // 4. Caching (can be complex)
+    new CachingGuardrail()
+  ]
+});
+```
+
+### Performance Considerations
+
+- **Fast First**: Put quick checks (rate limiting) before expensive operations
+- **Fail Fast**: Throw errors early for clear violations
+- **Async Wisely**: Use async operations only when necessary
+- **Memory Management**: Clean up caches and state periodically
+
+### Security Guidelines
+
+- **Input Validation**: Always validate and sanitize inputs
+- **Output Filtering**: Filter responses for sensitive information
+- **Rate Limiting**: Implement appropriate limits for your use case
+- **Logging**: Log security events without exposing sensitive data
+- **Error Messages**: Don't reveal system internals in error messages
+
+## Integration with Server Mode
+
+Guardrails work seamlessly with server mode:
+
+```javascript
+import { router, app } from 'llmjs2';
+
+const route = router({
+  model_list: modelList,
+  routing: 'random',
+  guardrails: [
+    {
+      name: "content_filter",
+      mode: "pre_call",
+      code: (processId, input) => {
+        // Content filtering logic
+        return input;
+      }
+    },
+    {
+      name: "request_logger",
+      mode: "pre_call",
+      code: (processId, input) => {
+        console.log(`[${processId}] Processing request`);
+        return input;
+      }
+    },
+    {
+      name: "rate_limiter",
+      mode: "pre_call",
+      code: (processId, input) => {
+        // Rate limiting logic
+        return input;
+      }
+    },
+    {
+      name: "response_logger",
+      mode: "post_call",
+      code: (processId, result) => {
+        console.log(`[${processId}] Response received`);
+        return result;
+      }
+    }
+  ]
+});
+
+app.use(route);
+app.listen(3000);
+```
+
+All API requests will automatically go through the guardrails before routing to models.
+
+## Creating Reusable Guardrails
+
+### Custom Base Class
+
+```javascript
+// Guardrail factory function for reusable guardrails
+function createLoggingGuardrail(options = {}) {
+  const enabled = options.enabled !== false;
+
+  return {
+    name: options.name || "logger",
+    mode: "post_call",
+    code: (processId, result) => {
+      if (!enabled) return result;
+
+      console.log(`[${new Date().toISOString()}] [${processId}] ${options.prefix || 'LOG'}:`, result);
+      return result;
+    }
+  };
+}
+
+// Usage
+const debugLogger = createLoggingGuardrail({ name: "debug_logger", prefix: "DEBUG" });
+const errorLogger = createLoggingGuardrail({ name: "error_logger", prefix: "ERROR" });
+
+// Custom guardrail function
+const customGuardrail = {
+  name: "custom_processor",
+  mode: "pre_call",
+  code: (processId, input) => {
+    const { model, messages } = input;
+    // Custom logic
+    return { model, messages };
+  }
+
+  processPostCall(processId, result) {
+    // Custom logic
+    return result;
+  }
+}
+```
+
+Guardrails provide a simple and powerful way to add custom logic around LLM calls using function-based configurations, enabling everything from basic logging to advanced content filtering and security measures.