@push.rocks/smartproxy 19.6.15 → 19.6.17

package/readme.plan.md

@@ -1,45 +1,281 @@
-# SmartProxy Connection Limiting Improvements Plan
+# SmartProxy Implementation Plan
 
-Command to re-read CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md`
+## Feature: Custom Certificate Provision Function
 
-## Issues Identified
+### Summary
+This plan implements the `certProvisionFunction` feature that allows users to provide their own certificate generation logic. The function can either return a custom certificate or delegate back to Let's Encrypt by returning 'http01'.
 
-1. **HttpProxy Bypass**: Connections forwarded to HttpProxy for TLS termination only check global limits, not per-IP limits
-2. **Missing Route-Level Connection Enforcement**: Routes can define `security.maxConnections` but it's never enforced
-3. **Cleanup Queue Race Condition**: New connections can be added to cleanup queue while processing
-4. **IP Tracking Memory Optimization**: IP entries remain in map even without active connections
+### Key Changes
+1. Add `certProvisionFunction` support to CertificateManager
+2. Modify `provisionAcmeCertificate()` to check custom function first
+3. Add certificate expiry parsing for custom certificates
+4. Support both initial provisioning and renewal
+5. Add fallback configuration option
 
-## Implementation Steps
+### Overview
+Implement the `certProvisionFunction` callback that's defined in the interface but currently not implemented. This will allow users to provide custom certificate generation logic while maintaining backward compatibility with the existing Let's Encrypt integration.
 
-### 1. Fix HttpProxy Per-IP Validation ✓
-- [x] Pass IP information to HttpProxy when forwarding connections
-- [x] Add per-IP validation in HttpProxy connection handler
-- [x] Ensure connection tracking is consistent between SmartProxy and HttpProxy
+### Requirements
+1. The function should be called for any new certificate provisioning or renewal
+2. Must support returning custom certificates or falling back to Let's Encrypt
+3. Should integrate seamlessly with the existing certificate lifecycle
+4. Must maintain backward compatibility
 
-### 2. Implement Route-Level Connection Limits ✓
-- [x] Add connection count tracking per route in ConnectionManager
-- [x] Update SharedSecurityManager.isAllowed() to check route-specific maxConnections
-- [x] Add route connection limit validation in route-connection-handler.ts
+### Implementation Steps
 
-### 3. Fix Cleanup Queue Race Condition ✓
-- [x] Implement proper queue snapshotting before processing
-- [x] Ensure new connections added during processing aren't missed
-- [x] Add proper synchronization for cleanup operations
+#### 1. Update Certificate Manager to Support Custom Provision Function
+**File**: `ts/proxies/smart-proxy/certificate-manager.ts`
 
-### 4. Optimize IP Tracking Memory Usage ✓
-- [x] Add periodic cleanup for IPs with no active connections
-- [x] Implement expiry for rate limit timestamps
-- [x] Add memory-efficient data structures for IP tracking
+- [ ] Add `certProvisionFunction` property to CertificateManager class
+- [ ] Pass the function from SmartProxy options during initialization
+- [ ] Modify `provisionCertificate()` method to check for custom function first
 
-### 5. Add Comprehensive Tests ✓
-- [x] Test per-IP limits with HttpProxy forwarding
-- [x] Test route-level connection limits
-- [x] Test cleanup queue edge cases
-- [x] Test memory usage with many unique IPs
+#### 2. Implement Custom Certificate Provisioning Logic
+**Location**: Modify `provisionAcmeCertificate()` method
 
-## Notes
+```typescript
+private async provisionAcmeCertificate(
+  route: IRouteConfig, 
+  domains: string[]
+): Promise<void> {
+  const primaryDomain = domains[0];
+  const routeName = route.name || primaryDomain;
+  
+  // Check for custom provision function first
+  if (this.certProvisionFunction) {
+    try {
+      logger.log('info', `Attempting custom certificate provision for ${primaryDomain}`, { domain: primaryDomain });
+      const result = await this.certProvisionFunction(primaryDomain);
+      
+      if (result === 'http01') {
+        logger.log('info', `Custom function returned 'http01', falling back to Let's Encrypt for ${primaryDomain}`);
+        // Continue with existing ACME logic below
+      } else {
+        // Use custom certificate
+        const customCert = result as plugins.tsclass.network.ICert;
+        
+        // Convert to internal certificate format
+        const certData: ICertificateData = {
+          cert: customCert.cert,
+          key: customCert.key,
+          ca: customCert.ca || '',
+          issueDate: new Date(),
+          expiryDate: this.extractExpiryDate(customCert.cert)
+        };
+        
+        // Store and apply certificate
+        await this.certStore.saveCertificate(routeName, certData);
+        await this.applyCertificate(primaryDomain, certData);
+        this.updateCertStatus(routeName, 'valid', 'custom', certData);
+        
+        logger.log('info', `Custom certificate applied for ${primaryDomain}`, { 
+          domain: primaryDomain,
+          expiryDate: certData.expiryDate
+        });
+        return;
+      }
+    } catch (error) {
+      logger.log('error', `Custom cert provision failed for ${primaryDomain}: ${error.message}`, {
+        domain: primaryDomain,
+        error: error.message
+      });
+      // Configuration option to control fallback behavior
+      if (this.smartProxy.settings.certProvisionFallbackToAcme !== false) {
+        logger.log('info', `Falling back to Let's Encrypt for ${primaryDomain}`);
+      } else {
+        throw error;
+      }
+    }
+  }
+  
+  // Existing Let's Encrypt logic continues here...
+  if (!this.smartAcme) {
+    throw new Error('SmartAcme not initialized...');
+  }
+  // ... rest of existing code
+}
+```
 
-- All connection limiting is now consistent across SmartProxy and HttpProxy
-- Route-level limits provide additional granular control
-- Memory usage is optimized for high-traffic scenarios
-- Comprehensive test coverage ensures reliability
+#### 3. Add Helper Method for Certificate Expiry Extraction
+**New method**: `extractExpiryDate()`
+
+- [ ] Parse PEM certificate to extract expiry date
+- [ ] Use existing certificate parsing utilities
+- [ ] Handle parse errors gracefully
+
+```typescript
+private extractExpiryDate(certPem: string): Date {
+  try {
+    // Use forge or similar library to parse certificate
+    const cert = forge.pki.certificateFromPem(certPem);
+    return cert.validity.notAfter;
+  } catch (error) {
+    // Default to 90 days if parsing fails
+    return new Date(Date.now() + 90 * 24 * 60 * 60 * 1000);
+  }
+}
+```
+
+#### 4. Update SmartProxy Initialization
+**File**: `ts/proxies/smart-proxy/index.ts`
+
+- [ ] Pass `certProvisionFunction` from options to CertificateManager
+- [ ] Validate function if provided
+
+#### 5. Add Type Safety and Validation
+**Tasks**:
+- [ ] Validate returned certificate has required fields (cert, key, ca)
+- [ ] Check certificate validity dates
+- [ ] Ensure certificate matches requested domain
+
+#### 6. Update Certificate Renewal Logic
+**Location**: `checkAndRenewCertificates()`
+
+- [ ] Ensure renewal checks work for both ACME and custom certificates
+- [ ] Custom certificates should go through the same `provisionAcmeCertificate()` path
+- [ ] The existing renewal logic already calls `provisionCertificate()` which will use our modified flow
+
+```typescript
+// No changes needed here - the existing renewal logic will automatically
+// use the custom provision function when calling provisionCertificate()
+private async checkAndRenewCertificates(): Promise<void> {
+  // Existing code already handles this correctly
+  for (const route of routes) {
+    if (this.shouldRenewCertificate(cert, renewThreshold)) {
+      // This will call provisionCertificate -> provisionAcmeCertificate
+      // which now includes our custom function check
+      await this.provisionCertificate(route);
+    }
+  }
+}
+```
+
+#### 7. Add Integration Tests
+**File**: `test/test.certificate-provision.ts`
+
+- [ ] Test custom certificate provision
+- [ ] Test fallback to Let's Encrypt ('http01' return)
+- [ ] Test error handling
+- [ ] Test renewal with custom function
+
+#### 8. Update Documentation
+**Files**: 
+- [ ] Update interface documentation
+- [ ] Add examples to README
+- [ ] Document ICert structure requirements
+
+### API Design
+
+```typescript
+// Example usage
+const proxy = new SmartProxy({
+  certProvisionFunction: async (domain: string) => {
+    // Option 1: Return custom certificate
+    const customCert = await myCustomCA.generateCert(domain);
+    return {
+      cert: customCert.certificate,
+      key: customCert.privateKey,
+      ca: customCert.chain
+    };
+    
+    // Option 2: Use Let's Encrypt for certain domains
+    if (domain.endsWith('.internal')) {
+      return customCert;
+    }
+    return 'http01'; // Fallback to Let's Encrypt
+  },
+  certProvisionFallbackToAcme: true, // Default: true
+  routes: [...]
+});
+```
+
+### Configuration Options to Add
+
+```typescript
+interface ISmartProxyOptions {
+  // Existing options...
+  
+  // Custom certificate provision function
+  certProvisionFunction?: (domain: string) => Promise<TSmartProxyCertProvisionObject>;
+  
+  // Whether to fallback to ACME if custom provision fails
+  certProvisionFallbackToAcme?: boolean; // Default: true
+}
+```
+
+### Error Handling Strategy
+
+1. **Custom Function Errors**:
+   - Log detailed error with domain context
+   - Option A: Fallback to Let's Encrypt (safer)
+   - Option B: Fail certificate provisioning (stricter)
+   - Make this configurable via option?
+
+2. **Invalid Certificate Returns**:
+   - Validate certificate structure
+   - Check expiry dates
+   - Verify domain match
+
+### Testing Plan
+
+1. **Unit Tests**:
+   - Mock certProvisionFunction returns
+   - Test validation logic
+   - Test error scenarios
+
+2. **Integration Tests**:
+   - Real certificate generation
+   - Renewal cycle testing
+   - Mixed custom/Let's Encrypt scenarios
+
+### Backward Compatibility
+
+- If no `certProvisionFunction` provided, behavior unchanged
+- Existing routes with 'auto' certificates continue using Let's Encrypt
+- No breaking changes to existing API
+
+### Future Enhancements
+
+1. **Per-Route Custom Functions**:
+   - Allow different provision functions per route
+   - Override global function at route level
+
+2. **Certificate Events**:
+   - Emit events for custom cert provisioning
+   - Allow monitoring/logging hooks
+
+3. **Async Certificate Updates**:
+   - Support updating certificates outside renewal cycle
+   - Hot-reload certificates without restart
+
+### Implementation Notes
+
+1. **Certificate Status Tracking**:
+   - The `updateCertStatus()` method needs to support a new type: 'custom'
+   - Current types are 'acme' and 'static'
+   - This helps distinguish custom certificates in monitoring/logs
+
+2. **Certificate Store Integration**:
+   - Custom certificates are stored the same way as ACME certificates
+   - They participate in the same renewal cycle
+   - The store handles persistence across restarts
+
+3. **Existing Methods to Reuse**:
+   - `applyCertificate()` - Already handles applying certs to routes
+   - `isCertificateValid()` - Can validate custom certificates
+   - `certStore.saveCertificate()` - Handles storage
+
+### Implementation Priority
+
+1. Core functionality (steps 1-3)
+2. Type safety and validation (step 5)
+3. Renewal support (step 6)
+4. Tests (step 7)
+5. Documentation (step 8)
+
+### Estimated Effort
+
+- Core implementation: 4-6 hours
+- Testing: 2-3 hours
+- Documentation: 1 hour
+- Total: ~8-10 hours

package/ts/core/utils/log-deduplicator.ts

@@ -269,6 +269,7 @@ export class LogDeduplicator {
   
   private flushIPRejections(aggregated: IAggregatedEvent): void {
     const byIP = new Map<string, { count: number; reasons: Set<string> }>();
+    const allReasons = new Map<string, number>();
     
     for (const [ip, event] of aggregated.events) {
       if (!byIP.has(ip)) {
@@ -278,9 +279,17 @@ export class LogDeduplicator {
       ipData.count += event.count;
       if (event.data?.reason) {
         ipData.reasons.add(event.data.reason);
+        // Track overall reason counts
+        allReasons.set(event.data.reason, (allReasons.get(event.data.reason) || 0) + event.count);
       }
     }
     
+    // Create reason summary
+    const reasonSummary = Array.from(allReasons.entries())
+      .sort((a, b) => b[1] - a[1])
+      .map(([reason, count]) => `${reason}: ${count}`)
+      .join(', ');
+    
     // Log top offenders
     const topOffenders = Array.from(byIP.entries())
       .sort((a, b) => b[1].count - a[1].count)
@@ -291,7 +300,7 @@ export class LogDeduplicator {
     const totalRejections = Array.from(byIP.values()).reduce((sum, data) => sum + data.count, 0);
     
     const duration = Date.now() - Math.min(...Array.from(aggregated.events.values()).map(e => e.firstSeen));
-    logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s`, {
+    logger.log('warn', `[SUMMARY] Rejected ${totalRejections} connections from ${byIP.size} IPs in ${Math.round(duration/1000)}s (${reasonSummary})`, {
       topOffenders,
       component: 'ip-dedup'
     });

package/ts/proxies/smart-proxy/certificate-manager.ts

@@ -12,7 +12,7 @@ export interface ICertStatus {
   status: 'valid' | 'pending' | 'expired' | 'error';
   expiryDate?: Date;
   issueDate?: Date;
-  source: 'static' | 'acme';
+  source: 'static' | 'acme' | 'custom';
   error?: string;
 }
 
@@ -22,6 +22,7 @@ export interface ICertificateData {
   ca?: string;
   expiryDate: Date;
   issueDate: Date;
+  source?: 'static' | 'acme' | 'custom';
 }
 
 export class SmartCertManager {
@@ -50,6 +51,12 @@ export class SmartCertManager {
   // ACME state manager reference
   private acmeStateManager: AcmeStateManager | null = null;
   
+  // Custom certificate provision function
+  private certProvisionFunction?: (domain: string) => Promise<plugins.tsclass.network.ICert | 'http01'>;
+  
+  // Whether to fallback to ACME if custom provision fails
+  private certProvisionFallbackToAcme: boolean = true;
+  
   constructor(
     private routes: IRouteConfig[],
     private certDir: string = './certs',
@@ -89,6 +96,20 @@ export class SmartCertManager {
     this.globalAcmeDefaults = defaults;
   }
   
+  /**
+   * Set custom certificate provision function
+   */
+  public setCertProvisionFunction(fn: (domain: string) => Promise<plugins.tsclass.network.ICert | 'http01'>): void {
+    this.certProvisionFunction = fn;
+  }
+  
+  /**
+   * Set whether to fallback to ACME if custom provision fails
+   */
+  public setCertProvisionFallbackToAcme(fallback: boolean): void {
+    this.certProvisionFallbackToAcme = fallback;
+  }
+  
   /**
    * Set callback for updating routes (used for challenge routes)
    */
@@ -212,15 +233,6 @@ export class SmartCertManager {
     route: IRouteConfig, 
     domains: string[]
   ): Promise<void> {
-    if (!this.smartAcme) {
-      throw new Error(
-        'SmartAcme not initialized. This usually means no ACME email was provided. ' +
-        'Please ensure you have configured ACME with an email address either:\n' +
-        '1. In the top-level "acme" configuration\n' +
-        '2. In the route\'s "tls.acme" configuration'
-      );
-    }
-    
     const primaryDomain = domains[0];
     const routeName = route.name || primaryDomain;
     
@@ -229,10 +241,68 @@ export class SmartCertManager {
     if (existingCert && this.isCertificateValid(existingCert)) {
       logger.log('info', `Using existing valid certificate for ${primaryDomain}`, { domain: primaryDomain, component: 'certificate-manager' });
       await this.applyCertificate(primaryDomain, existingCert);
-      this.updateCertStatus(routeName, 'valid', 'acme', existingCert);
+      this.updateCertStatus(routeName, 'valid', existingCert.source || 'acme', existingCert);
       return;
     }
     
+    // Check for custom provision function first
+    if (this.certProvisionFunction) {
+      try {
+        logger.log('info', `Attempting custom certificate provision for ${primaryDomain}`, { domain: primaryDomain, component: 'certificate-manager' });
+        const result = await this.certProvisionFunction(primaryDomain);
+        
+        if (result === 'http01') {
+          logger.log('info', `Custom function returned 'http01', falling back to Let's Encrypt for ${primaryDomain}`, { domain: primaryDomain, component: 'certificate-manager' });
+          // Continue with existing ACME logic below
+        } else {
+          // Use custom certificate
+          const customCert = result as plugins.tsclass.network.ICert;
+          
+          // Convert to internal certificate format
+          const certData: ICertificateData = {
+            cert: customCert.publicKey,
+            key: customCert.privateKey,
+            ca: '',
+            issueDate: new Date(),
+            expiryDate: this.extractExpiryDate(customCert.publicKey),
+            source: 'custom'
+          };
+          
+          // Store and apply certificate
+          await this.certStore.saveCertificate(routeName, certData);
+          await this.applyCertificate(primaryDomain, certData);
+          this.updateCertStatus(routeName, 'valid', 'custom', certData);
+          
+          logger.log('info', `Custom certificate applied for ${primaryDomain}`, { 
+            domain: primaryDomain,
+            expiryDate: certData.expiryDate,
+            component: 'certificate-manager'
+          });
+          return;
+        }
+      } catch (error) {
+        logger.log('error', `Custom cert provision failed for ${primaryDomain}: ${error.message}`, {
+          domain: primaryDomain,
+          error: error.message,
+          component: 'certificate-manager'
+        });
+        // Check if we should fallback to ACME
+        if (!this.certProvisionFallbackToAcme) {
+          throw error;
+        }
+        logger.log('info', `Falling back to Let's Encrypt for ${primaryDomain}`, { domain: primaryDomain, component: 'certificate-manager' });
+      }
+    }
+    
+    if (!this.smartAcme) {
+      throw new Error(
+        'SmartAcme not initialized. This usually means no ACME email was provided. ' +
+        'Please ensure you have configured ACME with an email address either:\n' +
+        '1. In the top-level "acme" configuration\n' +
+        '2. In the route\'s "tls.acme" configuration'
+      );
+    }
+    
     // Apply renewal threshold from global defaults or route config
     const renewThreshold = route.action.tls?.acme?.renewBeforeDays || 
                          this.globalAcmeDefaults?.renewThresholdDays || 
@@ -280,7 +350,8 @@ export class SmartCertManager {
         key: cert.privateKey, 
         ca: cert.publicKey, // Use same as cert for now
         expiryDate: new Date(cert.validUntil),
-        issueDate: new Date(cert.created)
+        issueDate: new Date(cert.created),
+        source: 'acme'
       };
       
       await this.certStore.saveCertificate(routeName, certData);
@@ -328,7 +399,8 @@ export class SmartCertManager {
         cert,
         key,
         expiryDate: certInfo.validTo,
-        issueDate: certInfo.validFrom
+        issueDate: certInfo.validFrom,
+        source: 'static'
       };
       
       // Save to store for consistency
@@ -399,6 +471,19 @@ export class SmartCertManager {
     return cert.expiryDate > expiryThreshold;
   }
   
+  /**
+   * Extract expiry date from a PEM certificate
+   */
+  private extractExpiryDate(_certPem: string): Date {
+    // For now, we'll default to 90 days for custom certificates
+    // In production, you might want to use a proper X.509 parser
+    // or require the custom cert provider to include expiry info
+    logger.log('info', 'Using default 90-day expiry for custom certificate', {
+      component: 'certificate-manager'
+    });
+    return new Date(Date.now() + 90 * 24 * 60 * 60 * 1000);
+  }
+  
   
   /**
    * Add challenge route to SmartProxy

package/ts/proxies/smart-proxy/models/interfaces.ts

@@ -135,6 +135,12 @@ export interface ISmartProxyOptions {
    * or a static certificate object for immediate provisioning.
    */
   certProvisionFunction?: (domain: string) => Promise<TSmartProxyCertProvisionObject>;
+  
+  /**
+   * Whether to fallback to ACME if custom certificate provision fails.
+   * Default: true
+   */
+  certProvisionFallbackToAcme?: boolean;
 }
 
 /**

package/ts/proxies/smart-proxy/smart-proxy.ts

@@ -243,6 +243,16 @@ export class SmartProxy extends plugins.EventEmitter {
       certManager.setGlobalAcmeDefaults(this.settings.acme);
     }
     
+    // Pass down the custom certificate provision function if available
+    if (this.settings.certProvisionFunction) {
+      certManager.setCertProvisionFunction(this.settings.certProvisionFunction);
+    }
+    
+    // Pass down the fallback to ACME setting
+    if (this.settings.certProvisionFallbackToAcme !== undefined) {
+      certManager.setCertProvisionFallbackToAcme(this.settings.certProvisionFallbackToAcme);
+    }
+    
     await certManager.initialize();
     return certManager;
   }