@synet/encoder 1.0.0

package/MANUAL.md

@@ -0,0 +1,735 @@
+# @synet/encoder Manual
+
+**Advanced Use Cases and Deep Integration Patterns**
+
+This manual covers advanced usage patterns, architecture deep-dives, and production deployment scenarios for the @synet/encoder unit.
+
+## Table of Contents
+
+1. [Unit Architecture Deep Dive](#unit-architecture-deep-dive)
+2. [Advanced Integration Patterns](#advanced-integration-patterns)
+3. [Production Deployment](#production-deployment)
+4. [Performance Optimization](#performance-optimization)
+5. [Error Handling Strategies](#error-handling-strategies)
+6. [Custom Format Extensions](#custom-format-extensions)
+7. [Unit Composition Patterns](#unit-composition-patterns)
+8. [Monitoring and Observability](#monitoring-and-observability)
+
+## Unit Architecture Deep Dive
+
+### Consciousness-Based Design
+
+The Encoder unit embodies the **conscious software architecture** philosophy:
+
+```typescript
+// The unit is aware of its identity and capabilities
+const encoder = Encoder.create();
+console.log(encoder.whoami()); // EncoderUnit[encoder@1.0.0]
+console.log(encoder.capabilities()); // Lists all available operations
+
+// Units are immutable value objects with identity
+const encoder2 = Encoder.create({ defaultFormat: 'hex' });
+// encoder !== encoder2 (different identity)
+// Both retain their essential nature while having different configurations
+```
+
+### Doctrine Compliance
+
+The encoder follows all 22 Unit Architecture doctrines:
+
+```typescript
+// Doctrine #1: ZERO DEPENDENCY - Only native APIs
+// Doctrine #3: PROPS CONTAIN EVERYTHING - No private field duplication
+// Doctrine #17: VALUE OBJECT FOUNDATION - Immutable with identity
+// Doctrine #22: STATELESS OPERATIONS - Deterministic given props
+
+// Example of stateless operations
+const encoder = Encoder.create();
+const result1 = encoder.encode('Hello', 'base64');
+const result2 = encoder.encode('Hello', 'base64');
+// result1 === result2 (same input = same output, no side effects)
+```
+
+## Advanced Integration Patterns
+
+### Multi-Unit Collaborative Systems
+
+```typescript
+// Crypto + Encoder collaboration
+import { Signer } from '@synet/signer';
+import { Encoder } from '@synet/encoder';
+
+const signer = Signer.create();
+const encoder = Encoder.create();
+
+// Units learn from each other
+encoder.learn([signer.teach()]);
+signer.learn([encoder.teach()]);
+
+// Complex operations through capability composition
+async function signAndEncode(data: string): Promise<string> {
+  const signature = await signer.execute('crypto.sign', data);
+  const encoded = encoder.execute('encoder.encode', signature, 'base64url');
+  return encoded;
+}
+```
+
+### Pipeline Architecture
+
+```typescript
+// Create encoding pipeline with multiple units
+class EncodingPipeline {
+  private units: Unit[] = [];
+  
+  constructor() {
+    this.units = [
+      Hasher.create(),
+      Encoder.create(),
+      Compressor.create()
+    ];
+    
+    // Enable cross-unit learning
+    this.enableCrossLearning();
+  }
+  
+  async process(data: string): Promise<ProcessedData> {
+    // Hash -> Encode -> Compress pipeline
+    let result = data;
+    
+    for (const unit of this.units) {
+      result = await unit.execute('transform', result);
+    }
+    
+    return {
+      original: data,
+      processed: result,
+      pipeline: this.units.map(u => u.whoami())
+    };
+  }
+  
+  private enableCrossLearning(): void {
+    const contracts = this.units.map(unit => unit.teach());
+    this.units.forEach(unit => unit.learn(contracts));
+  }
+}
+```
+
+### Event-Driven Architecture
+
+```typescript
+// Encoder in event-driven systems
+import { EventEmitter } from 'events';
+
+class EncodingService extends EventEmitter {
+  private encoder = Encoder.create({ 
+    defaultFormat: 'base64url',
+    validateOutput: true 
+  });
+  
+  async processDocument(doc: Document): Promise<void> {
+    this.emit('processing:start', { docId: doc.id });
+    
+    try {
+      // Multi-format encoding with validation
+      const results = await Promise.all([
+        this.encoder.encode(doc.content, 'base64'),
+        this.encoder.encode(doc.content, 'hex'),
+        this.encoder.chain(doc.content, ['base64', 'uri'])
+      ]);
+      
+      this.emit('processing:success', {
+        docId: doc.id,
+        formats: results.map(r => r.value.format),
+        sizes: results.map(r => r.value.encodedSize)
+      });
+      
+    } catch (error) {
+      this.emit('processing:error', { docId: doc.id, error });
+    }
+  }
+}
+```
+
+## Production Deployment
+
+### Serverless Optimization
+
+```typescript
+// Optimized for AWS Lambda/Vercel Edge Functions
+import { encode, decode, detectFormat } from '@synet/encoder/functions';
+
+// Pure functions for minimal cold start
+export const handler = async (event: APIGatewayEvent) => {
+  const { data, format } = JSON.parse(event.body);
+  
+  // No unit instantiation overhead
+  const encoded = encode(data, format as EncodingFormat);
+  
+  return {
+    statusCode: 200,
+    body: JSON.stringify({ encoded })
+  };
+};
+
+// Or use unit for complex validation
+export const validatedHandler = async (event: APIGatewayEvent) => {
+  const encoder = Encoder.create({
+    maxInputSize: 1024 * 1024, // 1MB limit for serverless
+    strictMode: true,
+    validateOutput: true
+  });
+  
+  const result = encoder.encode(event.body, 'base64url');
+  
+  return result.isSuccess 
+    ? { statusCode: 200, body: JSON.stringify(result.value) }
+    : { statusCode: 400, body: JSON.stringify({ error: result.error }) };
+};
+```
+
+### Microservice Architecture
+
+```typescript
+// Encoder microservice with health checks
+import express from 'express';
+import { Encoder } from '@synet/encoder';
+
+const app = express();
+const encoder = Encoder.create();
+
+// Health check endpoint
+app.get('/health', (req, res) => {
+  res.json({
+    service: encoder.whoami(),
+    capabilities: encoder.capabilities(),
+    status: 'healthy',
+    timestamp: new Date().toISOString()
+  });
+});
+
+// Batch encoding endpoint
+app.post('/encode/batch', async (req, res) => {
+  const { items } = req.body;
+  
+  const results = await Promise.allSettled(
+    items.map(({ data, format }) => encoder.encode(data, format))
+  );
+  
+  const successful = results.filter(r => r.status === 'fulfilled');
+  const failed = results.filter(r => r.status === 'rejected');
+  
+  res.json({
+    processed: results.length,
+    successful: successful.length,
+    failed: failed.length,
+    results: results.map(r => 
+      r.status === 'fulfilled' ? r.value : { error: r.reason }
+    )
+  });
+});
+```
+
+### Container Deployment
+
+```dockerfile
+# Dockerfile for encoder microservice
+FROM node:18-alpine
+
+WORKDIR /app
+COPY package*.json ./
+RUN npm ci --only=production
+
+COPY src/ ./src/
+COPY dist/ ./dist/
+
+# Health check using encoder unit
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+  CMD node -e "
+    const { Encoder } = require('./dist');
+    const encoder = Encoder.create();
+    console.log(encoder.whoami());
+    process.exit(0);
+  "
+
+EXPOSE 3000
+CMD ["npm", "start"]
+```
+
+## Performance Optimization
+
+### High-Throughput Scenarios
+
+```typescript
+// Optimized for high-throughput encoding
+class HighThroughputEncoder {
+  private static instance: Encoder;
+  
+  // Singleton pattern for shared instance
+  static getInstance(): Encoder {
+    if (!this.instance) {
+      this.instance = Encoder.create({
+        defaultFormat: 'base64url',
+        autoDetect: false, // Disable for performance
+        validateOutput: false, // Skip validation for speed
+        maxInputSize: 64 * 1024 // 64KB limit
+      });
+    }
+    return this.instance;
+  }
+  
+  // Batch processing with worker pools
+  static async processBatch(items: string[]): Promise<string[]> {
+    const encoder = this.getInstance();
+    const chunkSize = 1000;
+    const results: string[] = [];
+    
+    for (let i = 0; i < items.length; i += chunkSize) {
+      const chunk = items.slice(i, i + chunkSize);
+      const chunkResults = await Promise.all(
+        chunk.map(item => {
+          const result = encoder.encode(item, 'base64url');
+          return result.isSuccess ? result.value.encoded : '';
+        })
+      );
+      results.push(...chunkResults);
+    }
+    
+    return results;
+  }
+}
+```
+
+### Memory Optimization
+
+```typescript
+// Memory-efficient streaming encoder
+import { Readable, Transform } from 'stream';
+
+class StreamingEncoder extends Transform {
+  private encoder = Encoder.create();
+  
+  constructor(private format: EncodingFormat) {
+    super({ objectMode: true });
+  }
+  
+  _transform(chunk: Buffer, encoding: string, callback: Function) {
+    try {
+      const result = this.encoder.encode(chunk.toString(), this.format);
+      if (result.isSuccess) {
+        this.push(result.value.encoded);
+      }
+      callback();
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+// Usage for large file processing
+const fileStream = fs.createReadStream('large-file.txt');
+const encoder = new StreamingEncoder('base64url');
+const output = fs.createWriteStream('encoded-file.txt');
+
+fileStream.pipe(encoder).pipe(output);
+```
+
+## Error Handling Strategies
+
+### Comprehensive Error Recovery
+
+```typescript
+// Production-grade error handling
+class RobustEncodingService {
+  private encoder = Encoder.create({ strictMode: true });
+  private fallbackEncoder = Encoder.create({ strictMode: false });
+  
+  async encodeWithFallback(data: string, format: EncodingFormat): Promise<string> {
+    // Primary attempt with strict validation
+    const primaryResult = this.encoder.encode(data, format);
+    
+    if (primaryResult.isSuccess) {
+      return primaryResult.value.encoded;
+    }
+    
+    // Log primary failure
+    console.warn('Primary encoding failed:', {
+      error: primaryResult.error,
+      cause: primaryResult.errorCause,
+      data: data.slice(0, 100) + '...'
+    });
+    
+    // Fallback attempt with relaxed validation
+    const fallbackResult = this.fallbackEncoder.encode(data, format);
+    
+    if (fallbackResult.isSuccess) {
+      console.info('Fallback encoding succeeded');
+      return fallbackResult.value.encoded;
+    }
+    
+    // Both failed - comprehensive error
+    throw new EncodingError('All encoding attempts failed', {
+      primaryError: primaryResult.error,
+      fallbackError: fallbackResult.error,
+      data: data.slice(0, 50),
+      format
+    });
+  }
+}
+
+class EncodingError extends Error {
+  constructor(message: string, public context: Record<string, unknown>) {
+    super(message);
+    this.name = 'EncodingError';
+  }
+}
+```
+
+### Circuit Breaker Pattern
+
+```typescript
+// Circuit breaker for encoding operations
+class EncodingCircuitBreaker {
+  private failures = 0;
+  private lastFailure = 0;
+  private state: 'CLOSED' | 'OPEN' | 'HALF_OPEN' = 'CLOSED';
+  
+  constructor(
+    private encoder: Encoder,
+    private threshold = 5,
+    private timeout = 60000
+  ) {}
+  
+  async encode(data: string, format: EncodingFormat): Promise<string> {
+    if (this.state === 'OPEN') {
+      if (Date.now() - this.lastFailure > this.timeout) {
+        this.state = 'HALF_OPEN';
+      } else {
+        throw new Error('Circuit breaker is OPEN');
+      }
+    }
+    
+    try {
+      const result = this.encoder.encode(data, format);
+      
+      if (result.isFailure) {
+        this.recordFailure();
+        throw new Error(result.error);
+      }
+      
+      this.recordSuccess();
+      return result.value.encoded;
+      
+    } catch (error) {
+      this.recordFailure();
+      throw error;
+    }
+  }
+  
+  private recordFailure(): void {
+    this.failures++;
+    this.lastFailure = Date.now();
+    
+    if (this.failures >= this.threshold) {
+      this.state = 'OPEN';
+    }
+  }
+  
+  private recordSuccess(): void {
+    this.failures = 0;
+    this.state = 'CLOSED';
+  }
+}
+```
+
+## Custom Format Extensions
+
+### Adding New Formats
+
+While the encoder supports core formats, you can extend functionality:
+
+```typescript
+// Custom encoder wrapper with additional formats
+class ExtendedEncoder {
+  private coreEncoder = Encoder.create();
+  
+  encode(data: string, format: string): string {
+    // Handle core formats
+    if (['base64', 'base64url', 'hex', 'uri', 'ascii'].includes(format)) {
+      const result = this.coreEncoder.encode(data, format as EncodingFormat);
+      return result.isSuccess ? result.value.encoded : '';
+    }
+    
+    // Custom formats
+    switch (format) {
+      case 'rot13':
+        return this.rot13Encode(data);
+      case 'base32':
+        return this.base32Encode(data);
+      case 'binary':
+        return this.binaryEncode(data);
+      default:
+        throw new Error(`Unsupported format: ${format}`);
+    }
+  }
+  
+  private rot13Encode(data: string): string {
+    return data.replace(/[a-zA-Z]/g, char => {
+      const start = char <= 'Z' ? 65 : 97;
+      return String.fromCharCode(((char.charCodeAt(0) - start + 13) % 26) + start);
+    });
+  }
+  
+  private base32Encode(data: string): string {
+    // Base32 implementation
+    const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567';
+    // ... implementation
+    return ''; // Simplified
+  }
+  
+  private binaryEncode(data: string): string {
+    return Array.from(data)
+      .map(char => char.charCodeAt(0).toString(2).padStart(8, '0'))
+      .join(' ');
+  }
+}
+```
+
+## Unit Composition Patterns
+
+### Encoder + Compressor Chain
+
+```typescript
+// Composition with compression units
+import { Compressor } from '@synet/compressor';
+
+class CompressionEncodingPipeline {
+  private compressor = Compressor.create();
+  private encoder = Encoder.create();
+  
+  async processWithCompression(data: string): Promise<ProcessResult> {
+    // Step 1: Compress
+    const compressed = await this.compressor.compress(data);
+    
+    // Step 2: Encode compressed data
+    const encoded = this.encoder.encode(compressed, 'base64url');
+    
+    if (encoded.isFailure) {
+      throw new Error(`Encoding failed: ${encoded.error}`);
+    }
+    
+    return {
+      original: data,
+      compressed,
+      encoded: encoded.value.encoded,
+      compressionRatio: data.length / compressed.length,
+      totalRatio: data.length / encoded.value.encoded.length
+    };
+  }
+}
+```
+
+### Encoder + Crypto Integration
+
+```typescript
+// Secure encoding with encryption
+import { Crypto } from '@synet/crypto';
+
+class SecureEncoder {
+  private crypto = Crypto.create();
+  private encoder = Encoder.create();
+  
+  constructor() {
+    // Enable cross-learning
+    this.encoder.learn([this.crypto.teach()]);
+    this.crypto.learn([this.encoder.teach()]);
+  }
+  
+  async secureEncode(data: string, password: string): Promise<SecureResult> {
+    // Encrypt first
+    const encrypted = await this.crypto.execute('encrypt', data, password);
+    
+    // Then encode for safe transmission
+    const encoded = this.encoder.execute('encode', encrypted, 'base64url');
+    
+    return {
+      data: encoded,
+      algorithm: 'AES-256-GCM',
+      encoding: 'base64url',
+      timestamp: new Date().toISOString()
+    };
+  }
+  
+  async secureDecode(secureData: SecureResult, password: string): Promise<string> {
+    // Decode first
+    const decoded = this.encoder.execute('decode', secureData.data, 'base64url');
+    
+    // Then decrypt
+    const decrypted = await this.crypto.execute('decrypt', decoded, password);
+    
+    return decrypted;
+  }
+}
+```
+
+## Monitoring and Observability
+
+### Metrics Collection
+
+```typescript
+// Comprehensive metrics for production
+class EncoderMetrics {
+  private encoder = Encoder.create();
+  private metrics = new Map<string, number>();
+  
+  async encodeWithMetrics(data: string, format: EncodingFormat): Promise<string> {
+    const start = performance.now();
+    const operation = `encode_${format}`;
+    
+    try {
+      const result = this.encoder.encode(data, format);
+      
+      if (result.isSuccess) {
+        this.recordMetric(`${operation}_success`, 1);
+        this.recordMetric(`${operation}_input_size`, data.length);
+        this.recordMetric(`${operation}_output_size`, result.value.encoded.length);
+        this.recordMetric(`${operation}_compression_ratio`, result.value.compressionRatio);
+        
+        return result.value.encoded;
+      } else {
+        this.recordMetric(`${operation}_failure`, 1);
+        throw new Error(result.error);
+      }
+    } finally {
+      const duration = performance.now() - start;
+      this.recordMetric(`${operation}_duration_ms`, duration);
+    }
+  }
+  
+  private recordMetric(name: string, value: number): void {
+    this.metrics.set(name, (this.metrics.get(name) || 0) + value);
+  }
+  
+  getMetrics(): Record<string, number> {
+    return Object.fromEntries(this.metrics);
+  }
+  
+  // Integration with monitoring systems
+  async reportToPrometheus(): Promise<void> {
+    const metrics = this.getMetrics();
+    
+    for (const [name, value] of Object.entries(metrics)) {
+      // Report to Prometheus/StatsD/etc
+      console.log(`encoder_${name} ${value}`);
+    }
+  }
+}
+```
+
+### Health Monitoring
+
+```typescript
+// Health check implementation
+class EncoderHealthCheck {
+  private encoder = Encoder.create();
+  
+  async healthCheck(): Promise<HealthStatus> {
+    const checks = await Promise.allSettled([
+      this.testBasicOperations(),
+      this.testMemoryUsage(),
+      this.testPerformance()
+    ]);
+    
+    const results = checks.map((check, index) => ({
+      test: ['basic_operations', 'memory_usage', 'performance'][index],
+      status: check.status === 'fulfilled' ? 'pass' : 'fail',
+      details: check.status === 'fulfilled' ? check.value : check.reason
+    }));
+    
+    const allPassed = results.every(r => r.status === 'pass');
+    
+    return {
+      status: allPassed ? 'healthy' : 'unhealthy',
+      unit: this.encoder.whoami(),
+      capabilities: this.encoder.capabilities(),
+      checks: results,
+      timestamp: new Date().toISOString()
+    };
+  }
+  
+  private async testBasicOperations(): Promise<string> {
+    const testData = 'health-check-test';
+    const encoded = this.encoder.encode(testData, 'base64');
+    
+    if (encoded.isFailure) {
+      throw new Error(`Basic encoding failed: ${encoded.error}`);
+    }
+    
+    const decoded = this.encoder.decode(encoded.value.encoded, 'base64');
+    
+    if (decoded.isFailure || decoded.value.decoded !== testData) {
+      throw new Error('Basic round-trip failed');
+    }
+    
+    return 'Basic operations working';
+  }
+  
+  private async testMemoryUsage(): Promise<string> {
+    const before = process.memoryUsage();
+    
+    // Stress test with multiple operations
+    for (let i = 0; i < 1000; i++) {
+      this.encoder.encode(`test-${i}`, 'base64');
+    }
+    
+    const after = process.memoryUsage();
+    const growth = after.heapUsed - before.heapUsed;
+    
+    if (growth > 10 * 1024 * 1024) { // 10MB threshold
+      throw new Error(`Memory usage too high: ${growth} bytes`);
+    }
+    
+    return `Memory usage acceptable: ${growth} bytes`;
+  }
+  
+  private async testPerformance(): Promise<string> {
+    const start = performance.now();
+    
+    for (let i = 0; i < 100; i++) {
+      this.encoder.encode('performance-test-data', 'base64url');
+    }
+    
+    const duration = performance.now() - start;
+    
+    if (duration > 1000) { // 1 second threshold
+      throw new Error(`Performance too slow: ${duration}ms`);
+    }
+    
+    return `Performance acceptable: ${duration}ms for 100 operations`;
+  }
+}
+
+interface HealthStatus {
+  status: 'healthy' | 'unhealthy';
+  unit: string;
+  capabilities: string[];
+  checks: Array<{
+    test: string;
+    status: 'pass' | 'fail';
+    details: string;
+  }>;
+  timestamp: string;
+}
+```
+
+---
+
+## Summary
+
+This manual demonstrates how the @synet/encoder unit scales from simple encoding operations to complex production systems. The Unit Architecture's consciousness-based design enables:
+
+- **Composability** - Units learn from each other
+- **Observability** - Built-in identity and capability reporting  
+- **Reliability** - Immutable design with comprehensive error handling
+- **Scalability** - Stateless operations suitable for any deployment model
+
+The encoder serves as a foundation for larger encoding/decoding ecosystems while maintaining its essential identity and purpose.