@hiliosai/sdk 0.1.0 → 0.1.1

package/INTEGRATION_SERVICE.md

@@ -0,0 +1,394 @@
+# Integration Service Architecture
+
+## Overview
+
+Integration Services are specialized Moleculer services that provide a standardized way to connect external messaging platforms (WhatsApp, Telegram, Discord, etc.) with the application. They handle webhook reception, message normalization, and platform-specific communication patterns.
+
+## Purpose
+
+The Integration Service pattern solves several key challenges:
+
+1. **Platform Abstraction**: Provides unified interface for different messaging platforms
+2. **Webhook Security**: Implements signature validation and replay attack prevention  
+3. **Message Normalization**: Converts platform-specific messages to unified format
+4. **Retry Logic**: Handles transient failures with exponential backoff
+5. **Event Broadcasting**: Emits standardized events for message processing
+6. **Health Monitoring**: Provides health checks and status reporting
+
+## Architecture
+
+```mermaid
+graph TB
+    subgraph "External Platforms"
+        WA[WhatsApp]
+        TG[Telegram]
+        DC[Discord]
+        SL[Slack]
+    end
+    
+    subgraph "Integration Layer"
+        subgraph "Integration Services"
+            WAS[WhatsApp Service]
+            TGS[Telegram Service] 
+            DCS[Discord Service]
+            SLS[Slack Service]
+        end
+        
+        subgraph "Service Actions"
+            RW[i_receiveWebhook]
+            SM[i_sendMessage]
+            HC[i_healthCheck]
+            VW[i_verifyWebhook]
+            ST[i_status]
+            VC[i_validateCredentials]
+        end
+    end
+    
+    subgraph "Core Application"
+        subgraph "Message Processing"
+            MP[Message Processor]
+            MR[Message Router]
+            MS[Message Store]
+        end
+        
+        subgraph "Event System"
+            EB[Event Bus]
+            EH[Event Handlers]
+        end
+    end
+    
+    subgraph "Infrastructure"
+        DS[Datasources]
+        CA[Cache]
+        PE[Permissions]
+        CH[Context Helpers]
+    end
+
+    %% External webhook flows
+    WA -->|Webhook| WAS
+    TG -->|Webhook| TGS
+    DC -->|Webhook| DCS
+    SL -->|Webhook| SLS
+    
+    %% Integration service actions
+    WAS --> RW
+    WAS --> SM
+    WAS --> HC
+    
+    %% Service infrastructure
+    WAS --> DS
+    TGS --> CA
+    DCS --> PE
+    SLS --> CH
+    
+    %% Event flows
+    RW -->|integration.message.received| EB
+    SM -->|integration.message.sent| EB
+    SM -->|integration.message.failed| EB
+    
+    %% Processing flows
+    EB --> EH
+    EH --> MP
+    MP --> MR
+    MR --> MS
+    
+    %% Response flows
+    MP -->|Send Response| SM
+```
+
+## Integration Service Structure
+
+### Core Components
+
+1. **defineIntegration()** - Factory function that creates integration services
+2. **Security Helpers** - Webhook validation, timestamp checking, correlation IDs
+3. **Retry Mechanism** - Exponential backoff for failed operations
+4. **Standard Actions** - Predefined actions for common integration operations
+
+### Standard Actions
+
+All integration services automatically get these actions:
+
+| Action | REST Endpoint | Purpose |
+|--------|---------------|---------|
+| `i_receiveWebhook` | `POST /webhook` | Receive and process incoming webhooks |
+| `i_sendMessage` | `POST /send` | Send messages to external platform |
+| `i_healthCheck` | `GET /health` | Check integration health status |
+| `i_verifyWebhook` | `GET /webhook` | Verify webhook subscriptions |
+| `i_status` | `GET /status` | Get integration status information |
+| `i_validateCredentials` | `POST /validate` | Validate platform credentials |
+
+### Service Methods
+
+Integration-specific logic is implemented as service methods:
+
+| Method | Purpose | Required |
+|--------|---------|----------|
+| `normalize()` | Convert platform message to unified format | ✅ |
+| `sendMessage()` | Send message to platform | ✅ |
+| `transform()` | Transform outbound message to platform format | ❌ |
+| `validateWebhook()` | Validate webhook payload structure | ❌ |
+| `validateSignature()` | Validate webhook signature | ❌ |
+| `verifyWebhook()` | Handle webhook verification challenge | ❌ |
+| `checkHealth()` | Custom health check logic | ❌ |
+| `validateCredentials()` | Validate platform credentials | ❌ |
+
+## Usage Example
+
+### Basic WhatsApp Integration
+
+```typescript
+import {defineIntegration} from '@pkg/sdk';
+import type {BaseIntegration, NormalizedMessage, WebhookEvent} from '@pkg/sdk';
+
+const whatsappIntegration: BaseIntegration = {
+  id: 'whatsapp-business',
+  name: 'WhatsApp Business',
+  platform: 'whatsapp',
+  version: '1.0.0',
+  status: 'ACTIVE',
+  capabilities: ['SEND_MESSAGE', 'RECEIVE_MESSAGE', 'WEBHOOK_VALIDATION']
+};
+
+export default defineIntegration({
+  name: 'whatsapp',
+  integration: whatsappIntegration,
+  
+  // Required: Convert WhatsApp webhook to normalized message
+  async normalize(webhook: WebhookEvent): Promise<NormalizedMessage[]> {
+    const messages: NormalizedMessage[] = [];
+    
+    for (const entry of webhook.rawPayload.entry || []) {
+      for (const change of entry.changes || []) {
+        if (change.field === 'messages') {
+          for (const message of change.value.messages || []) {
+            messages.push({
+              id: message.id,
+              conversationId: message.from,
+              from: {
+                id: message.from,
+                name: change.value.contacts?.[0]?.profile?.name,
+              },
+              to: {
+                id: webhook.tenantId,
+              },
+              content: {
+                type: message.type === 'text' ? 'TEXT' : 'FILE',
+                text: message.text?.body,
+                media: message.image ? {
+                  url: message.image.id,
+                  mimeType: message.image.mime_type
+                } : undefined
+              },
+              timestamp: parseInt(message.timestamp) * 1000,
+              platform: 'whatsapp',
+              metadata: {
+                phoneNumberId: change.value.metadata.phone_number_id
+              }
+            });
+          }
+        }
+      }
+    }
+    
+    return messages;
+  },
+
+  // Required: Send message to WhatsApp
+  async sendMessage(ctx, message, config) {
+    const phoneNumberId = config.credentials.phoneNumberId;
+    const accessToken = config.credentials.accessToken;
+    
+    const response = await fetch(
+      `https://graph.facebook.com/v18.0/${phoneNumberId}/messages`,
+      {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${accessToken}`,
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          messaging_product: 'whatsapp',
+          to: message.to.id,
+          text: { body: message.content.text }
+        })
+      }
+    );
+
+    if (!response.ok) {
+      const error = await response.text();
+      return {
+        success: false,
+        error: new Error(`WhatsApp API error: ${error}`)
+      };
+    }
+
+    const result = await response.json();
+    return {
+      success: true,
+      messageId: result.messages[0].id,
+      metadata: { phoneNumberId }
+    };
+  },
+
+  // Optional: Validate webhook signature
+  validateSignature(webhook) {
+    const signature = webhook.headers['x-hub-signature-256'];
+    const secret = process.env.WHATSAPP_WEBHOOK_SECRET;
+    
+    if (!signature || !secret) return false;
+    
+    const expectedSignature = 'sha256=' + 
+      crypto.createHmac('sha256', secret)
+        .update(webhook.rawBody || '')
+        .digest('hex');
+    
+    return SecurityHelpers.secureCompare(signature, expectedSignature);
+  },
+
+  // Optional: Handle webhook verification
+  verifyWebhook(params) {
+    const { mode, token, challenge } = params;
+    const verifyToken = process.env.WHATSAPP_VERIFY_TOKEN;
+    
+    if (mode === 'subscribe' && token === verifyToken) {
+      return challenge;
+    }
+    return null;
+  },
+
+  // Optional: Custom health check
+  async checkHealth(ctx, config) {
+    try {
+      const response = await fetch(
+        `https://graph.facebook.com/v18.0/${config.credentials.phoneNumberId}`,
+        {
+          headers: {
+            'Authorization': `Bearer ${config.credentials.accessToken}`
+          }
+        }
+      );
+      
+      return {
+        status: response.ok ? 'healthy' : 'unhealthy',
+        message: response.ok ? 'WhatsApp API accessible' : 'WhatsApp API error',
+        details: {
+          statusCode: response.status,
+          timestamp: new Date().toISOString()
+        }
+      };
+    } catch (error) {
+      return {
+        status: 'unhealthy',
+        message: error.message,
+        details: { timestamp: new Date().toISOString() }
+      };
+    }
+  }
+});
+```
+
+### Advanced Integration with Datasources and Cache
+
+```typescript
+import {defineIntegration} from '@pkg/sdk';
+import {WhatsAppDatasource} from './datasources';
+
+export default defineIntegration({
+  name: 'whatsapp',
+  integration: whatsappIntegration,
+  
+  // Configure per-service datasources
+  datasources: {
+    whatsapp: WhatsAppDatasource
+  },
+  
+  // Configure per-service cache
+  cache: {
+    redisUrl: process.env.WHATSAPP_REDIS_URL,
+    ttl: 5 * 60 * 1000, // 5 minutes
+    namespace: 'whatsapp'
+  },
+
+  async normalize(webhook) {
+    // Access datasource via context in actions
+    // this.datasources is not available in methods
+    // Use dependency injection pattern instead
+    return await this.processWebhookMessages(webhook);
+  },
+
+  async sendMessage(ctx, message, config) {
+    // Use cache for rate limiting
+    const rateLimitKey = `rate_limit:${config.credentials.phoneNumberId}`;
+    const currentCount = await ctx.cache.get(rateLimitKey) || 0;
+    
+    if (currentCount >= 1000) {
+      return {
+        success: false,
+        error: new Error('Rate limit exceeded')
+      };
+    }
+    
+    // Send message...
+    const result = await this.sendToWhatsApp(message, config);
+    
+    // Update rate limit counter
+    await ctx.cache.set(rateLimitKey, currentCount + 1, 60 * 60 * 1000);
+    
+    return result;
+  }
+});
+```
+
+## Event Flow
+
+### Inbound Message Flow
+
+1. **Webhook Received** → `i_receiveWebhook` action
+2. **Security Validation** → Timestamp, signature, payload checks  
+3. **Message Normalization** → Convert to unified format
+4. **Event Emission** → `integration.message.received` event
+5. **Message Processing** → Application-specific logic
+6. **Response Generation** → Optional automated responses
+
+### Outbound Message Flow
+
+1. **Send Request** → `i_sendMessage` action
+2. **Message Transformation** → Convert to platform format
+3. **Retry Logic** → Exponential backoff for failures
+4. **Platform API Call** → Send via platform API
+5. **Event Emission** → `integration.message.sent` or `integration.message.failed`
+6. **Status Tracking** → Update message delivery status
+
+## Security Features
+
+### Webhook Validation
+
+- **Signature Verification**: HMAC-SHA256 validation using platform secrets
+- **Replay Attack Prevention**: Timestamp validation (5-minute window)
+- **Timing Attack Protection**: Constant-time string comparison
+- **Correlation IDs**: Request tracking for debugging and audit trails
+
+### Best Practices
+
+1. **Always validate webhook signatures** in production
+2. **Use environment variables** for secrets and tokens
+3. **Implement proper error handling** with structured errors
+4. **Add correlation IDs** for request tracking
+5. **Use retry logic** for transient failures
+6. **Monitor health endpoints** for platform availability
+7. **Cache frequently accessed data** to reduce API calls
+
+## Infrastructure Integration
+
+Integration services automatically inherit:
+
+- **Datasource Access**: Database connections, external APIs
+- **Cache Layer**: Redis/in-memory caching with TTL
+- **Permission System**: Role-based access control
+- **Context Helpers**: Common utilities and operations
+- **Memoization**: Action result caching
+- **Health Monitoring**: Automated health checks
+- **Event System**: Pub/sub event broadcasting
+
+This architecture provides a robust, scalable foundation for building messaging platform integrations while maintaining consistency, security, and observability across all platforms.

package/package.json

@@ -1,17 +1,19 @@
 {
   "name": "@hiliosai/sdk",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "type": "module",
   "exports": {
     ".": "./src/index.ts"
   },
   "scripts": {
     "build": "tsup",
-    "format": "bunx prettier --write ./**/*.{ts,json,md,yaml}"
+    "format": "bunx prettier --write ./**/*.{ts,json,md,yaml}",
+    "typecheck": "tsc --noEmit"
   },
   "dependencies": {
     "@keyv/redis": "5.1.5",
     "@ltv/env": "4.0.3",
+    "@moleculer/channels": "0.2.0",
     "keyv": "5.5.5",
     "lodash": "4.17.21",
     "moleculer": "0.14.35"
@@ -24,4 +26,4 @@
     "bun-types": "latest"
   },
   "prettier": "@hiliosai/prettier"
-}
+}

package/src/configs/constants.ts

@@ -0,0 +1,122 @@
+import env from '@ltv/env';
+
+/**
+ * Base namespace for all channels
+ */
+export const NAMESPACE = env.string('NAMESPACE', 'hios').toLowerCase();
+
+export const CHANNELS = {
+  // Webhook processing channels
+  WEBHOOK: {
+    // Pattern: hios.webhook.{tenantId}.{platform}
+    PATTERN: `${NAMESPACE}.webhook.*.*`,
+    PREFIX: `${NAMESPACE}.webhook`,
+    build: (tenantId: string, platform: string) =>
+      `${NAMESPACE}.webhook.${tenantId}.${platform}`,
+  },
+
+  // Message processing channels
+  PROCESSING: {
+    // Pattern: hios.processing.{tenantId}.{messageType}
+    PATTERN: `${NAMESPACE}.processing.*.*`,
+    PREFIX: `${NAMESPACE}.processing`,
+    build: (tenantId: string, messageType: string) =>
+      `${NAMESPACE}.processing.${tenantId}.${messageType}`,
+  },
+
+  // Response/outbound message channels
+  RESPONSE: {
+    // Pattern: hios.response.{tenantId}.{platform}
+    PATTERN: `${NAMESPACE}.response.*.*`,
+    PREFIX: `${NAMESPACE}.response`,
+    build: (tenantId: string, platform: string) =>
+      `${NAMESPACE}.response.${tenantId}.${platform}`,
+  },
+
+  // System channels
+  SYSTEM: {
+    // Error handling
+    ERRORS: `${NAMESPACE}.system.errors`,
+
+    // Metrics and monitoring
+    METRICS: `${NAMESPACE}.system.metrics`,
+
+    // Health checks
+    HEALTH: `${NAMESPACE}.system.health`,
+
+    // Integration lifecycle events
+    INTEGRATION_REGISTERED: `${NAMESPACE}.system.integration.registered`,
+    INTEGRATION_UNREGISTERED: `${NAMESPACE}.system.integration.unregistered`,
+  },
+
+  // Dead letter queues
+  DLQ: {
+    // Failed webhook processing
+    WEBHOOK_FAILED: `${NAMESPACE}.dlq.webhook.failed`,
+
+    // Failed message sends
+    SEND_FAILED: `${NAMESPACE}.dlq.send.failed`,
+
+    // Failed processing
+    PROCESSING_FAILED: `${NAMESPACE}.dlq.processing.failed`,
+
+    // Build DLQ name for specific integration
+    buildSendFailed: (platform: string) =>
+      `${NAMESPACE}.dlq.send.${platform}.failed`,
+  },
+} as const;
+
+/**
+ * Channel Configuration Constants
+ */
+export const CHANNEL_CONFIG = {
+  // Default settings for message channels
+  DEFAULTS: {
+    maxInFlight: 10,
+    maxRetries: 3,
+    deadLettering: {
+      enabled: true,
+    },
+  },
+
+  // High-priority channels (webhooks)
+  HIGH_PRIORITY: {
+    maxInFlight: 50,
+    maxRetries: 5,
+    deadLettering: {
+      enabled: true,
+    },
+  },
+
+  // Low-priority channels (metrics, logs)
+  LOW_PRIORITY: {
+    maxInFlight: 5,
+    maxRetries: 1,
+    deadLettering: {
+      enabled: false,
+    },
+  },
+} as const;
+
+/**
+ * Subject patterns for NATS JetStream
+ */
+export const SUBJECTS = {
+  // All webhook subjects
+  WEBHOOK_ALL: `${NAMESPACE}.webhook.>`,
+
+  // All processing subjects
+  PROCESSING_ALL: `${NAMESPACE}.processing.>`,
+
+  // All response subjects
+  RESPONSE_ALL: `${NAMESPACE}.response.>`,
+
+  // All system subjects
+  SYSTEM_ALL: `${NAMESPACE}.system.>`,
+
+  // All DLQ subjects
+  DLQ_ALL: `${NAMESPACE}.dlq.>`,
+
+  // Wildcard for all HIOS subjects
+  ALL: `${NAMESPACE}.>`,
+} as const;

package/src/configs/index.ts

@@ -1 +1,2 @@
 export * from './permissions';
+export {default as moleculer} from './moleculer';

package/src/configs/moleculer/bulkhead.ts

@@ -0,0 +1,8 @@
+export const bulkheadConfig = {
+  // Enable feature.
+  enabled: false,
+  // Maximum concurrent executions.
+  concurrency: 10,
+  // Maximum size of queue
+  maxQueueSize: 100,
+};

package/src/configs/moleculer/channels.ts

@@ -0,0 +1,102 @@
+import env from '@ltv/env';
+import {Middleware as createChannelsMiddleware} from '@moleculer/channels';
+import {type Middleware} from 'moleculer';
+
+import {SUBJECTS, NAMESPACE as cfgNamespace} from '../constants';
+
+const NAMESPACE = cfgNamespace.toUpperCase();
+
+const middleware = createChannelsMiddleware({
+  adapter: {
+    type: 'NATS',
+    options: {
+      nats: {
+        url: env.string('NATS_URL', 'nats://localhost:4222'),
+
+        /** Connection options for reliability */
+        connectionOptions: {
+          name: 'hios',
+          timeout: 10000,
+          reconnect: true,
+          maxReconnectAttempts: 10,
+          reconnectTimeWait: 2000,
+          maxReconnectTimeWait: 30000,
+          pingInterval: 20000,
+          maxPingOut: 2,
+        },
+
+        /**
+         * Stream configuration for multi-tenant messaging
+         *
+         * Environment variables for production:
+         * - NATS_MAX_MESSAGES: Default 100K (dev) -> 10M+ (prod)
+         * - NATS_MAX_BYTES_GB: Default 1GB (dev) -> 100GB+ (prod)
+         * - NATS_MAX_AGE_DAYS: Default 7 (dev) -> 30+ (prod)
+         * - NATS_MAX_MSG_SIZE_MB: Default 1MB (dev) -> 5MB (prod)
+         * - NATS_REPLICAS: Default 1 (dev) -> 3 (prod)
+         */
+        streamConfig: {
+          name: `${NAMESPACE}_MESSAGES`,
+          subjects: [
+            SUBJECTS.WEBHOOK_ALL,
+            SUBJECTS.PROCESSING_ALL,
+            SUBJECTS.RESPONSE_ALL,
+            SUBJECTS.SYSTEM_ALL,
+            SUBJECTS.DLQ_ALL,
+          ],
+          retention: 'limits',
+          max_msgs: env.int('NATS_MAX_MESSAGES', 100_000), // 100K for dev, 10M+ for prod
+          max_bytes: env.int('NATS_MAX_BYTES_GB', 1) * 1024 * 1024 * 1024, // 1GB for dev, 100GB+ for prod
+          max_age: env.int('NATS_MAX_AGE_DAYS', 7) * 24 * 60 * 60 * 1000000000, // 7 days dev, 30+ days prod
+          max_msg_size: env.int('NATS_MAX_MSG_SIZE_MB', 1) * 1024 * 1024, // 1MB dev, 5MB prod
+          storage: 'file', // Persistent storage
+          num_replicas: env.int('NATS_REPLICAS', 1), // 1 for dev, 3 for prod
+          discard: 'old', // Remove old messages when limits hit
+          duplicate_window: 2 * 60 * 1000000000, // 2 minutes dedup window
+        },
+
+        /** Consumer options optimized for LLM processing */
+        consumerOptions: {
+          config: {
+            // Start with new messages (don't replay old ones on restart)
+            deliver_policy: 'new',
+
+            // Explicit acknowledgment required (critical for LLM processing)
+            ack_policy: 'explicit',
+
+            // Allow 5 unacknowledged messages per consumer (rate limiting)
+            max_ack_pending: 5,
+
+            // Acknowledgment timeout for LLM processing (2 minutes)
+            ack_wait: 120 * 1000000000, // 2 minutes in nanoseconds
+
+            // Maximum delivery attempts before dead letter
+            max_deliver: 3,
+
+            // Backoff for failed message retries
+            backoff: [
+              1000000000, // 1 second
+              5000000000, // 5 seconds
+              30000000000, // 30 seconds
+            ],
+          },
+        },
+      },
+
+      /** Application-level flow control */
+      maxInFlight: 5, // Limit concurrent LLM requests per service
+      maxRetries: 2, // App-level retries (NATS handles delivery retries)
+
+      /** Dead letter queue for failed messages */
+      deadLettering: {
+        enabled: true,
+        queueName: 'FAILED_MESSAGES',
+        // Send to dead letter after NATS max_deliver attempts
+      },
+    },
+  },
+}) as unknown as Middleware;
+
+export const ChannelsMiddleware: Middleware = {
+  ...middleware,
+};

package/src/configs/moleculer/circuit-breaker.ts

@@ -0,0 +1,17 @@
+import type {Errors} from 'moleculer';
+
+export const circuitBreakerConfig = {
+  // Enable feature
+  enabled: false,
+  // Threshold value. 0.5 means that 50% should be failed for tripping.
+  threshold: 0.5,
+  // Minimum request count. Below it, CB does not trip.
+  minRequestCount: 20,
+  // Number of seconds for time window.
+  windowTime: 60,
+  // Number of milliseconds to switch from open to half-open state
+  halfOpenTime: 10 * 1000,
+  // A function to check failed requests.
+  check: (err: Errors.MoleculerError | Error) =>
+    (err as Errors.MoleculerError).code >= 500,
+};