@pioneer-platform/blockbook 8.32.0 → 8.32.2

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @pioneer-platform/blockbook
 
+## 8.32.2
+
+### Patch Changes
+
+- 163653b: chore: fix(swap): Fix critical swap amount bug causing 0 ETH transfers
+
+## 8.32.1
+
+### Patch Changes
+
+- fix(swap): Fix critical swap amount bug causing 0 ETH transfers
+
 ## 8.32.0
 
 ### Minor Changes

package/FAIL_FAST_IMPLEMENTATION.md

@@ -0,0 +1,696 @@
+# Fail-Fast Implementation Guide
+
+## Purpose
+
+Implement robust health monitoring and fail-fast mechanisms to detect when the BlockbookWebSocket event system is broken, and signal failure to container orchestration layers immediately.
+
+## Core Philosophy
+
+**Silent failures are catastrophic failures.** If we can't receive events, we MUST:
+1. Detect the failure quickly (within minutes, not hours)
+2. Log clear diagnostic information
+3. Signal unhealthy status to orchestration layer
+4. Exit with non-zero exit code to trigger container restart
+
+---
+
+## Implementation Layers
+
+### Layer 1: BlockbookWebSocket Class Health Tracking
+
+Add health monitoring directly to `BlockbookWebSocket.ts`:
+
+```typescript
+export class BlockbookWebSocket extends EventEmitter {
+  // ... existing fields ...
+
+  // Health tracking
+  private lastEventTime?: number;
+  private healthCheckInterval?: NodeJS.Timeout;
+  private eventCounts = {
+    blocks: 0,
+    transactions: 0,
+    total: 0
+  };
+
+  // Configuration
+  private readonly HEALTH_CHECK_INTERVAL = 60000; // 1 minute
+  private readonly EVENT_TIMEOUT = 300000; // 5 minutes
+
+  constructor(url: string) {
+    super();
+    this.url = url;
+  }
+
+  async connect(): Promise<void> {
+    // ... existing connection code ...
+
+    this.ws.once('open', () => {
+      this.connected = true;
+      this.reconnectAttempts = 0;
+      this.emit('connected', this.url);
+      this.startPing();
+      this.startHealthMonitoring(); // ← NEW: Start health checks
+      resolve();
+    });
+
+    // ... rest of connection code ...
+  }
+
+  private handleMessage(message: BlockbookMessage): void {
+    const { id, data } = message;
+
+    if (!id) {
+      this.emit('error', new Error('Received message without ID'));
+      return;
+    }
+
+    // Check if this is a response to a pending request
+    const pending = this.pendingRequests.get(id);
+    if (pending) {
+      this.pendingRequests.delete(id);
+
+      if (data?.error) {
+        pending.reject(new Error(data.error.message || 'Request failed'));
+      } else {
+        pending.resolve(data);
+      }
+      return;
+    }
+
+    // Check if this is a subscription notification
+    const subscription = this.subscriptions.get(id);
+    if (subscription) {
+      // ← NEW: Track event reception
+      this.lastEventTime = Date.now();
+      this.eventCounts.total++;
+
+      // Track by type
+      if (subscription.method === 'subscribeNewBlock') {
+        this.eventCounts.blocks++;
+      } else if (subscription.method === 'subscribeAddresses') {
+        this.eventCounts.transactions++;
+      }
+
+      if (data?.error) {
+        this.emit('error', new Error(`Subscription error for ${subscription.method}: ${data.error.message}`));
+      } else {
+        try {
+          const result = subscription.callback(data);
+          if (result instanceof Promise) {
+            result.catch((error) => {
+              this.emit('error', new Error(`Subscription callback error: ${error}`));
+            });
+          }
+        } catch (error) {
+          this.emit('error', new Error(`Subscription callback error: ${error}`));
+        }
+      }
+      return;
+    }
+
+    // Unknown message
+    this.emit('unknown-message', message);
+  }
+
+  private startHealthMonitoring(): void {
+    this.stopHealthMonitoring();
+
+    this.healthCheckInterval = setInterval(() => {
+      this.checkHealth();
+    }, this.HEALTH_CHECK_INTERVAL);
+  }
+
+  private stopHealthMonitoring(): void {
+    if (this.healthCheckInterval) {
+      clearInterval(this.healthCheckInterval);
+      this.healthCheckInterval = undefined;
+    }
+  }
+
+  private checkHealth(): void {
+    const now = Date.now();
+    const hasActiveSubscriptions = this.subscriptions.size > 0;
+
+    if (!hasActiveSubscriptions) {
+      // No subscriptions, nothing to monitor
+      return;
+    }
+
+    const timeSinceLastEvent = this.lastEventTime
+      ? now - this.lastEventTime
+      : Infinity;
+
+    const healthStatus = {
+      connected: this.connected,
+      subscriptions: this.subscriptions.size,
+      lastEventTime: this.lastEventTime,
+      timeSinceLastEvent,
+      eventCounts: { ...this.eventCounts },
+      url: this.url
+    };
+
+    // Emit health status for monitoring
+    this.emit('health-check', healthStatus);
+
+    // FAIL FAST: No events received within timeout
+    if (timeSinceLastEvent > this.EVENT_TIMEOUT) {
+      const errorMessage = `CRITICAL: No events received in ${this.EVENT_TIMEOUT / 1000}s despite ${this.subscriptions.size} active subscriptions`;
+
+      this.emit('health-failure', {
+        ...healthStatus,
+        error: errorMessage,
+        severity: 'critical'
+      });
+
+      // Log to stderr for container log collection
+      console.error(`[BlockbookWebSocket] ${errorMessage}`);
+      console.error(`[BlockbookWebSocket] Health Status:`, JSON.stringify(healthStatus, null, 2));
+    }
+  }
+
+  getHealthStatus(): {
+    connected: boolean;
+    subscriptions: number;
+    lastEventTime?: number;
+    timeSinceLastEvent: number;
+    eventCounts: typeof this.eventCounts;
+    healthy: boolean;
+  } {
+    const now = Date.now();
+    const timeSinceLastEvent = this.lastEventTime
+      ? now - this.lastEventTime
+      : Infinity;
+
+    const hasActiveSubscriptions = this.subscriptions.size > 0;
+    const healthy = !hasActiveSubscriptions || timeSinceLastEvent < this.EVENT_TIMEOUT;
+
+    return {
+      connected: this.connected,
+      subscriptions: this.subscriptions.size,
+      lastEventTime: this.lastEventTime,
+      timeSinceLastEvent,
+      eventCounts: { ...this.eventCounts },
+      healthy
+    };
+  }
+
+  async disconnect(): Promise<void> {
+    if (!this.ws || !this.connected) {
+      return;
+    }
+
+    this.reconnecting = false; // Prevent auto-reconnect
+    this.stopHealthMonitoring(); // ← NEW: Stop health checks
+
+    return new Promise((resolve) => {
+      this.ws!.once('close', () => resolve());
+      this.ws!.close(1000, 'Normal closure');
+    });
+  }
+}
+```
+
+---
+
+### Layer 2: Watchtower Service Health Endpoint
+
+Add health check endpoint to watchtower service that aggregates all websocket health statuses:
+
+```typescript
+// services/pioneer-watchtower/src/health.ts
+
+import { network } from './blockbook-integration';
+
+export interface WatchtowerHealth {
+  status: 'healthy' | 'degraded' | 'failing';
+  timestamp: number;
+  chains: {
+    [symbol: string]: {
+      connected: boolean;
+      subscriptions: number;
+      timeSinceLastEvent: number;
+      eventCounts: {
+        blocks: number;
+        transactions: number;
+        total: number;
+      };
+      healthy: boolean;
+    };
+  };
+  overall: {
+    totalChains: number;
+    healthyChains: number;
+    degradedChains: number;
+    failingChains: number;
+  };
+}
+
+export function getWatchtowerHealth(): WatchtowerHealth {
+  const sockets = network.getBlockbookSockets();
+  const chains: WatchtowerHealth['chains'] = {};
+
+  let healthyCount = 0;
+  let degradedCount = 0;
+  let failingCount = 0;
+
+  for (const [symbol, socket] of Object.entries(sockets)) {
+    const health = socket.getHealthStatus();
+    chains[symbol] = health;
+
+    if (health.healthy) {
+      healthyCount++;
+    } else if (health.timeSinceLastEvent < 600000) { // 10 minutes
+      degradedCount++;
+    } else {
+      failingCount++;
+    }
+  }
+
+  const totalChains = Object.keys(sockets).length;
+
+  let status: 'healthy' | 'degraded' | 'failing' = 'healthy';
+  if (failingCount > 0) {
+    status = 'failing';
+  } else if (degradedCount > 0) {
+    status = 'degraded';
+  }
+
+  return {
+    status,
+    timestamp: Date.now(),
+    chains,
+    overall: {
+      totalChains,
+      healthyChains: healthyCount,
+      degradedChains: degradedCount,
+      failingChains: failingCount
+    }
+  };
+}
+
+// HTTP endpoint
+app.get('/health', (req, res) => {
+  const health = getWatchtowerHealth();
+
+  const statusCode = health.status === 'healthy' ? 200 :
+                      health.status === 'degraded' ? 200 :
+                      503; // Service Unavailable for failing
+
+  res.status(statusCode).json(health);
+});
+
+// Kubernetes-style liveness probe (simple boolean)
+app.get('/healthz', (req, res) => {
+  const health = getWatchtowerHealth();
+
+  if (health.status === 'failing') {
+    res.status(503).send('FAILING');
+  } else {
+    res.status(200).send('OK');
+  }
+});
+
+// Kubernetes-style readiness probe (ready to accept traffic)
+app.get('/readyz', (req, res) => {
+  const health = getWatchtowerHealth();
+
+  // Only ready if at least 80% of chains are healthy
+  const healthyPercent = health.overall.healthyChains / health.overall.totalChains;
+
+  if (healthyPercent >= 0.8) {
+    res.status(200).send('READY');
+  } else {
+    res.status(503).send('NOT_READY');
+  }
+});
+```
+
+---
+
+### Layer 3: Automatic Fail-Fast Exit
+
+Automatically exit with error code when event system is critically broken:
+
+```typescript
+// services/pioneer-watchtower/src/fail-fast.ts
+
+import { network } from './blockbook-integration';
+
+const CRITICAL_TIMEOUT = 600000; // 10 minutes
+const CHECK_INTERVAL = 60000;    // Check every minute
+
+export function enableFailFast(): void {
+  setInterval(() => {
+    const sockets = network.getBlockbookSockets();
+
+    for (const [symbol, socket] of Object.entries(sockets)) {
+      const health = socket.getHealthStatus();
+
+      // If we have subscriptions but no events in 10 minutes, FAIL HARD
+      if (health.subscriptions > 0 && health.timeSinceLastEvent > CRITICAL_TIMEOUT) {
+        console.error(`\n${'='.repeat(60)}`);
+        console.error(`🚨 CRITICAL: ${symbol} event system is BROKEN`);
+        console.error(`${'='.repeat(60)}`);
+        console.error(`Subscriptions: ${health.subscriptions}`);
+        console.error(`Last Event: ${health.lastEventTime ? new Date(health.lastEventTime).toISOString() : 'NEVER'}`);
+        console.error(`Time Since Last Event: ${Math.floor(health.timeSinceLastEvent / 1000)}s`);
+        console.error(`Event Counts:`, health.eventCounts);
+        console.error(`${'='.repeat(60)}`);
+        console.error(`\n❌ Exiting with code 1 to signal container orchestration`);
+        console.error(`Container will be restarted by orchestrator\n`);
+
+        process.exit(1);
+      }
+    }
+  }, CHECK_INTERVAL);
+
+  console.log(`✅ Fail-fast monitoring enabled (${CRITICAL_TIMEOUT/1000}s timeout, ${CHECK_INTERVAL/1000}s check interval)`);
+}
+
+// Listen for health-failure events from individual sockets
+export function monitorSocketHealth(): void {
+  const sockets = network.getBlockbookSockets();
+
+  for (const [symbol, socket] of Object.entries(sockets)) {
+    socket.on('health-failure', (status) => {
+      console.error(`\n⚠️ Health failure detected for ${symbol}:`, status);
+
+      if (status.severity === 'critical') {
+        console.error(`🚨 CRITICAL health failure for ${symbol} - considering fail-fast exit`);
+      }
+    });
+
+    socket.on('health-check', (status) => {
+      // Log health checks at debug level
+      if (process.env.DEBUG_HEALTH) {
+        console.log(`[${symbol}] Health:`, {
+          subscriptions: status.subscriptions,
+          timeSinceLastEvent: Math.floor(status.timeSinceLastEvent / 1000),
+          events: status.eventCounts.total
+        });
+      }
+    });
+  }
+}
+```
+
+---
+
+### Layer 4: Container Integration
+
+#### Docker Compose Health Check
+
+```yaml
+services:
+  pioneer-watchtower:
+    image: pioneer-watchtower:latest
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/healthz"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s
+    restart: unless-stopped
+```
+
+#### Kubernetes Probes
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pioneer-watchtower
+spec:
+  containers:
+  - name: watchtower
+    image: pioneer-watchtower:latest
+    ports:
+    - containerPort: 3000
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: 3000
+      initialDelaySeconds: 60
+      periodSeconds: 30
+      timeoutSeconds: 10
+      failureThreshold: 3
+    readinessProbe:
+      httpGet:
+        path: /readyz
+        port: 3000
+      initialDelaySeconds: 30
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 2
+```
+
+---
+
+## Monitoring and Alerting
+
+### Prometheus Metrics
+
+```typescript
+// services/pioneer-watchtower/src/metrics.ts
+
+import { network } from './blockbook-integration';
+import { register, Gauge, Counter } from 'prom-client';
+
+// Metrics
+const subscriptionsGauge = new Gauge({
+  name: 'blockbook_subscriptions_active',
+  help: 'Number of active websocket subscriptions',
+  labelNames: ['chain']
+});
+
+const eventCounter = new Counter({
+  name: 'blockbook_events_total',
+  help: 'Total number of events received',
+  labelNames: ['chain', 'type'] // type: block, transaction
+});
+
+const lastEventGauge = new Gauge({
+  name: 'blockbook_last_event_seconds',
+  help: 'Seconds since last event received',
+  labelNames: ['chain']
+});
+
+const healthGauge = new Gauge({
+  name: 'blockbook_health_status',
+  help: 'Health status (1 = healthy, 0 = unhealthy)',
+  labelNames: ['chain']
+});
+
+// Update metrics periodically
+setInterval(() => {
+  const sockets = network.getBlockbookSockets();
+
+  for (const [symbol, socket] of Object.entries(sockets)) {
+    const health = socket.getHealthStatus();
+
+    subscriptionsGauge.set({ chain: symbol }, health.subscriptions);
+    lastEventGauge.set({ chain: symbol }, health.timeSinceLastEvent / 1000);
+    healthGauge.set({ chain: symbol }, health.healthy ? 1 : 0);
+
+    // Event counters are incremented as events occur
+  }
+}, 10000); // Update every 10 seconds
+
+// Expose metrics endpoint
+app.get('/metrics', async (req, res) => {
+  res.set('Content-Type', register.contentType);
+  res.end(await register.metrics());
+});
+```
+
+### Alert Rules (Prometheus Alertmanager)
+
+```yaml
+groups:
+  - name: blockbook_events
+    interval: 30s
+    rules:
+      - alert: BlockbookNoEvents
+        expr: time() - blockbook_last_event_seconds > 300
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Blockbook events not received for {{ $labels.chain }}"
+          description: "No events received for {{ $labels.chain }} in over 5 minutes despite active subscriptions"
+
+      - alert: BlockbookUnhealthy
+        expr: blockbook_health_status == 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Blockbook websocket unhealthy for {{ $labels.chain }}"
+          description: "Health check failing for {{ $labels.chain }} chain"
+
+      - alert: BlockbookSubscriptionsZero
+        expr: blockbook_subscriptions_active == 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "No active subscriptions for {{ $labels.chain }}"
+          description: "Watchtower has no active subscriptions for {{ $labels.chain }}"
+```
+
+---
+
+## Testing the Fail-Fast System
+
+### Manual Test 1: Simulate Event Timeout
+
+```bash
+# Start watchtower
+npm start
+
+# Wait for subscriptions to be established
+sleep 30
+
+# Kill the blockbook websocket connection (simulate network failure)
+# The service should:
+# 1. Detect no events within timeout
+# 2. Log health failures
+# 3. Exit with code 1
+# 4. Container orchestrator restarts service
+```
+
+### Manual Test 2: Check Health Endpoints
+
+```bash
+# Health endpoint (detailed)
+curl http://localhost:3000/health | jq
+
+# Expected output:
+{
+  "status": "healthy",
+  "timestamp": 1736496000000,
+  "chains": {
+    "ETH": {
+      "connected": true,
+      "subscriptions": 2,
+      "timeSinceLastEvent": 5432,
+      "eventCounts": {
+        "blocks": 42,
+        "transactions": 7,
+        "total": 49
+      },
+      "healthy": true
+    }
+  },
+  "overall": {
+    "totalChains": 1,
+    "healthyChains": 1,
+    "degradedChains": 0,
+    "failingChains": 0
+  }
+}
+
+# Liveness probe (simple)
+curl http://localhost:3000/healthz
+# Expected: "OK" with 200 status
+
+# Readiness probe
+curl http://localhost:3000/readyz
+# Expected: "READY" with 200 status if >= 80% chains healthy
+```
+
+### Automated Integration Test
+
+```javascript
+// __tests__/test-fail-fast.js
+
+const network = require('../lib/index');
+
+describe('Fail-Fast System', () => {
+  it('should detect event timeout and emit health-failure', async () => {
+    await network.init();
+    const sockets = network.getBlockbookSockets();
+    const ethSocket = sockets.ETH;
+
+    // Subscribe but never receive events
+    await ethSocket.connect();
+    await ethSocket.subscribeNewBlock(() => {});
+
+    // Mock: prevent events from firing
+    jest.spyOn(ethSocket, 'handleMessage').mockImplementation(() => {});
+
+    // Wait for health failure event
+    const healthFailure = new Promise((resolve) => {
+      ethSocket.on('health-failure', resolve);
+    });
+
+    const failure = await healthFailure;
+    expect(failure.severity).toBe('critical');
+    expect(failure.timeSinceLastEvent).toBeGreaterThan(300000);
+  });
+
+  it('should expose healthy status when events are flowing', async () => {
+    await network.init();
+    const health = getWatchtowerHealth();
+
+    // Wait for at least one event
+    await waitForEvents(60000);
+
+    const updatedHealth = getWatchtowerHealth();
+    expect(updatedHealth.status).toBe('healthy');
+    expect(updatedHealth.overall.healthyChains).toBeGreaterThan(0);
+  });
+});
+```
+
+---
+
+## Deployment Checklist
+
+- [ ] BlockbookWebSocket health tracking implemented
+- [ ] Watchtower health endpoints added (`/health`, `/healthz`, `/readyz`)
+- [ ] Fail-fast auto-exit mechanism enabled
+- [ ] Container health checks configured
+- [ ] Prometheus metrics exposed
+- [ ] Alert rules configured
+- [ ] Integration tests passing
+- [ ] Documentation updated
+- [ ] Monitoring dashboards created
+- [ ] On-call runbooks updated
+
+---
+
+## Incident Response
+
+### When Fail-Fast Triggers
+
+1. **Container Restarts**: Orchestrator automatically restarts the container
+2. **Check Logs**: Review stderr logs for health failure details
+3. **Verify Network**: Check if blockbook provider is accessible
+4. **Check Subscriptions**: Verify addresses are correctly subscribed
+5. **Monitor Recovery**: Watch for events to start flowing again
+6. **Escalate**: If container restart doesn't resolve, escalate to on-call
+
+### Common Issues
+
+| Symptom | Likely Cause | Resolution |
+|---------|-------------|------------|
+| No events, connection OK | ID mismatch bug regression | Review subscription code |
+| Connection drops | Network/provider issue | Check provider status |
+| Events stop after N hours | Connection timeout | Verify ping/keepalive |
+| Subscriptions = 0 | Initialization failure | Check startup logs |
+| High CPU, no events | Message handling loop | Check for infinite loops |
+
+---
+
+## Conclusion
+
+The fail-fast system provides multiple layers of protection:
+
+1. **Real-time detection** via health checks every minute
+2. **Automatic recovery** via container restarts
+3. **Visibility** via health endpoints and Prometheus metrics
+4. **Alerting** for human intervention when auto-recovery fails
+
+**Remember**: A service that appears healthy but doesn't process events is worse than a service that crashes. Fail fast, fail visibly, and let orchestration handle recovery.