@avalw/search-worker 1.1.0 → 2.1.0

package/index.js

@@ -0,0 +1,887 @@
+/**
+ * @avalw/search-worker v2.1.0
+ * Distributed Cache Worker for AVALW Search Network
+ *
+ * Workers connect to the coordinator via WebSocket and serve as
+ * distributed cache nodes, reducing load on AVALW servers.
+ *
+ * Features:
+ * - Intelligent cache with size limits (500MB - 2GB)
+ * - Time-based eviction (7+ days unused = auto-remove)
+ * - Popularity scoring (frequently accessed stay longer)
+ * - Background auto-cleanup
+ */
+
+const os = require('os');
+const WebSocket = require('ws');
+const https = require('https');
+const http = require('http');
+
+// Intelligent Cache implementation
+// - Size-based limits (bytes)
+// - Time-based eviction (configurable max age)
+// - Popularity scoring (access frequency)
+// - Auto-cleanup background process
+class SmartCache {
+  constructor(options = {}) {
+    this.cache = new Map();
+
+    // Size limits (in bytes)
+    this.maxSizeBytes = options.maxSizeBytes || 500 * 1024 * 1024; // Default 500MB
+    this.currentSizeBytes = 0;
+
+    // Count limits (fallback)
+    this.maxEntries = options.maxEntries || 10000;
+
+    // Time-based eviction (default 7 days)
+    this.maxAgeDays = options.maxAgeDays || 7;
+    this.maxAgeMs = this.maxAgeDays * 24 * 60 * 60 * 1000;
+
+    // Default TTL for new entries (5 minutes minimum cache)
+    this.defaultTtlMs = options.defaultTtlMs || 5 * 60 * 1000;
+
+    // Cleanup interval (default every 1 hour)
+    this.cleanupIntervalMs = options.cleanupIntervalMs || 60 * 60 * 1000;
+    this.cleanupTimer = null;
+
+    // Stats
+    this.stats = {
+      hits: 0,
+      misses: 0,
+      evictions: 0,
+      sizeEvictions: 0,
+      timeEvictions: 0,
+      popularityEvictions: 0
+    };
+
+    // Callbacks
+    this.onEvict = options.onEvict || (() => {});
+    this.onCleanup = options.onCleanup || (() => {});
+
+    // Start cleanup process
+    this._startCleanup();
+  }
+
+  // Calculate byte size of a value
+  _getByteSize(value) {
+    if (typeof value === 'string') {
+      return Buffer.byteLength(value, 'utf8');
+    }
+    return Buffer.byteLength(JSON.stringify(value), 'utf8');
+  }
+
+  // Get item from cache
+  get(key) {
+    const item = this.cache.get(key);
+    if (!item) {
+      this.stats.misses++;
+      return null;
+    }
+
+    const now = Date.now();
+
+    // Check TTL expiry
+    if (item.expiry && now > item.expiry) {
+      this._evict(key, 'ttl');
+      this.stats.misses++;
+      return null;
+    }
+
+    // Check max age (time-based eviction)
+    const ageMs = now - item.createdAt;
+    if (ageMs > this.maxAgeMs) {
+      this._evict(key, 'age');
+      this.stats.misses++;
+      return null;
+    }
+
+    // Update access stats
+    item.lastAccess = now;
+    item.accessCount++;
+    item.popularityScore = this._calculatePopularity(item);
+
+    this.stats.hits++;
+    return item.value;
+  }
+
+  // Set item in cache
+  set(key, value, ttl = null) {
+    const now = Date.now();
+    const byteSize = this._getByteSize(value);
+    const evictedKeys = [];
+
+    // Check if value is too large (max 10% of total cache)
+    if (byteSize > this.maxSizeBytes * 0.1) {
+      console.warn(`[SmartCache] Value too large (${this._formatBytes(byteSize)}), skipping`);
+      return evictedKeys;
+    }
+
+    // Check if key already exists
+    if (this.cache.has(key)) {
+      const existing = this.cache.get(key);
+      this.currentSizeBytes -= existing.byteSize;
+      // Update existing entry
+      existing.value = value;
+      existing.byteSize = byteSize;
+      existing.lastAccess = now;
+      existing.accessCount++;
+      existing.expiry = now + (ttl || this.defaultTtlMs);
+      existing.popularityScore = this._calculatePopularity(existing);
+      this.currentSizeBytes += byteSize;
+      return evictedKeys;
+    }
+
+    // Evict items if necessary to make space
+    while (
+      (this.currentSizeBytes + byteSize > this.maxSizeBytes ||
+        this.cache.size >= this.maxEntries) &&
+      this.cache.size > 0
+    ) {
+      const evictedKey = this._evictLowestPriority();
+      if (evictedKey) {
+        evictedKeys.push(evictedKey);
+      } else {
+        break; // No more items to evict
+      }
+    }
+
+    // Store new item
+    this.cache.set(key, {
+      value,
+      byteSize,
+      createdAt: now,
+      lastAccess: now,
+      accessCount: 1,
+      popularityScore: 1,
+      expiry: now + (ttl || this.defaultTtlMs)
+    });
+
+    this.currentSizeBytes += byteSize;
+    return evictedKeys;
+  }
+
+  // Check if key exists
+  has(key) {
+    const item = this.cache.get(key);
+    if (!item) return false;
+
+    const now = Date.now();
+
+    // Check expiry
+    if (item.expiry && now > item.expiry) {
+      this._evict(key, 'ttl');
+      return false;
+    }
+
+    // Check max age
+    if (now - item.createdAt > this.maxAgeMs) {
+      this._evict(key, 'age');
+      return false;
+    }
+
+    return true;
+  }
+
+  // Get all valid keys
+  keys() {
+    const validKeys = [];
+    const now = Date.now();
+
+    for (const [key, item] of this.cache.entries()) {
+      // Skip expired
+      if (item.expiry && now > item.expiry) {
+        this._evict(key, 'ttl');
+        continue;
+      }
+      // Skip too old
+      if (now - item.createdAt > this.maxAgeMs) {
+        this._evict(key, 'age');
+        continue;
+      }
+      validKeys.push(key);
+    }
+
+    return validKeys;
+  }
+
+  // Get entry count
+  size() {
+    return this.cache.size;
+  }
+
+  // Get current size in bytes
+  sizeBytes() {
+    return this.currentSizeBytes;
+  }
+
+  // Get formatted size
+  getSizeFormatted() {
+    return this._formatBytes(this.currentSizeBytes);
+  }
+
+  // Clear all entries
+  clear() {
+    this.cache.clear();
+    this.currentSizeBytes = 0;
+  }
+
+  // Get cache stats
+  getStats() {
+    return {
+      entries: this.cache.size,
+      sizeBytes: this.currentSizeBytes,
+      sizeFormatted: this._formatBytes(this.currentSizeBytes),
+      maxSizeBytes: this.maxSizeBytes,
+      maxSizeFormatted: this._formatBytes(this.maxSizeBytes),
+      utilizationPercent: Math.round((this.currentSizeBytes / this.maxSizeBytes) * 100 * 10) / 10,
+      maxAgeDays: this.maxAgeDays,
+      ...this.stats,
+      hitRate: this.stats.hits + this.stats.misses > 0
+        ? Math.round((this.stats.hits / (this.stats.hits + this.stats.misses)) * 100)
+        : 0
+    };
+  }
+
+  // Calculate popularity score
+  // Higher score = more valuable to keep
+  _calculatePopularity(item) {
+    const now = Date.now();
+    const ageHours = (now - item.createdAt) / (1000 * 60 * 60);
+    const recentHours = (now - item.lastAccess) / (1000 * 60 * 60);
+
+    // Factors:
+    // - Access count (more accesses = more popular)
+    // - Recency (recently accessed = more valuable)
+    // - Age (older items with high access = stable popularity)
+
+    const accessFactor = Math.log2(item.accessCount + 1); // Logarithmic growth
+    const recencyFactor = Math.max(0, 1 - recentHours / (24 * 7)); // Decay over 7 days
+    const stabilityFactor = ageHours > 1 ? Math.log2(item.accessCount / ageHours + 1) : 1;
+
+    return accessFactor * recencyFactor * stabilityFactor;
+  }
+
+  // Evict a specific key
+  _evict(key, reason = 'manual') {
+    const item = this.cache.get(key);
+    if (!item) return false;
+
+    this.currentSizeBytes -= item.byteSize;
+    this.cache.delete(key);
+
+    this.stats.evictions++;
+    if (reason === 'size') this.stats.sizeEvictions++;
+    if (reason === 'age' || reason === 'ttl') this.stats.timeEvictions++;
+    if (reason === 'popularity') this.stats.popularityEvictions++;
+
+    this.onEvict(key, reason);
+    return true;
+  }
+
+  // Evict the lowest priority item
+  _evictLowestPriority() {
+    if (this.cache.size === 0) return null;
+
+    const now = Date.now();
+    let lowestKey = null;
+    let lowestScore = Infinity;
+
+    for (const [key, item] of this.cache.entries()) {
+      // First check for expired items
+      if (item.expiry && now > item.expiry) {
+        this._evict(key, 'ttl');
+        return key;
+      }
+
+      // Check for items past max age
+      if (now - item.createdAt > this.maxAgeMs) {
+        this._evict(key, 'age');
+        return key;
+      }
+
+      // Calculate eviction priority (lower = evict first)
+      const score = item.popularityScore;
+      if (score < lowestScore) {
+        lowestScore = score;
+        lowestKey = key;
+      }
+    }
+
+    if (lowestKey) {
+      this._evict(lowestKey, 'popularity');
+      return lowestKey;
+    }
+
+    return null;
+  }
+
+  // Background cleanup process
+  _startCleanup() {
+    if (this.cleanupTimer) return;
+
+    this.cleanupTimer = setInterval(() => {
+      this._runCleanup();
+    }, this.cleanupIntervalMs);
+  }
+
+  // Stop cleanup process
+  stopCleanup() {
+    if (this.cleanupTimer) {
+      clearInterval(this.cleanupTimer);
+      this.cleanupTimer = null;
+    }
+  }
+
+  // Run cleanup - remove expired and aged entries
+  _runCleanup() {
+    const now = Date.now();
+    const toRemove = [];
+
+    for (const [key, item] of this.cache.entries()) {
+      // Remove expired
+      if (item.expiry && now > item.expiry) {
+        toRemove.push({ key, reason: 'ttl' });
+        continue;
+      }
+
+      // Remove items not accessed for maxAge days
+      const sinceLastAccess = now - item.lastAccess;
+      if (sinceLastAccess > this.maxAgeMs) {
+        toRemove.push({ key, reason: 'age' });
+        continue;
+      }
+
+      // Update popularity scores during cleanup
+      item.popularityScore = this._calculatePopularity(item);
+    }
+
+    // Evict marked entries
+    for (const { key, reason } of toRemove) {
+      this._evict(key, reason);
+    }
+
+    this.onCleanup(toRemove.length);
+
+    // Log cleanup results
+    if (toRemove.length > 0) {
+      console.log(`[SmartCache] Cleanup: removed ${toRemove.length} stale entries, ${this._formatBytes(this.currentSizeBytes)} / ${this._formatBytes(this.maxSizeBytes)} used`);
+    }
+  }
+
+  // Format bytes to human readable
+  _formatBytes(bytes) {
+    if (bytes === 0) return '0 B';
+    const k = 1024;
+    const sizes = ['B', 'KB', 'MB', 'GB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+  }
+
+  // Set max size (for runtime configuration)
+  setMaxSize(sizeBytes) {
+    this.maxSizeBytes = Math.max(100 * 1024 * 1024, Math.min(sizeBytes, 2 * 1024 * 1024 * 1024)); // 100MB - 2GB
+
+    // Evict if over new limit
+    while (this.currentSizeBytes > this.maxSizeBytes && this.cache.size > 0) {
+      this._evictLowestPriority();
+    }
+  }
+
+  // Set max age in days (for runtime configuration)
+  setMaxAgeDays(days) {
+    this.maxAgeDays = Math.max(1, Math.min(days, 30)); // 1 - 30 days
+    this.maxAgeMs = this.maxAgeDays * 24 * 60 * 60 * 1000;
+
+    // Run cleanup to evict newly expired items
+    this._runCleanup();
+  }
+}
+
+// Legacy LRU Cache for backward compatibility
+class LRUCache extends SmartCache {
+  constructor(maxSize = 500, ttlMs = 5 * 60 * 1000) {
+    super({
+      maxEntries: maxSize,
+      defaultTtlMs: ttlMs,
+      maxSizeBytes: 500 * 1024 * 1024 // 500MB default
+    });
+    this.maxSize = maxSize;
+    this.ttlMs = ttlMs;
+  }
+}
+
+class SearchWorker {
+  constructor(options = {}) {
+    this.workerToken = options.workerToken || process.env.AVALW_WORKER_TOKEN;
+    this.dashboardUrl = options.dashboardUrl || 'https://worker.avalw.org';
+    this.wsUrl = options.wsUrl || 'wss://worker.avalw.org/ws/cache';
+
+    // Worker state
+    this.workerId = null;
+    this.isConnected = false;
+    this.ws = null;
+    this.startTime = Date.now();
+    this.heartbeatInterval = null;
+    this.reconnectTimeout = null;
+    this.reconnectAttempts = 0;
+    this.maxReconnectAttempts = 10;
+
+    // Smart cache configuration
+    // Size: 500MB - 2GB (default 500MB)
+    // Max age: 1-30 days (default 7 days for unused entries)
+    const cacheSizeMB = options.cacheSizeMB || 500;
+    const maxAgeDays = options.maxAgeDays || 7;
+
+    // Local cache for distributed caching
+    this.cache = new SmartCache({
+      maxSizeBytes: Math.min(Math.max(cacheSizeMB, 100), 2048) * 1024 * 1024, // 100MB - 2GB
+      maxAgeDays: Math.min(Math.max(maxAgeDays, 1), 30), // 1 - 30 days
+      maxEntries: options.maxCacheEntries || 10000,
+      defaultTtlMs: options.cacheTtl || 5 * 60 * 1000,
+      cleanupIntervalMs: options.cleanupIntervalMs || 60 * 60 * 1000, // 1 hour
+      onEvict: (key, reason) => {
+        // Notify coordinator about evicted key
+        if (this.isConnected) {
+          this._send({ type: 'cache_evicted', key, reason });
+        }
+      }
+    });
+
+    // Stats
+    this.stats = {
+      cacheHits: 0,
+      cacheMisses: 0,
+      cacheStored: 0,
+      requestsServed: 0
+    };
+
+    // Callbacks
+    this.onConnected = options.onConnected || (() => {});
+    this.onDisconnected = options.onDisconnected || (() => {});
+    this.onCacheHit = options.onCacheHit || (() => {});
+    this.onCacheStore = options.onCacheStore || (() => {});
+    this.onError = options.onError || ((err) => console.error('[Worker] Error:', err));
+    this.onLog = options.onLog || ((msg) => console.log('[Worker]', msg));
+  }
+
+  // Get system information
+  getSystemInfo() {
+    const cpus = os.cpus();
+    const totalMemory = os.totalmem();
+    const freeMemory = os.freemem();
+    const cpuSpeed = cpus[0]?.speed || 2400;
+    const speedFactor = cpuSpeed / 3000; // Normalized to 3GHz
+
+    return {
+      platform: os.platform(),
+      arch: os.arch(),
+      cores: cpus.length,
+      cpu_model: cpus[0]?.model || 'Unknown',
+      cpu_speed_ghz: cpuSpeed / 1000,
+      speed_factor: Math.round(speedFactor * 100) / 100,
+      total_memory_gb: Math.round(totalMemory / (1024 * 1024 * 1024) * 10) / 10,
+      free_memory_gb: Math.round(freeMemory / (1024 * 1024 * 1024) * 10) / 10,
+      worker_uptime_hours: Math.round((Date.now() - this.startTime) / (1000 * 60 * 60) * 100) / 100,
+      cache_size: this.cache.size()
+    };
+  }
+
+  // Calculate current CPU usage
+  async getCpuUsage() {
+    return new Promise((resolve) => {
+      const startMeasure = this._cpuAverage();
+      setTimeout(() => {
+        const endMeasure = this._cpuAverage();
+        const idleDiff = endMeasure.idle - startMeasure.idle;
+        const totalDiff = endMeasure.total - startMeasure.total;
+        const usage = 100 - (100 * idleDiff / totalDiff);
+        resolve(Math.round(usage * 10) / 10);
+      }, 100);
+    });
+  }
+
+  _cpuAverage() {
+    const cpus = os.cpus();
+    let totalIdle = 0, totalTick = 0;
+    for (const cpu of cpus) {
+      for (const type in cpu.times) {
+        totalTick += cpu.times[type];
+      }
+      totalIdle += cpu.times.idle;
+    }
+    return { idle: totalIdle / cpus.length, total: totalTick / cpus.length };
+  }
+
+  // Connect to WebSocket coordinator
+  connect() {
+    if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+      this.onLog('Already connected to coordinator');
+      return;
+    }
+
+    this.onLog(`Connecting to ${this.wsUrl}...`);
+
+    try {
+      this.ws = new WebSocket(this.wsUrl);
+
+      this.ws.on('open', () => {
+        this.onLog('Connected to coordinator');
+        this.isConnected = true;
+        this.reconnectAttempts = 0;
+
+        // Register cached keys
+        this._sendCacheRegistry();
+
+        // Start heartbeat
+        this._startHeartbeat();
+
+        this.onConnected(this);
+      });
+
+      this.ws.on('message', (data) => {
+        try {
+          const message = JSON.parse(data.toString());
+          this._handleMessage(message);
+        } catch (e) {
+          this.onError(`Invalid message: ${e.message}`);
+        }
+      });
+
+      this.ws.on('close', () => {
+        this.onLog('Disconnected from coordinator');
+        this.isConnected = false;
+        this._stopHeartbeat();
+        this.onDisconnected(this);
+        this._scheduleReconnect();
+      });
+
+      this.ws.on('error', (error) => {
+        this.onError(`WebSocket error: ${error.message}`);
+      });
+
+    } catch (e) {
+      this.onError(`Connection failed: ${e.message}`);
+      this._scheduleReconnect();
+    }
+  }
+
+  // Handle incoming messages from coordinator
+  _handleMessage(message) {
+    switch (message.type) {
+      case 'welcome':
+        this.workerId = message.workerId;
+        this.onLog(`Registered as worker ${this.workerId.substring(0, 12)}...`);
+        if (message.config) {
+          // Update cache config from server
+          if (message.config.maxSizeMB) {
+            this.cache.setMaxSize(message.config.maxSizeMB * 1024 * 1024);
+          }
+          if (message.config.maxAgeDays) {
+            this.cache.setMaxAgeDays(message.config.maxAgeDays);
+          }
+        }
+        break;
+
+      case 'cache_get':
+        // Coordinator asking for a cached value
+        this._handleCacheGet(message.requestId, message.key);
+        break;
+
+      case 'cache_set':
+        // Coordinator sending a value to cache
+        this._handleCacheSet(message.key, message.value, message.ttl);
+        break;
+
+      case 'cache_invalidate':
+        // Invalidate a cache key
+        if (message.key) {
+          this.cache._evict(message.key, 'invalidate');
+          this.onLog(`Cache invalidated: ${message.key.substring(0, 30)}...`);
+        }
+        break;
+
+      case 'config_update':
+        // Update cache configuration at runtime
+        if (message.maxSizeMB) {
+          this.cache.setMaxSize(message.maxSizeMB * 1024 * 1024);
+          this.onLog(`Cache size limit updated to ${message.maxSizeMB}MB`);
+        }
+        if (message.maxAgeDays) {
+          this.cache.setMaxAgeDays(message.maxAgeDays);
+          this.onLog(`Cache max age updated to ${message.maxAgeDays} days`);
+        }
+        break;
+
+      case 'ping':
+        this._send({ type: 'pong' });
+        break;
+
+      default:
+        this.onLog(`Unknown message type: ${message.type}`);
+    }
+  }
+
+  // Handle cache get request from coordinator
+  _handleCacheGet(requestId, key) {
+    const value = this.cache.get(key);
+
+    if (value) {
+      this.stats.cacheHits++;
+      this.stats.requestsServed++;
+      this.onCacheHit(key);
+
+      this._send({
+        type: 'cache_response',
+        requestId,
+        hit: true,
+        value
+      });
+
+      this.onLog(`Cache HIT: ${key.substring(0, 30)}...`);
+    } else {
+      this.stats.cacheMisses++;
+
+      this._send({
+        type: 'cache_response',
+        requestId,
+        hit: false
+      });
+    }
+  }
+
+  // Handle cache set from coordinator
+  _handleCacheSet(key, value, ttl) {
+    const evictedKeys = this.cache.set(key, value, ttl);
+
+    // Notify coordinator about stored key
+    this._send({
+      type: 'cache_stored',
+      key,
+      cacheStats: this.cache.getStats()
+    });
+
+    // Notify coordinator about evicted keys (already sent via onEvict callback)
+    // The SmartCache handles this via the onEvict callback
+
+    this.stats.cacheStored++;
+    this.onCacheStore(key);
+
+    const cacheStats = this.cache.getStats();
+    this.onLog(`Cached: ${key.substring(0, 30)}... (${cacheStats.entries} entries, ${cacheStats.sizeFormatted})`);
+  }
+
+  // Send cache registry to coordinator
+  _sendCacheRegistry() {
+    const keys = this.cache.keys();
+    this._send({
+      type: 'register_cache',
+      keys
+    });
+    this.onLog(`Registered ${keys.length} cached keys with coordinator`);
+  }
+
+  // Send message to coordinator
+  _send(message) {
+    if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify(message));
+    }
+  }
+
+  // Start heartbeat interval
+  _startHeartbeat() {
+    this._stopHeartbeat();
+    this.heartbeatInterval = setInterval(async () => {
+      await this._sendHeartbeat();
+    }, 30000); // 30 seconds
+
+    // Send initial heartbeat
+    this._sendHeartbeat();
+  }
+
+  // Stop heartbeat
+  _stopHeartbeat() {
+    if (this.heartbeatInterval) {
+      clearInterval(this.heartbeatInterval);
+      this.heartbeatInterval = null;
+    }
+  }
+
+  // Send heartbeat to coordinator
+  async _sendHeartbeat() {
+    const cpuUsage = await this.getCpuUsage();
+    const systemInfo = this.getSystemInfo();
+
+    // Send to WebSocket coordinator
+    this._send({
+      type: 'heartbeat',
+      cacheSize: this.cache.size(),
+      stats: this.stats,
+      cpu_usage: cpuUsage
+    });
+
+    // Also send to HTTP dashboard for tier tracking
+    if (this.workerToken) {
+      this._sendHttpHeartbeat(cpuUsage, systemInfo);
+    }
+  }
+
+  // Send heartbeat to HTTP dashboard
+  _sendHttpHeartbeat(cpuUsage, systemInfo) {
+    const data = JSON.stringify({
+      worker_token: this.workerToken,
+      system_info: {
+        ...systemInfo,
+        cpu_usage: cpuUsage,
+        memory_usage: Math.round((1 - os.freemem() / os.totalmem()) * 100 * 10) / 10,
+        session_core_hours: this._calculateCoreHours()
+      }
+    });
+
+    const url = new URL('/api/worker/heartbeat', this.dashboardUrl);
+    const isHttps = url.protocol === 'https:';
+
+    const options = {
+      hostname: url.hostname,
+      port: url.port || (isHttps ? 443 : 80),
+      path: url.pathname,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(data)
+      }
+    };
+
+    const req = (isHttps ? https : http).request(options, (res) => {
+      let body = '';
+      res.on('data', chunk => body += chunk);
+      res.on('end', () => {
+        try {
+          const response = JSON.parse(body);
+          if (response.tier) {
+            this.currentTier = response.tier;
+          }
+        } catch (e) {
+          // Ignore parse errors
+        }
+      });
+    });
+
+    req.on('error', () => {
+      // Silent fail on heartbeat errors
+    });
+
+    req.write(data);
+    req.end();
+  }
+
+  // Calculate core-hours for this session
+  _calculateCoreHours() {
+    const cpus = os.cpus();
+    const cores = cpus.length;
+    const cpuSpeed = cpus[0]?.speed || 2400;
+    const speedFactor = cpuSpeed / 3000;
+    const uptimeHours = (Date.now() - this.startTime) / (1000 * 60 * 60);
+    const avgUsage = 0.5; // Assume 50% average usage
+
+    return cores * speedFactor * avgUsage * uptimeHours;
+  }
+
+  // Schedule reconnection
+  _scheduleReconnect() {
+    if (this.reconnectTimeout) {
+      clearTimeout(this.reconnectTimeout);
+    }
+
+    if (this.reconnectAttempts >= this.maxReconnectAttempts) {
+      this.onError('Max reconnection attempts reached');
+      return;
+    }
+
+    const delay = Math.min(1000 * Math.pow(2, this.reconnectAttempts), 30000);
+    this.reconnectAttempts++;
+
+    this.onLog(`Reconnecting in ${delay / 1000}s (attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})`);
+
+    this.reconnectTimeout = setTimeout(() => {
+      this.connect();
+    }, delay);
+  }
+
+  // Disconnect from coordinator
+  disconnect() {
+    this._stopHeartbeat();
+
+    // Stop cache cleanup process
+    this.cache.stopCleanup();
+
+    if (this.reconnectTimeout) {
+      clearTimeout(this.reconnectTimeout);
+      this.reconnectTimeout = null;
+    }
+
+    if (this.ws) {
+      this.ws.close();
+      this.ws = null;
+    }
+
+    this.isConnected = false;
+    this.onLog('Disconnected');
+  }
+
+  // Get worker stats
+  getStats() {
+    const cacheStats = this.cache.getStats();
+    return {
+      workerId: this.workerId,
+      isConnected: this.isConnected,
+      uptime: Date.now() - this.startTime,
+      // Worker-level stats
+      cacheHits: this.stats.cacheHits,
+      cacheMisses: this.stats.cacheMisses,
+      cacheStored: this.stats.cacheStored,
+      requestsServed: this.stats.requestsServed,
+      hitRate: this.stats.cacheHits + this.stats.cacheMisses > 0
+        ? Math.round((this.stats.cacheHits / (this.stats.cacheHits + this.stats.cacheMisses)) * 100)
+        : 0,
+      // Cache-level stats
+      cache: cacheStats
+    };
+  }
+
+  // Get cache configuration
+  getCacheConfig() {
+    return {
+      maxSizeBytes: this.cache.maxSizeBytes,
+      maxSizeFormatted: this.cache._formatBytes(this.cache.maxSizeBytes),
+      maxAgeDays: this.cache.maxAgeDays,
+      currentSizeBytes: this.cache.currentSizeBytes,
+      currentSizeFormatted: this.cache._formatBytes(this.cache.currentSizeBytes),
+      entries: this.cache.size(),
+      utilizationPercent: Math.round((this.cache.currentSizeBytes / this.cache.maxSizeBytes) * 100 * 10) / 10
+    };
+  }
+
+  // Set cache size limit (MB)
+  setCacheSize(sizeMB) {
+    this.cache.setMaxSize(sizeMB * 1024 * 1024);
+    this.onLog(`Cache size limit set to ${sizeMB}MB`);
+  }
+
+  // Set cache max age (days)
+  setCacheMaxAge(days) {
+    this.cache.setMaxAgeDays(days);
+    this.onLog(`Cache max age set to ${days} days`);
+  }
+}
+
+// Factory function
+function createWorker(options) {
+  return new SearchWorker(options);
+}
+
+module.exports = {
+  SearchWorker,
+  SmartCache,
+  LRUCache, // Backward compatible
+  createWorker
+};