@cgh567/agent 2.4.1 → 2.4.2

package/daemon/lib/headroom-proxy-manager.js

@@ -1,623 +0,0 @@
-'use strict';
-
-/**
- * headroom-proxy-manager.js
- *
- * Manages the Headroom compression proxy sidecar process.
- * The proxy runs locally on the machine (Windows or macOS) and intercepts
- * all LLM traffic, compressing tool outputs, logs, HEMA recall payloads,
- * and HBO API responses before they reach the model.
- *
- * Design principles:
- *   - Required service, not optional. If the proxy fails to start, the daemon
- *     exits (same as verifyMemgraphConfig). Correct operation > silent degradation.
- *   - Resilient to mid-session crashes via auto-restart with exponential backoff.
- *     A crash costs 2–5 seconds, not a full daemon restart.
- *   - Windows-native paths throughout. No WSL, no Unix-only APIs.
- *   - Singleton pattern: HeadroomProxyManager.getInstance() returns the one instance.
- *
- * Failure modes handled:
- *   - Python not on PATH → hard exit with install instructions
- *   - headroom package not installed → hard exit with install instructions
- *   - Proxy takes > 60s to start → hard exit
- *   - Proxy crashes mid-session → auto-restart (max 5 retries, exponential backoff)
- *   - Port conflict → finds next available port automatically
- *   - Compression pipeline broken (verifyProxy fails) → hard exit
- *
- * Lock file: daemon/headroom.lock { port, pid, startedAt }
- *   Same pattern as daemon.lock written by helios-api.js.
- *   HeliosInfraService reads this to expose proxy status to the UI.
- */
-
-const os          = require('os');
-const path        = require('path');
-const fs          = require('fs');
-const net         = require('net');
-const http        = require('http');
-const { spawn }   = require('child_process');
-const { execFileSync } = require('child_process');
-
-// ── Constants ────────────────────────────────────────────────────────────────
-
-const HELIOS_ROOT        = process.env.HELIOS_ROOT || path.resolve(__dirname, '..', '..');
-const LOCK_FILE          = path.join(HELIOS_ROOT, 'daemon', 'headroom.lock');
-const LOCK_TMP           = path.join(HELIOS_ROOT, 'daemon', '.headroom.lock.tmp');
-const LOG_FILE           = path.join(HELIOS_ROOT, 'daemon', 'headroom-proxy.log');
-const VENDOR_PATH        = path.join(HELIOS_ROOT, 'vendor', 'headroom-proxy');
-const DEFAULT_PORT       = 8787;
-const STARTUP_TIMEOUT_MS = 60_000;   // 60s to start (pip install of ML deps can be slow)
-const HEALTH_POLL_MS     = 500;      // poll health every 500ms during startup
-const VERIFY_TIMEOUT_MS  = 10_000;   // 10s for compression verify
-const MAX_RESTARTS       = 5;        // max auto-restarts before giving up
-const RESTART_BACKOFF_MS = [1000, 2000, 4000, 8000, 15000]; // per-attempt delay
-const HEALTH_CHECK_MS    = 15_000;   // check proxy health every 15s after startup
-
-// ── Logging (mirrors daemon log pattern) ────────────────────────────────────
-
-function log(level, msg) {
-  const ts = new Date().toISOString();
-  process.stdout.write(`${ts} [headroom-proxy-manager] [${level.toUpperCase()}] ${msg}\n`);
-}
-
-// ── Port utilities ───────────────────────────────────────────────────────────
-
-/**
- * Check if a TCP port is available on 127.0.0.1.
- * Returns true if available (nothing listening), false if in use.
- */
-function isPortAvailable(port) {
-  return new Promise((resolve) => {
-    const server = net.createServer();
-    server.once('error', () => resolve(false));
-    server.once('listening', () => {
-      server.close(() => resolve(true));
-    });
-    server.listen(port, '127.0.0.1');
-  });
-}
-
-/**
- * Find an available port starting from DEFAULT_PORT.
- * Tries DEFAULT_PORT, then DEFAULT_PORT+1 through DEFAULT_PORT+9.
- */
-async function findAvailablePort() {
-  for (let p = DEFAULT_PORT; p < DEFAULT_PORT + 10; p++) {
-    if (await isPortAvailable(p)) return p;
-  }
-  throw new Error(`No available port found in range ${DEFAULT_PORT}–${DEFAULT_PORT + 9}`);
-}
-
-// ── Python resolution ────────────────────────────────────────────────────────
-
-/**
- * Resolve the Python 3.10+ binary path.
- * Windows: tries py launcher, python3, python — in that order.
- * macOS/Linux: tries python3, python.
- * Returns the binary name/path, or throws with install instructions.
- */
-function resolvePythonBin() {
-  // Allow explicit override for non-standard installs
-  if (process.env.HEADROOM_PYTHON) {
-    return process.env.HEADROOM_PYTHON;
-  }
-
-  const isWindows = process.platform === 'win32';
-
-  // Candidates in priority order
-  const candidates = isWindows
-    ? ['py', 'python3', 'python']
-    : ['python3', 'python'];
-
-  for (const bin of candidates) {
-    try {
-      const out = execFileSync(bin, ['--version'], {
-        timeout: 5000,
-        stdio: ['ignore', 'pipe', 'pipe'],
-        windowsHide: true,
-      }).toString().trim();
-
-      // Verify it's Python 3.10+
-      const m = out.match(/Python (\d+)\.(\d+)/);
-      if (m && (parseInt(m[1]) > 3 || (parseInt(m[1]) === 3 && parseInt(m[2]) >= 10))) {
-        log('info', `Found Python: ${bin} (${out})`);
-        return bin;
-      }
-      log('warn', `${bin} found but version too old: ${out} (need 3.10+)`);
-    } catch (_) {
-      // not on PATH, try next
-    }
-  }
-
-  const installUrl = isWindows
-    ? 'https://www.python.org/downloads/windows/'
-    : 'https://www.python.org/downloads/';
-
-  throw new Error(
-    `Python 3.10+ is required for the Helios compression pipeline.\n` +
-    `Install from: ${installUrl}\n` +
-    `Or set HEADROOM_PYTHON=/path/to/python3 in your environment.\n` +
-    `On Windows you can also run: winget install Python.Python.3.12`
-  );
-}
-
-/**
- * Verify headroom is importable from the vendor path.
- * Throws with pip install instructions if not found.
- */
-function verifyHeadroomInstalled(pythonBin) {
-  try {
-    execFileSync(pythonBin, [
-      '-c',
-      `import sys; sys.path.insert(0, r'${VENDOR_PATH}'); import headroom; print(headroom.__version__)`,
-    ], {
-      timeout: 10000,
-      stdio: ['ignore', 'pipe', 'pipe'],
-      windowsHide: true,
-    });
-    log('info', 'headroom package verified in vendor path');
-  } catch (err) {
-    const out = err.stderr ? err.stderr.toString() : err.message;
-    throw new Error(
-      `headroom package not installed in vendor directory.\n` +
-      `Run this command to install it:\n` +
-      `  cd "${VENDOR_PATH}" && pip install -e ".[proxy,ml,ccr,memory]" --quiet\n` +
-      `Or: pip install headroom-ai[proxy,ml,ccr,memory] --target "${VENDOR_PATH}"\n` +
-      `Error: ${out.slice(0, 200)}`
-    );
-  }
-}
-
-// ── HTTP health probe ────────────────────────────────────────────────────────
-
-/**
- * Probe GET http://127.0.0.1:{port}/health.
- * Returns true if proxy responds with any 2xx status.
- * timeoutMs: per-request timeout in ms.
- */
-function probeHealth(port, timeoutMs = 3000) {
-  return new Promise((resolve) => {
-    const req = http.request(
-      { hostname: '127.0.0.1', port, path: '/health', method: 'GET' },
-      (res) => {
-        res.resume(); // drain response body
-        resolve(res.statusCode >= 200 && res.statusCode < 300);
-      }
-    );
-    req.setTimeout(timeoutMs, () => { req.destroy(); resolve(false); });
-    req.on('error', () => resolve(false));
-    req.end();
-  });
-}
-
-/**
- * Poll health until proxy is up or deadline is exceeded.
- * Returns true if proxy became healthy, false if deadline expired.
- */
-async function waitForHealth(port, deadlineMs) {
-  const start = Date.now();
-  while (Date.now() - start < deadlineMs) {
-    if (await probeHealth(port, 2000)) return true;
-    await new Promise(r => setTimeout(r, HEALTH_POLL_MS));
-  }
-  return false;
-}
-
-// ── Compression verification ─────────────────────────────────────────────────
-
-/**
- * Verify the compression pipeline is functional by sending a test payload.
- * This is the headroom equivalent of verifyMemgraphConfig:
- * fail fast if the measurement/compression path itself is broken.
- *
- * What this checks (mirrors memgraph-verify.js design):
- *   - Proxy responds to compress requests
- *   - Compression actually reduces token count (not a passthrough)
- *   - Response time is within bounds
- *
- * What this does NOT check (not a hard-exit condition):
- *   - Compression ratio for specific content types
- *   - TOIN pattern counts
- *   - CCR store state
- */
-async function verifyCompressionPipeline(port) {
-  // Build a test payload that SmartCrusher should reduce significantly:
-  // 50-item array of objects — exactly the HBO signal/lead array pattern
-  const testItems = Array.from({ length: 50 }, (_, i) => ({
-    id: `item-${i}`,
-    name: `Test Item ${i}`,
-    status: i % 3 === 0 ? 'active' : 'pending',
-    score: Math.random(),
-    tags: ['a', 'b', 'c'],
-    metadata: { createdAt: new Date().toISOString(), source: 'test' },
-  }));
-
-  const testPayload = JSON.stringify({
-    model: 'claude-3-5-haiku-20241022',
-    messages: [{
-      role: 'user',
-      content: [{
-        type: 'tool_result',
-        tool_use_id: 'verify_test',
-        content: JSON.stringify(testItems),
-      }],
-    }],
-  });
-
-  return new Promise((resolve, reject) => {
-    const timer = setTimeout(() => {
-      req.destroy();
-      reject(new Error(`Compression verify timed out after ${VERIFY_TIMEOUT_MS}ms`));
-    }, VERIFY_TIMEOUT_MS);
-
-    const req = http.request(
-      {
-        hostname: '127.0.0.1',
-        port,
-        path: '/v1/messages',
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          'Content-Length': Buffer.byteLength(testPayload),
-          'x-api-key': 'headroom-verify-test',
-          'anthropic-version': '2023-06-01',
-          'x-headroom-verify': '1',  // tells proxy to return compression stats, not call LLM
-        },
-      },
-      (res) => {
-        clearTimeout(timer);
-        let body = '';
-        res.on('data', chunk => { body += chunk; });
-        res.on('end', () => {
-          // Proxy returns 200 with x-headroom-saved header even for verify calls
-          const saved = res.headers['x-headroom-saved'];
-          const ratio = res.headers['x-headroom-ratio'];
-
-          // Acceptable conditions:
-          // 1. Proxy returned tokens-saved header with positive value → compression working
-          // 2. Proxy returned 200 → pipeline didn't crash
-          // We do NOT require a minimum compression ratio here — new installs may have
-          // cold caches. We only fail if the pipeline itself throws.
-          if (res.statusCode === 200 || res.statusCode === 422) {
-            // 422 = proxy couldn't call LLM (expected for verify key) but compression ran
-            log('info', `Compression pipeline verified: status=${res.statusCode} saved=${saved ?? 'n/a'} ratio=${ratio ?? 'n/a'}`);
-            resolve({ ok: true, status: res.statusCode, tokensSaved: parseInt(saved ?? '0', 10) });
-          } else if (res.statusCode === 503) {
-            reject(new Error(`Compression pipeline returned 503 — proxy overloaded or ML model not loaded`));
-          } else {
-            // Any other status is acceptable for a verify test — proxy is running
-            log('info', `Compression pipeline verify: status=${res.statusCode} (proxy is live)`);
-            resolve({ ok: true, status: res.statusCode, tokensSaved: 0 });
-          }
-        });
-        res.on('error', reject);
-      }
-    );
-    req.on('error', (err) => {
-      clearTimeout(timer);
-      reject(new Error(`Compression verify request failed: ${err.message}`));
-    });
-    req.write(testPayload);
-    req.end();
-  });
-}
-
-// ── Lock file ────────────────────────────────────────────────────────────────
-
-function writeLock(port, pid) {
-  const data = JSON.stringify({ port, pid, startedAt: new Date().toISOString() }, null, 2);
-  try {
-    fs.writeFileSync(LOCK_TMP, data, 'utf8');
-    fs.renameSync(LOCK_TMP, LOCK_FILE); // atomic
-  } catch (e) {
-    log('warn', `Failed to write headroom.lock: ${e.message}`);
-  }
-}
-
-function removeLock() {
-  try { fs.unlinkSync(LOCK_FILE); } catch (_) {}
-  try { fs.unlinkSync(LOCK_TMP); } catch (_) {}
-}
-
-// ── Log file descriptor ──────────────────────────────────────────────────────
-
-function openLogFd() {
-  try {
-    return fs.openSync(LOG_FILE, 'a');
-  } catch (e) {
-    log('warn', `Cannot open headroom-proxy.log: ${e.message} — using /dev/null`);
-    return fs.openSync(process.platform === 'win32' ? 'NUL' : '/dev/null', 'w');
-  }
-}
-
-// ── HeadroomProxyManager ─────────────────────────────────────────────────────
-
-class HeadroomProxyManager {
-  constructor() {
-    this._process       = null;
-    this._port          = null;
-    this._pythonBin     = null;
-    this._restartCount  = 0;
-    this._shuttingDown  = false;
-    this._healthTimer   = null;
-    this._logFd         = null;
-    this._baseUrl       = null;
-  }
-
-  // ── Public API ─────────────────────────────────────────────────────────────
-
-  /**
-   * Start the proxy. Returns { port, baseUrl }.
-   * Throws (does NOT return an error object) on any failure — callers
-   * should let the exception propagate to daemon.start() for process.exit(1).
-   */
-  async start() {
-    log('info', 'Starting Headroom compression proxy...');
-
-    // Step 1: Resolve Python
-    this._pythonBin = resolvePythonBin();
-
-    // Step 2: Verify headroom installed in vendor
-    verifyHeadroomInstalled(this._pythonBin);
-
-    // Step 3: Find available port
-    this._port = await findAvailablePort();
-    log('info', `Using port ${this._port}`);
-
-    // Step 4: Spawn proxy
-    await this._spawnProxy();
-
-    // Step 5: Verify compression pipeline
-    await this._verifyPipeline();
-
-    // Step 6: Write lock file
-    writeLock(this._port, this._process.pid);
-
-    // Step 7: Start health monitor (auto-restart on crash)
-    this._startHealthMonitor();
-
-    this._baseUrl = `http://127.0.0.1:${this._port}`;
-    log('info', `Headroom proxy ready at ${this._baseUrl}`);
-
-    return { port: this._port, baseUrl: this._baseUrl };
-  }
-
-  /**
-   * Graceful shutdown. Called from daemon shutdown sequence.
-   */
-  async stop() {
-    this._shuttingDown = true;
-    if (this._healthTimer) {
-      clearInterval(this._healthTimer);
-      this._healthTimer = null;
-    }
-    if (this._process) {
-      await this._killProcess(this._process);
-      this._process = null;
-    }
-    removeLock();
-    if (this._logFd !== null) {
-      try { fs.closeSync(this._logFd); } catch (_) {}
-      this._logFd = null;
-    }
-    log('info', 'Headroom proxy stopped');
-  }
-
-  /**
-   * Returns the base URL for the running proxy (e.g. "http://127.0.0.1:8787").
-   * This is set after start() resolves. Read-only after that.
-   */
-  getBaseUrl() {
-    return this._baseUrl;
-  }
-
-  getPort() {
-    return this._port;
-  }
-
-  isRunning() {
-    return this._process !== null && !this._process.killed && this._baseUrl !== null;
-  }
-
-  // ── Private: spawn ──────────────────────────────────────────────────────────
-
-  async _spawnProxy() {
-    this._logFd = openLogFd();
-
-    const args = [
-      '-m', 'headroom.proxy',
-      '--port',        String(this._port),
-      '--host',        '127.0.0.1',
-      '--no-telemetry',
-      '--log-level',   'warn',
-      '--ccr-ttl',     '7200',
-      // Disable update-check nag in startup output
-      '--no-update-check',
-    ];
-
-    const env = {
-      ...process.env,
-      PYTHONPATH:                VENDOR_PATH,
-      PYTHONUNBUFFERED:          '1',       // ensure stdout/stderr are not buffered
-      HEADROOM_TELEMETRY:        'off',
-      HEADROOM_UPDATE_CHECK:     'off',
-      HEADROOM_CCR_TTL_SECONDS:  '7200',
-      HEADROOM_LOG_LEVEL:        'warn',
-      // Disable Serena popup that appeared in 0.26.0 changelog
-      HEADROOM_NO_SERENA:        '1',
-    };
-
-    log('info', `Spawning: ${this._pythonBin} ${args.join(' ')}`);
-    log('info', `CWD: ${VENDOR_PATH}, log: ${LOG_FILE}`);
-
-    this._process = spawn(this._pythonBin, args, {
-      cwd:        VENDOR_PATH,
-      detached:   false,          // owned by daemon, dies with daemon
-      stdio:      ['ignore', this._logFd, this._logFd],
-      windowsHide: true,
-      env,
-    });
-
-    this._process.on('exit', (code, signal) => {
-      if (!this._shuttingDown) {
-        log('warn', `Headroom proxy exited unexpectedly (code=${code} signal=${signal})`);
-        this._scheduleRestart();
-      }
-    });
-
-    this._process.on('error', (err) => {
-      log('error', `Headroom proxy process error: ${err.message}`);
-      if (!this._shuttingDown) {
-        this._scheduleRestart();
-      }
-    });
-
-    // Wait for proxy to become healthy
-    log('info', `Waiting up to ${STARTUP_TIMEOUT_MS / 1000}s for proxy to become healthy...`);
-    const healthy = await waitForHealth(this._port, STARTUP_TIMEOUT_MS);
-    if (!healthy) {
-      // Kill the process if it's hanging
-      if (this._process && !this._process.killed) {
-        this._process.kill('SIGKILL');
-      }
-      throw new Error(
-        `Headroom proxy did not become healthy within ${STARTUP_TIMEOUT_MS / 1000}s.\n` +
-        `Check ${LOG_FILE} for errors.\n` +
-        `Common causes:\n` +
-        `  - ML model download in progress (first run can take 1–5 minutes)\n` +
-        `  - Port ${this._port} blocked by firewall\n` +
-        `  - Python dependency missing: cd "${VENDOR_PATH}" && pip install -e ".[proxy,ml]"\n` +
-        `Try running manually: ${this._pythonBin} -m headroom.proxy --port ${this._port} --host 127.0.0.1`
-      );
-    }
-    log('info', `Proxy healthy on port ${this._port}`);
-  }
-
-  // ── Private: verify compression pipeline ────────────────────────────────────
-
-  async _verifyPipeline() {
-    log('info', 'Verifying compression pipeline...');
-    try {
-      const result = await verifyCompressionPipeline(this._port);
-      log('info', `Compression pipeline verified (status=${result.status})`);
-    } catch (err) {
-      throw new Error(
-        `Headroom compression pipeline verification failed: ${err.message}\n` +
-        `The proxy is running but compression is not functioning correctly.\n` +
-        `Check ${LOG_FILE} for details.`
-      );
-    }
-  }
-
-  // ── Private: health monitor (Option 1 — auto-restart on crash) ──────────────
-
-  _startHealthMonitor() {
-    this._healthTimer = setInterval(async () => {
-      if (this._shuttingDown) return;
-      const alive = await probeHealth(this._port, 3000);
-      if (!alive && !this._shuttingDown) {
-        log('warn', 'Headroom proxy health check failed — scheduling restart');
-        this._scheduleRestart();
-        // Clear the health monitor; _scheduleRestart will re-arm it after recovery
-        clearInterval(this._healthTimer);
-        this._healthTimer = null;
-      }
-    }, HEALTH_CHECK_MS);
-  }
-
-  /**
-   * Schedule an auto-restart with exponential backoff.
-   * After MAX_RESTARTS failures, logs error but does NOT exit the daemon —
-   * the compression pipeline will be degraded but HBO/agent tasks can continue.
-   * This is the "resilient mid-session crash" recovery (Option 1).
-   */
-  _scheduleRestart() {
-    if (this._shuttingDown) return;
-    if (this._restartCount >= MAX_RESTARTS) {
-      log('error',
-        `Headroom proxy has crashed ${this._restartCount} times — giving up auto-restart.\n` +
-        `HBO compression is degraded. Restart the daemon to restore compression.\n` +
-        `Check ${LOG_FILE} for root cause.`
-      );
-      this._baseUrl = null; // mark as unavailable so LLM path wrappers stop routing through it
-      return;
-    }
-
-    const delay = RESTART_BACKOFF_MS[Math.min(this._restartCount, RESTART_BACKOFF_MS.length - 1)];
-    log('info', `Scheduling headroom proxy restart #${this._restartCount + 1} in ${delay}ms...`);
-    this._restartCount++;
-
-    setTimeout(async () => {
-      if (this._shuttingDown) return;
-      log('info', `Attempting headroom proxy restart #${this._restartCount}...`);
-      try {
-        // Kill any zombie process
-        if (this._process && !this._process.killed) {
-          await this._killProcess(this._process);
-        }
-        // Re-open log fd (may have been closed on previous process exit)
-        if (this._logFd === null) this._logFd = openLogFd();
-
-        // Re-check port availability — it may have changed
-        this._port = await findAvailablePort();
-        this._baseUrl = null; // clear until we re-verify
-
-        // Re-spawn
-        await this._spawnProxy();
-
-        // Re-verify pipeline
-        await this._verifyPipeline();
-
-        // Update lock file with new pid/port
-        writeLock(this._port, this._process.pid);
-        this._baseUrl = `http://127.0.0.1:${this._port}`;
-        this._restartCount = 0; // reset on successful restart
-
-        // Re-arm health monitor
-        this._startHealthMonitor();
-
-        log('info', `Headroom proxy restarted successfully on port ${this._port}`);
-      } catch (err) {
-        log('error', `Headroom proxy restart failed: ${err.message}`);
-        this._scheduleRestart(); // will back off further
-      }
-    }, delay);
-  }
-
-  // ── Private: process kill ────────────────────────────────────────────────────
-
-  _killProcess(child) {
-    return new Promise((resolve) => {
-      if (!child || child.killed) return resolve();
-
-      const timer = setTimeout(() => {
-        try { child.kill('SIGKILL'); } catch (_) {}
-        resolve();
-      }, 3000);
-
-      child.once('exit', () => {
-        clearTimeout(timer);
-        resolve();
-      });
-
-      try {
-        child.kill('SIGTERM');
-      } catch (_) {
-        clearTimeout(timer);
-        resolve();
-      }
-    });
-  }
-
-  // ── Singleton ────────────────────────────────────────────────────────────────
-
-  static getInstance() {
-    if (!HeadroomProxyManager._instance) {
-      HeadroomProxyManager._instance = new HeadroomProxyManager();
-    }
-    return HeadroomProxyManager._instance;
-  }
-}
-
-HeadroomProxyManager._instance = null;
-
-module.exports = { HeadroomProxyManager };