llama-cpp-pro 0.2.2

package/desktop/src/main/ipc-handlers.cjs

@@ -0,0 +1,212 @@
+/**
+ * Electron main-process IPC handlers for llama-cpp desktop sidecar.
+ *
+ * Usage in your Electron main process (after app.whenReady):
+ *   const { registerLlamaDesktopIpc } = require('llama-cpp-pro/desktop/ipc-handlers');
+ *   registerLlamaDesktopIpc({ ipcMain, app });
+ */
+
+// Require leaf modules directly — do NOT require ./index.cjs here (circular:
+// index exports registerLlamaDesktopIpc from this file).
+const { probe } = require('./gpu-probe.cjs');
+const {
+  selectBackend,
+  getUserOverride,
+  setUserOverride,
+} = require('./backend-selector.cjs');
+const { createSidecarManager } = require('./sidecar-manager.cjs');
+const os = require('os');
+const { execSync } = require('child_process');
+
+const CHANNEL_ENSURE = 'llama-desktop:ensure-sidecar';
+const CHANNEL_STOP = 'llama-desktop:stop-sidecar';
+const CHANNEL_STATUS = 'llama-desktop:sidecar-status';
+const CHANNEL_BACKEND = 'llama-desktop:backend-status';
+const CHANNEL_OVERRIDE = 'llama-desktop:set-backend-override';
+const CHANNEL_GET_OVERRIDE = 'llama-desktop:get-backend-override';
+const CHANNEL_MEMORY = 'llama-desktop:memory-snapshot';
+
+/**
+ * macOS `os.freemem()` only counts truly free pages and is usually tiny while
+ * inactive/purgeable pages are reclaimable. Prefer an "available" estimate so
+ * model admission does not falsely reject on Darwin.
+ */
+function getAvailableSystemBytes(totalBytes) {
+  if (process.platform === 'darwin') {
+    try {
+      const out = execSync('vm_stat', { encoding: 'utf8' });
+      const pageSizeMatch = out.match(/page size of\s+(\d+)/i);
+      const pageSize = pageSizeMatch ? Number(pageSizeMatch[1]) : 16384;
+      const pages = (label) => {
+        const m = out.match(new RegExp(`${label}:\\s+([\\d.]+)`, 'i'));
+        return m ? Math.floor(Number(m[1].replace(/\./g, '')) * pageSize) : 0;
+      };
+      const available =
+        pages('Pages free') +
+        pages('Pages inactive') +
+        pages('Pages speculative') +
+        pages('Pages purgeable');
+      if (available > 0) return Math.min(totalBytes, available);
+    } catch {
+      /* fall through */
+    }
+  }
+
+  // Electron/Chromium may expose a better free figure than Node on some OSes.
+  try {
+    const info = process.getSystemMemoryInfo?.();
+    if (info && typeof info.free === 'number' && info.free > 0) {
+      // Chromium reports KB
+      const chromeFree = info.free * 1024;
+      if (chromeFree > os.freemem()) return Math.min(totalBytes, chromeFree);
+    }
+  } catch {
+    /* ignore */
+  }
+
+  return os.freemem();
+}
+
+function detectBackend(deps) {
+  const probeResult = probe(deps);
+  const override = getUserOverride(deps);
+  const selection = selectBackend(probeResult, override, deps);
+  return { probe: probeResult, selection };
+}
+
+/**
+ * @param {object} opts
+ * @param {import('electron').IpcMain} opts.ipcMain
+ * @param {import('electron').App} [opts.app]
+ * @param {object} [opts.deps] - forwarded to sidecar manager / probe
+ */
+function registerLlamaDesktopIpc(opts) {
+  const ipcMain = opts && opts.ipcMain;
+  if (!ipcMain) {
+    throw new Error('registerLlamaDesktopIpc requires ipcMain');
+  }
+
+  const manager = createSidecarManager(opts && opts.deps);
+  let lastSelection = null;
+
+  // Idempotent re-register (Electron hot reload / double whenReady).
+  for (const ch of [
+    CHANNEL_ENSURE,
+    CHANNEL_STOP,
+    CHANNEL_STATUS,
+    CHANNEL_BACKEND,
+    CHANNEL_OVERRIDE,
+    CHANNEL_GET_OVERRIDE,
+    CHANNEL_MEMORY,
+  ]) {
+    try {
+      ipcMain.removeHandler(ch);
+    } catch {
+      /* ignore */
+    }
+  }
+
+  ipcMain.handle(CHANNEL_ENSURE, async (_evt, payload) => {
+    const { selection } = detectBackend(opts && opts.deps);
+    lastSelection = selection;
+
+    if (selection.type === 'wasm-cpu') {
+      return { ok: false, reason: 'wasm-fallback', selection };
+    }
+
+    const st = manager.getStatus();
+    let result = { ok: true, port: st.port };
+    if (!st.running) {
+      result = await manager.start({
+        modelPath: payload && payload.modelPath,
+        modelId: payload && payload.modelId,
+        selection,
+        n_ctx: payload && payload.n_ctx,
+        n_gpu_layers: payload && payload.n_gpu_layers,
+        n_threads: payload && payload.n_threads,
+        retryCpu: true,
+      });
+    }
+
+    if (!result.ok) {
+      return { ...result, selection };
+    }
+
+    const gpuEnabled =
+      selection.type === 'sidecar-gpu' || selection.type === 'sidecar-npu';
+    return {
+      ok: true,
+      port: result.port,
+      gpuEnabled,
+      gpuBackend: selection.gpuBackend,
+      reasonNoGpu: gpuEnabled ? undefined : 'Native CPU inference (OpenBLAS / Accelerate)',
+      selection,
+    };
+  });
+
+  ipcMain.handle(CHANNEL_STOP, async () => {
+    await manager.stop();
+    return { ok: true };
+  });
+
+  ipcMain.handle(CHANNEL_STATUS, async () => {
+    const st = manager.getStatus();
+    return {
+      running: st.running,
+      port: st.port,
+      backend: st.backend,
+      variant: st.variant,
+      permanentWasmFallback: st.permanentWasmFallback,
+    };
+  });
+
+  ipcMain.handle(CHANNEL_BACKEND, async () => {
+    const detected = detectBackend(opts && opts.deps);
+    const st = manager.getStatus();
+    return { ...detected, sidecar: st, lastSelection };
+  });
+
+  ipcMain.handle(CHANNEL_OVERRIDE, async (_evt, value) => {
+    setUserOverride(value, opts && opts.deps);
+    return { ok: true };
+  });
+
+  ipcMain.handle(CHANNEL_GET_OVERRIDE, async () => {
+    const value = getUserOverride(opts && opts.deps);
+    return value || 'auto';
+  });
+
+  ipcMain.handle(CHANNEL_MEMORY, async () => {
+    const totalBytes = os.totalmem();
+    const freeBytes = getAvailableSystemBytes(totalBytes);
+    const usedBytes = Math.max(0, totalBytes - freeBytes);
+    const usedRatio = totalBytes > 0 ? usedBytes / totalBytes : 0;
+    return {
+      totalBytes,
+      usedBytes,
+      freeBytes,
+      pressure: usedRatio >= 0.85 ? 'high' : usedRatio >= 0.7 ? 'medium' : 'low',
+    };
+  });
+
+  if (opts && opts.app) {
+    opts.app.on('before-quit', () => {
+      manager.stop().catch(() => {});
+    });
+  }
+
+  return {
+    manager,
+    channels: {
+      CHANNEL_ENSURE,
+      CHANNEL_STOP,
+      CHANNEL_STATUS,
+      CHANNEL_BACKEND,
+      CHANNEL_OVERRIDE,
+      CHANNEL_GET_OVERRIDE,
+      CHANNEL_MEMORY,
+    },
+  };
+}
+
+module.exports = { registerLlamaDesktopIpc };

package/desktop/src/main/model-store.cjs

@@ -0,0 +1,50 @@
+/**
+ * Shared model storage paths for desktop (Windows, macOS, Linux).
+ * Mirrors ref-code/annadata-vad-dt/src/main/model-store.cjs.
+ */
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+function getModelsDir(deps) {
+  const _platform = (deps && deps.platform) || process.platform;
+  const _fs = (deps && deps.fs) || fs;
+  const _os = (deps && deps.os) || os;
+
+  let primary;
+  switch (_platform) {
+    case 'win32':
+      primary = path.join(
+        process.env.PROGRAMDATA || 'C:\\ProgramData',
+        'annadata',
+        'llama-cpp',
+        'models',
+      );
+      break;
+    case 'darwin':
+      primary = '/Users/Shared/annadata/llama-cpp/models';
+      break;
+    case 'linux':
+      primary = '/var/lib/annadata/llama-cpp/models';
+      break;
+    default:
+      primary = path.join(_os.homedir(), 'annadata', 'llama-cpp', 'models');
+  }
+
+  const fallback = path.join(_os.homedir(), 'annadata', 'llama-cpp', 'models');
+
+  try {
+    _fs.mkdirSync(primary, { recursive: true });
+    return primary;
+  } catch (_) {
+    _fs.mkdirSync(fallback, { recursive: true });
+    return fallback;
+  }
+}
+
+function getSettingsDir(deps) {
+  return path.dirname(getModelsDir(deps));
+}
+
+module.exports = { getModelsDir, getSettingsDir };

package/desktop/src/main/preload.cjs

@@ -0,0 +1,39 @@
+/**
+ * Electron preload bridge — exposes annadataLlama API to renderer.
+ *
+ * Usage in preload script:
+ *   require('llama-cpp-pro/desktop/preload')(contextBridge, ipcRenderer);
+ */
+
+/**
+ * @param {import('electron').ContextBridge} contextBridge
+ * @param {import('electron').IpcRenderer} ipcRenderer
+ */
+function installLlamaDesktopPreload(contextBridge, ipcRenderer) {
+  const api = {
+  ensureSidecar: (opts) => ipcRenderer.invoke('llama-desktop:ensure-sidecar', opts),
+  stopSidecar: () => ipcRenderer.invoke('llama-desktop:stop-sidecar'),
+  getSidecarStatus: () => ipcRenderer.invoke('llama-desktop:sidecar-status'),
+  getBackendStatus: () => ipcRenderer.invoke('llama-desktop:backend-status'),
+  setBackendOverride: (value) => ipcRenderer.invoke('llama-desktop:set-backend-override', value),
+  getBackendOverride: () => ipcRenderer.invoke('llama-desktop:get-backend-override'),
+  getMemorySnapshot: () => ipcRenderer.invoke('llama-desktop:memory-snapshot'),
+};
+
+  contextBridge.exposeInMainWorld('annadataLlama', api);
+  contextBridge.exposeInMainWorld('__annadataDesktop', true);
+
+  ipcRenderer.invoke('llama-desktop:sidecar-status').then((st) => {
+    if (st && st.running && st.port) {
+      contextBridge.exposeInMainWorld('__annadataSidecarPort', st.port);
+    }
+  }).catch(() => {});
+
+  ipcRenderer.on('llama-desktop:sidecar-port', (_evt, port) => {
+    if (typeof port === 'number') {
+      (globalThis).__annadataSidecarPort = port;
+    }
+  });
+}
+
+module.exports = installLlamaDesktopPreload;

package/desktop/src/main/sidecar-client.cjs

@@ -0,0 +1,76 @@
+/**
+ * HTTP client for the llama-cpp cap-native-server sidecar.
+ * Targets OpenAI-compatible endpoints on 127.0.0.1:{port}.
+ */
+
+const http = require('http');
+
+function createSidecarClient(port, deps) {
+  const _http = (deps && deps.http) || http;
+
+  function request(method, urlPath, body, timeoutMs) {
+    return new Promise((resolve, reject) => {
+      const payload = body != null ? JSON.stringify(body) : null;
+      const opts = {
+        hostname: '127.0.0.1',
+        port,
+        path: urlPath,
+        method,
+        headers: { 'Content-Type': 'application/json' },
+        timeout: timeoutMs,
+      };
+      if (payload != null) {
+        opts.headers['Content-Length'] = Buffer.byteLength(payload);
+      }
+
+      const req = _http.request(opts, (res) => {
+        const chunks = [];
+        res.on('data', (c) => chunks.push(c));
+        res.on('end', () => {
+          const raw = Buffer.concat(chunks).toString('utf8');
+          if (res.statusCode >= 400) {
+            reject(new Error(`Sidecar HTTP ${res.statusCode}: ${raw.slice(0, 300)}`));
+            return;
+          }
+          if (!raw) {
+            resolve({});
+            return;
+          }
+          try {
+            resolve(JSON.parse(raw));
+          } catch (_) {
+            resolve({ raw });
+          }
+        });
+      });
+
+      req.on('timeout', () => {
+        req.destroy();
+        reject(new Error(`Sidecar timeout: ${method} ${urlPath}`));
+      });
+      req.on('error', (err) => reject(err));
+      if (payload != null) req.write(payload);
+      req.end();
+    });
+  }
+
+  return {
+    health: () => request('GET', '/health', null, 5000),
+    chatCompletion: (body, timeoutMs = 120000) =>
+      request('POST', '/v1/chat/completions', body, timeoutMs),
+    completion: (body, timeoutMs = 120000) =>
+      request('POST', '/v1/completions', body, timeoutMs),
+    embeddings: (body, timeoutMs = 60000) =>
+      request('POST', '/v1/embeddings', body, timeoutMs),
+    models: () => request('GET', '/v1/models', null, 5000),
+    memory: () => request('GET', '/v1/internal/memory', null, 5000),
+    loadModel: (body, timeoutMs = 300000) =>
+      request('POST', '/v1/internal/models/load', body, timeoutMs),
+    unloadModel: (modelId, timeoutMs = 60000) =>
+      request('DELETE', `/v1/internal/models/${encodeURIComponent(modelId)}`, null, timeoutMs),
+    setContextLimit: (limit, timeoutMs = 5000) =>
+      request('POST', '/v1/internal/context-limit', { limit }, timeoutMs),
+  };
+}
+
+module.exports = { createSidecarClient };