llama-cpp-pro 0.2.2

package/desktop/electron-builder.config.cjs

@@ -0,0 +1,157 @@
+/**
+ * electron-builder configuration for apps using llama-cpp-pro on desktop.
+ *
+ * Merge into your Electron app's package.json:
+ *   const llamaDesktop = require('llama-cpp-pro/desktop/electron-builder');
+ *   module.exports = { ...yourConfig, ...llamaDesktop.merge(yourConfig) };
+ *
+ * Or spread extraResources / entitlements manually from this module.
+ */
+
+const path = require('path');
+
+const PACKAGE_ROOT = path.resolve(__dirname, '..');
+
+/** @param {string} [root] - llama-cpp-pro package root (default: auto) */
+function getSidecarResourceEntries(root = PACKAGE_ROOT) {
+  return {
+    mac: [
+      {
+        from: path.join(root, 'extraResources/sidecar/darwin-${arch}'),
+        to: 'sidecar/darwin-${arch}',
+      },
+      {
+        from: path.join(root, 'extraResources/sidecar/ggml-plugins/darwin-${arch}'),
+        to: 'sidecar/ggml-plugins/darwin-${arch}',
+        filter: ['**/*'],
+      },
+    ],
+    win: [
+      {
+        from: path.join(root, 'extraResources/sidecar/win32-${arch}.exe'),
+        to: 'sidecar/win32-${arch}.exe',
+      },
+      {
+        from: path.join(root, 'extraResources/sidecar/win32-${arch}-rocm.exe'),
+        to: 'sidecar/win32-${arch}-rocm.exe',
+      },
+      {
+        from: path.join(root, 'extraResources/sidecar/win32-${arch}-cuda.exe'),
+        to: 'sidecar/win32-${arch}-cuda.exe',
+      },
+      {
+        from: path.join(root, 'extraResources/sidecar/ggml-plugins/win32-${arch}'),
+        to: 'sidecar/ggml-plugins/win32-${arch}',
+        filter: ['**/*.dll'],
+      },
+      {
+        from: path.join(root, 'extraResources/sidecar/runtime-libs/win32-${arch}'),
+        to: 'sidecar/runtime-libs/win32-${arch}',
+        filter: ['**/*.dll'],
+      },
+    ],
+    linux: [
+      {
+        from: path.join(root, 'extraResources/sidecar/linux-${arch}'),
+        to: 'sidecar/linux-${arch}',
+      },
+      {
+        from: path.join(root, 'extraResources/sidecar/linux-${arch}-rocm'),
+        to: 'sidecar/linux-${arch}-rocm',
+      },
+      {
+        from: path.join(root, 'extraResources/sidecar/linux-${arch}-cuda'),
+        to: 'sidecar/linux-${arch}-cuda',
+      },
+      {
+        from: path.join(root, 'extraResources/sidecar/ggml-plugins/linux-${arch}'),
+        to: 'sidecar/ggml-plugins/linux-${arch}',
+        filter: ['**/*.so', '**/*.so.*'],
+      },
+      {
+        from: path.join(root, 'extraResources/sidecar/runtime-libs/linux-${arch}'),
+        to: 'sidecar/runtime-libs/linux-${arch}',
+        filter: ['**/*.so', '**/*.so.*'],
+      },
+    ],
+    wasm: [
+      {
+        from: path.join(root, 'dist/wasm'),
+        to: 'llama-wasm',
+        filter: ['**/*'],
+      },
+    ],
+  };
+}
+
+/**
+ * Recommended electron-builder fragment for llama-cpp desktop support.
+ * @param {object} [opts]
+ * @param {string} [opts.packageRoot]
+ * @returns {object}
+ */
+function buildConfig(opts = {}) {
+  const root = opts.packageRoot || PACKAGE_ROOT;
+  const sidecar = getSidecarResourceEntries(root);
+
+  return {
+    asar: false,
+    mac: {
+      entitlements: path.join(root, 'desktop/entitlements.mac.plist'),
+      entitlementsInherit: path.join(root, 'desktop/entitlements.mac.plist'),
+      hardenedRuntime: true,
+      gatekeeperAssess: false,
+      extraResources: [...sidecar.mac, ...sidecar.wasm],
+    },
+    win: {
+      extraResources: [...sidecar.win, ...sidecar.wasm],
+    },
+    linux: {
+      extraResources: [...sidecar.linux, ...sidecar.wasm],
+    },
+    nsis: {
+      oneClick: false,
+      perMachine: true,
+      allowToChangeInstallationDirectory: true,
+      createDesktopShortcut: true,
+    },
+    deb: {
+      depends: ['libvulkan1', 'libgomp1'],
+    },
+  };
+}
+
+/**
+ * Deep-merge llama desktop resources into an existing electron-builder config.
+ * @param {object} base - your app's electron-builder config
+ * @param {object} [opts]
+ * @returns {object}
+ */
+function merge(base = {}, opts = {}) {
+  const llama = buildConfig(opts);
+  const out = { ...base, ...llama, asar: base.asar ?? llama.asar };
+
+  for (const platform of ['mac', 'win', 'linux']) {
+    if (!out[platform]) out[platform] = {};
+    const baseExtra = base[platform]?.extraResources ?? [];
+    const llamaExtra = llama[platform]?.extraResources ?? [];
+    out[platform] = {
+      ...llama[platform],
+      ...out[platform],
+      extraResources: [...baseExtra, ...llamaExtra],
+    };
+    if (platform === 'mac' && base.mac?.entitlements) {
+      out.mac.entitlements = base.mac.entitlements;
+    }
+  }
+
+  return out;
+}
+
+module.exports = {
+  buildConfig,
+  merge,
+  getSidecarResourceEntries,
+  entitlementsMac: path.join(PACKAGE_ROOT, 'desktop/entitlements.mac.plist'),
+  packageRoot: PACKAGE_ROOT,
+};

package/desktop/entitlements.mac.plist

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+    <true/>
+    <key>com.apple.security.cs.allow-jit</key>
+    <true/>
+    <key>com.apple.security.cs.disable-library-validation</key>
+    <true/>
+  </dict>
+</plist>

package/desktop/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "llama-cpp-pro-desktop",
+  "version": "0.2.0",
+  "private": true,
+  "description": "Desktop runtime modules for llama-cpp-pro (Electron main process)",
+  "main": "src/main/index.cjs",
+  "type": "commonjs",
+  "engines": {
+    "node": ">=18"
+  }
+}

package/desktop/resolve-package-root.cjs

@@ -0,0 +1,88 @@
+/**
+ * Portable resolution of the llama-cpp-pro package root and dev extraResources.
+ *
+ * Used by host apps (PKC, annadata-app) during co-development (file:../llama-cpp-pro)
+ * and after npm publish (require('llama-cpp-pro')).
+ */
+
+const path = require('path');
+const fs = require('fs');
+
+const SIBLING_DIR = 'llama-cpp-pro';
+
+/**
+ * @param {object} [opts]
+ * @param {string} [opts.fromDir] - resolve installed package from this directory
+ * @param {string} [opts.siblingOf] - fallback: <siblingOf>/../llama-cpp-pro
+ * @returns {string}
+ */
+function resolvePackageRoot(opts = {}) {
+  const fromEnv = process.env.LLAMA_CPP_PRO_ROOT;
+  if (fromEnv) return path.resolve(fromEnv);
+
+  const fromDir = opts.fromDir || process.cwd();
+  try {
+    return path.dirname(require.resolve('llama-cpp-pro/package.json', { paths: [fromDir] }));
+  } catch {
+    /* not installed from this tree yet */
+  }
+
+  if (opts.siblingOf) {
+    return path.resolve(opts.siblingOf, '..', SIBLING_DIR);
+  }
+
+  // Loaded from llama-cpp-pro/desktop/resolve-package-root.cjs
+  return path.resolve(__dirname, '..');
+}
+
+/**
+ * Dev-only extraResources directory (sidecar + wasm staged by stage:desktop).
+ * @param {object} [opts] - forwarded to resolvePackageRoot
+ * @returns {string}
+ */
+function getDevExtraResourcesDir(opts = {}) {
+  return path.join(resolvePackageRoot(opts), 'extraResources');
+}
+
+/**
+ * resourcesPath for Electron: packaged → process.resourcesPath; dev → extraResources/.
+ * @param {import('electron').App | null | undefined} app
+ * @param {object} [opts] - forwarded to resolvePackageRoot (fromDir defaults to app.getAppPath())
+ * @returns {string}
+ */
+function getResourcesPathForApp(app, opts = {}) {
+  if (app && app.isPackaged) {
+    return process.resourcesPath;
+  }
+  const resolveOpts = { ...opts };
+  if (!resolveOpts.fromDir && app && typeof app.getAppPath === 'function') {
+    resolveOpts.fromDir = app.getAppPath();
+  }
+  return getDevExtraResourcesDir(resolveOpts);
+}
+
+/**
+ * Ensure a platform sidecar binary exists under resourcesPath (throws with build hint).
+ * @param {string} resourcesPath
+ * @param {object} [deps] - forwarded to resolveBinaryPath
+ * @returns {string}
+ */
+function assertSidecarBinary(resourcesPath, deps) {
+  const { resolveBinaryPath } = require('./src/main/sidecar-manager.cjs');
+  const binary = resolveBinaryPath({ ...(deps || {}), resourcesPath });
+  const _fs = (deps && deps.fs) || fs;
+  if (!_fs.existsSync(binary)) {
+    throw new Error(
+      `llama-cpp-pro sidecar missing at ${binary}. ` +
+        'From llama-cpp-pro run: npm run build:sidecar && npm run stage:desktop',
+    );
+  }
+  return binary;
+}
+
+module.exports = {
+  resolvePackageRoot,
+  getDevExtraResourcesDir,
+  getResourcesPathForApp,
+  assertSidecarBinary,
+};

package/desktop/src/main/backend-selector.cjs

@@ -0,0 +1,148 @@
+/**
+ * Backend ranking and selection for desktop llama-cpp.
+ * Adapted from ref-code/annadata-vad-dt/src/main/backend-selector.cjs.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { getSettingsDir } = require('./model-store.cjs');
+
+const NPU_RANK = ['coreml-npu', 'openvino-npu'];
+const VALID_OVERRIDES = ['auto', 'sidecar-npu', 'sidecar-gpu', 'sidecar-cpu', 'wasm-cpu'];
+
+function getGpuRank(platform = process.platform) {
+  if (platform === 'darwin') {
+    return ['metal', 'cuda', 'rocm', 'openvino-gpu', 'vulkan'];
+  }
+  return ['vulkan', 'cuda', 'rocm', 'openvino-gpu'];
+}
+
+function getSettingsPath(deps) {
+  return path.join(getSettingsDir(deps), 'settings.json');
+}
+
+function getUserOverride(deps) {
+  const _fs = (deps && deps.fs) || fs;
+  try {
+    const raw = _fs.readFileSync(getSettingsPath(deps), 'utf8');
+    const data = JSON.parse(raw);
+    const value = data && data.backendOverride;
+    if (!value || value === 'auto') return null;
+    if (VALID_OVERRIDES.includes(value)) return value;
+    return null;
+  } catch (err) {
+    if (err && err.code === 'ENOENT') return null;
+    return null;
+  }
+}
+
+function setUserOverride(backend, deps) {
+  const _fs = (deps && deps.fs) || fs;
+  const settingsPath = getSettingsPath(deps);
+  let data = {};
+  try {
+    data = JSON.parse(_fs.readFileSync(settingsPath, 'utf8'));
+  } catch (_) { /* new file */ }
+  data.backendOverride = backend;
+  _fs.mkdirSync(path.dirname(settingsPath), { recursive: true });
+  _fs.writeFileSync(settingsPath, JSON.stringify(data, null, 2), 'utf8');
+}
+
+function findBestBackend(probeResult, rank) {
+  for (const name of rank) {
+    const b = probeResult.backends.find((x) => x.name === name && x.available);
+    if (b) return name;
+  }
+  return null;
+}
+
+/**
+ * Map detected GPU backend name to sidecar binary variant suffix.
+ * @param {string|null} gpuBackend
+ * @param {NodeJS.Platform} platform
+ * @returns {string}
+ */
+function backendToVariant(gpuBackend, platform) {
+  if (platform === 'darwin') {
+    if (gpuBackend === 'coreml-npu') return 'metal-coreml';
+    return 'metal';
+  }
+  switch (gpuBackend) {
+    case 'cuda': return 'cuda';
+    case 'rocm': return 'rocm';
+    case 'vulkan': return 'vulkan';
+    case 'openvino-gpu':
+    case 'openvino-npu': return 'cuda-openvino';
+    default: return 'vulkan-openblas';
+  }
+}
+
+function selectBackend(probeResult, userOverride, deps) {
+  const platform = (deps && deps.platform) || process.platform;
+  const gpuRank = getGpuRank(platform);
+
+  if (userOverride === 'wasm-cpu') {
+    return { type: 'wasm-cpu', gpuBackend: null, variant: null, reason: 'User selected WASM CPU' };
+  }
+  if (userOverride === 'sidecar-cpu') {
+    return { type: 'sidecar-cpu', gpuBackend: null, variant: 'cpu', reason: 'User selected native CPU' };
+  }
+  if (userOverride === 'sidecar-gpu') {
+    const bestGpu = findBestBackend(probeResult, gpuRank);
+    if (bestGpu) {
+      return {
+        type: 'sidecar-gpu', gpuBackend: bestGpu, variant: backendToVariant(bestGpu, platform),
+        reason: `User GPU override: ${bestGpu}`,
+      };
+    }
+    return { type: 'wasm-cpu', gpuBackend: null, variant: null, reason: 'No GPU detected' };
+  }
+  if (userOverride === 'sidecar-npu') {
+    const bestNpu = findBestBackend(probeResult, NPU_RANK);
+    if (bestNpu) {
+      return {
+        type: 'sidecar-npu', gpuBackend: bestNpu, variant: backendToVariant(bestNpu, platform),
+        reason: `User NPU override: ${bestNpu}`,
+      };
+    }
+  }
+
+  const bestNpu = findBestBackend(probeResult, NPU_RANK);
+  if (bestNpu) {
+    return {
+      type: 'sidecar-npu', gpuBackend: bestNpu, variant: backendToVariant(bestNpu, platform),
+      reason: `Auto-selected NPU: ${bestNpu}`,
+    };
+  }
+
+  const bestGpu = findBestBackend(probeResult, gpuRank);
+  if (bestGpu) {
+    return {
+      type: 'sidecar-gpu', gpuBackend: bestGpu, variant: backendToVariant(bestGpu, platform),
+      reason: `Auto-selected GPU: ${bestGpu}`,
+    };
+  }
+
+  if (platform !== 'darwin') {
+    return {
+      type: 'sidecar-cpu', gpuBackend: null, variant: 'openblas',
+      reason: 'No accelerator detected — native CPU sidecar',
+    };
+  }
+
+  return {
+    type: 'wasm-cpu', gpuBackend: null, variant: null,
+    reason: 'No accelerator on macOS — WASM fallback (or set sidecar-cpu override)',
+  };
+}
+
+module.exports = {
+  selectBackend,
+  getUserOverride,
+  setUserOverride,
+  findBestBackend,
+  getGpuRank,
+  backendToVariant,
+  NPU_RANK,
+  VALID_OVERRIDES,
+};

package/desktop/src/main/gpu-probe.cjs

@@ -0,0 +1,142 @@
+/**
+ * GPU / NPU backend detection for desktop llama-cpp sidecar.
+ * Adapted from ref-code/annadata-vad-dt/src/main/gpu-probe.cjs.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+function findLibrary(candidates, _fs) {
+  for (const p of candidates) {
+    try {
+      if (_fs.existsSync(p)) return { found: true, path: p };
+    } catch (_) { /* skip */ }
+  }
+  return { found: false };
+}
+
+function versionedCandidates(dirs, base) {
+  const suffixes = ['', '.1', '.2', '.530', '.520', '.515', '.510'];
+  const out = [];
+  for (const d of dirs) {
+    for (const s of suffixes) out.push(path.join(d, base + s));
+  }
+  return out;
+}
+
+function probeWindows(_fs) {
+  const sys32 = path.join(process.env.SYSTEMROOT || 'C:\\Windows', 'System32');
+  const programFiles = process.env.ProgramFiles || 'C:\\Program Files';
+  const backends = [];
+
+  const cuda = findLibrary([path.join(sys32, 'nvcuda.dll')], _fs);
+  backends.push({
+    name: 'cuda', kind: 'gpu', available: cuda.found,
+    reason: cuda.found ? `found ${cuda.path}` : 'nvcuda.dll not found',
+  });
+
+  const rocmCandidates = [path.join(sys32, 'amdhip64.dll')];
+  const rocm = findLibrary(rocmCandidates, _fs);
+  backends.push({
+    name: 'rocm', kind: 'gpu', available: rocm.found,
+    reason: rocm.found ? `found ${rocm.path}` : 'amdhip64.dll not found',
+  });
+
+  const vulkan = findLibrary([path.join(sys32, 'vulkan-1.dll')], _fs);
+  backends.push({
+    name: 'vulkan', kind: 'gpu', available: vulkan.found,
+    reason: vulkan.found ? `found ${vulkan.path}` : 'vulkan-1.dll not found',
+  });
+
+  const openvinoCandidates = [
+    path.join(sys32, 'openvino.dll'),
+    path.join(programFiles, 'Intel', 'openvino_2025', 'runtime', 'bin', 'intel64', 'Release', 'openvino.dll'),
+  ];
+  const openvino = findLibrary(openvinoCandidates, _fs);
+  backends.push({
+    name: 'openvino-npu', kind: 'npu', available: openvino.found,
+    reason: openvino.found ? `found ${openvino.path}` : 'OpenVINO runtime not found',
+  });
+  backends.push({
+    name: 'openvino-gpu', kind: 'gpu', available: openvino.found,
+    reason: openvino.found ? `found ${openvino.path}` : 'OpenVINO runtime not found',
+  });
+
+  return backends;
+}
+
+function probeDarwin(_fs) {
+  const backends = [];
+  backends.push({
+    name: 'metal', kind: 'gpu', available: true,
+    reason: 'Metal is always available on macOS',
+  });
+  const coreml = findLibrary([
+    '/System/Library/Frameworks/CoreML.framework/CoreML',
+  ], _fs);
+  backends.push({
+    name: 'coreml-npu', kind: 'npu', available: coreml.found,
+    reason: coreml.found ? 'CoreML framework found (ANE on Apple Silicon)' : 'CoreML not found',
+  });
+  return backends;
+}
+
+function probeLinux(_fs) {
+  const libDirs = [
+    '/usr/lib', '/usr/lib64', '/usr/lib/x86_64-linux-gnu',
+    '/usr/local/lib', '/usr/local/cuda/lib64', '/opt/rocm/lib',
+  ];
+  const backends = [];
+
+  const cuda = findLibrary(versionedCandidates(libDirs, 'libcuda.so'), _fs);
+  backends.push({
+    name: 'cuda', kind: 'gpu', available: cuda.found,
+    reason: cuda.found ? `found ${cuda.path}` : 'libcuda.so not found',
+  });
+
+  const rocm = findLibrary(versionedCandidates(libDirs, 'libamdhip64.so'), _fs);
+  backends.push({
+    name: 'rocm', kind: 'gpu', available: rocm.found,
+    reason: rocm.found ? `found ${rocm.path}` : 'libamdhip64.so not found',
+  });
+
+  const vulkan = findLibrary(versionedCandidates(libDirs, 'libvulkan.so'), _fs);
+  backends.push({
+    name: 'vulkan', kind: 'gpu', available: vulkan.found,
+    reason: vulkan.found ? `found ${vulkan.path}` : 'libvulkan.so not found',
+  });
+
+  const openvino = findLibrary(versionedCandidates(libDirs, 'libopenvino.so'), _fs);
+  backends.push({
+    name: 'openvino-gpu', kind: 'gpu', available: openvino.found,
+    reason: openvino.found ? `found ${openvino.path}` : 'libopenvino.so not found',
+  });
+  backends.push({
+    name: 'openvino-npu', kind: 'npu', available: openvino.found,
+    reason: openvino.found ? `found ${openvino.path}` : 'OpenVINO not found',
+  });
+
+  return backends;
+}
+
+function probe(opts) {
+  const _platform = (opts && opts.platform) || process.platform;
+  const _fs = (opts && opts.fs) || fs;
+  let backends = [];
+
+  switch (_platform) {
+    case 'win32': backends = probeWindows(_fs); break;
+    case 'darwin': backends = probeDarwin(_fs); break;
+    case 'linux': backends = probeLinux(_fs); break;
+    default: backends = []; break;
+  }
+
+  backends.push({
+    name: 'cpu', kind: 'cpu', available: true,
+    reason: 'CPU fallback is always available',
+  });
+
+  return { backends };
+}
+
+module.exports = { probe };

package/desktop/src/main/index.cjs

@@ -0,0 +1,58 @@
+/**
+ * Desktop runtime entry — GPU probe, backend selection, sidecar lifecycle.
+ *
+ * Usage (Electron main process):
+ *   const desktop = require('llama-cpp-pro/desktop');
+ *   const probe = desktop.probeGpu();
+ *   const selection = desktop.selectBackend(probe);
+ *   const manager = desktop.createSidecarManager();
+ *   await manager.start({ modelPath: '/path/to/model.gguf', selection });
+ */
+
+const { probe } = require('./gpu-probe.cjs');
+const {
+  selectBackend,
+  getUserOverride,
+  setUserOverride,
+} = require('./backend-selector.cjs');
+const { createSidecarManager, resolveBinaryPath } = require('./sidecar-manager.cjs');
+const { createSidecarClient } = require('./sidecar-client.cjs');
+const { getModelsDir, getSettingsDir } = require('./model-store.cjs');
+const {
+  resolvePackageRoot,
+  getDevExtraResourcesDir,
+  getResourcesPathForApp,
+  assertSidecarBinary,
+} = require('../../resolve-package-root.cjs');
+
+/**
+ * Full desktop bootstrap: probe hardware, read user settings, select backend.
+ * @param {object} [deps]
+ * @returns {{ probe: object, selection: object }}
+ */
+function detectBackend(deps) {
+  const probeResult = probe(deps);
+  const override = getUserOverride(deps);
+  const selection = selectBackend(probeResult, override, deps);
+  return { probe: probeResult, selection };
+}
+
+module.exports = {
+  probeGpu: probe,
+  selectBackend,
+  getUserOverride,
+  setUserOverride,
+  detectBackend,
+  createSidecarManager,
+  createSidecarClient,
+  resolveBinaryPath,
+  resolvePackageRoot,
+  getDevExtraResourcesDir,
+  getResourcesPathForApp,
+  assertSidecarBinary,
+  getModelsDir,
+  getSettingsDir,
+  registerLlamaDesktopIpc: require('./ipc-handlers.cjs').registerLlamaDesktopIpc,
+  installPreload: require('./preload.cjs'),
+  electronBuilder: require('../../electron-builder.config.cjs'),
+};