llama-cpp-pro 0.2.2

package/scripts/ensure-desktop-sidecar-bundle.cjs

@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+/**
+ * Preflight check before electron-builder packaging.
+ *
+ * Usage:
+ *   node scripts/ensure-desktop-sidecar-bundle.cjs
+ *   node scripts/ensure-desktop-sidecar-bundle.cjs --platform=darwin|linux|win32
+ *   node scripts/ensure-desktop-sidecar-bundle.cjs --arch=arm64|x64|universal
+ *
+ * By default, only the current host platform/arch binary is required
+ * (correct for local Electron dev on Apple Silicon). Pass --arch=universal
+ * on macOS to require both darwin-arm64 and darwin-x64.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const root = path.resolve(__dirname, '..');
+const extra = path.join(root, 'extraResources', 'sidecar');
+
+function argValue(name) {
+  const prefix = `--${name}=`;
+  const hit = process.argv.find((a) => a.startsWith(prefix));
+  if (hit) return hit.slice(prefix.length);
+  const idx = process.argv.indexOf(`--${name}`);
+  if (idx >= 0 && process.argv[idx + 1] && !process.argv[idx + 1].startsWith('-')) {
+    return process.argv[idx + 1];
+  }
+  return null;
+}
+
+const platform = argValue('platform') || process.platform;
+const archOpt = argValue('arch');
+
+function hostArchLabel() {
+  const a = os.arch();
+  if (a === 'arm64') return 'arm64';
+  if (a === 'x64' || a === 'x86_64') return 'x64';
+  return a;
+}
+
+function requiredBinaries(plat, archMode) {
+  if (plat === 'darwin') {
+    if (archMode === 'universal' || archMode === 'all') {
+      return ['darwin-arm64', 'darwin-x64'];
+    }
+    const arch = archMode === 'x64' || archMode === 'arm64' ? archMode : hostArchLabel();
+    return [`darwin-${arch}`];
+  }
+  if (plat === 'linux') {
+    return ['linux-x64'];
+  }
+  if (plat === 'win32') {
+    return ['win32-x64.exe'];
+  }
+  return [];
+}
+
+const optional = {
+  darwin: [],
+  linux: ['linux-x64-rocm', 'linux-x64-cuda', 'linux-x64-vulkan', 'linux-x64-openblas'],
+  win32: ['win32-x64-rocm.exe', 'win32-x64-cuda.exe', 'win32-x64-vulkan.exe', 'win32-x64-openblas.exe'],
+};
+
+const required = requiredBinaries(platform, archOpt);
+const errors = [];
+
+if (!required.length) {
+  process.stderr.write(`Unsupported platform: ${platform}\n`);
+  process.exit(1);
+}
+
+for (const name of required) {
+  const p = path.join(extra, name);
+  if (!fs.existsSync(p)) {
+    errors.push(`Missing required: extraResources/sidecar/${name}`);
+  }
+}
+
+if (errors.length) {
+  process.stderr.write('Desktop sidecar bundle incomplete:\n\n');
+  for (const e of errors) process.stderr.write(`  • ${e}\n`);
+  process.stderr.write('\nRun: npm run build:sidecar && npm run stage:desktop\n');
+  if (platform === 'darwin') {
+    process.stderr.write(
+      'Tip: on Apple Silicon only darwin-arm64 is required for local Electron.\n' +
+        '     For universal Mac packages: npm run verify:desktop:bundle -- --arch=universal\n',
+    );
+  }
+  process.exit(1);
+}
+
+const foundOptional = (optional[platform] || []).filter((n) =>
+  fs.existsSync(path.join(extra, n)),
+);
+process.stdout.write(
+  `Desktop bundle OK (${platform}): ${required.join(', ')}` +
+    (foundOptional.length ? `; GPU variants: ${foundOptional.join(', ')}` : '') +
+    '\n',
+);

package/scripts/ensure-llama-ios-xcframework.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Build llama-cpp.xcframework (device + simulator) inside this npm package.
+#
+# Consumer Capacitor apps — run from app root after npm install (do not copy this file):
+#   bash node_modules/llama-cpp-capacitor/scripts/ensure-llama-ios-xcframework.sh
+#
+# Plugin developers — from this repo:
+#   npm run build:ios:xcframework
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+# Plugin root: this script lives in scripts/ inside the npm package.
+if [ -f "$SCRIPT_DIR/../ios/CMakeLists.txt" ] && [ -f "$SCRIPT_DIR/../package.json" ]; then
+  PLUGIN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+else
+  echo "Error: could not locate llama-cpp-capacitor plugin root (expected ios/CMakeLists.txt next to package.json)."
+  exit 1
+fi
+
+IOS_DIR="$PLUGIN_DIR/ios"
+FRAMEWORK_DIR="$IOS_DIR/Frameworks"
+XCFRAMEWORK_DIR="$FRAMEWORK_DIR/llama-cpp.xcframework"
+BUILD_PROFILE_FILE="$FRAMEWORK_DIR/.llama-cpp-build-profile"
+# Bump when native sources change (e.g. Metal embed) so cached xcframework is rebuilt.
+EXPECTED_BUILD_PROFILE="metal-embed-cpu-fallback-v1"
+DEVICE_BUILD_DIR="$IOS_DIR/build-device"
+SIM_BUILD_DIR="$IOS_DIR/build-simulator"
+
+if [[ "$OSTYPE" != darwin* ]]; then
+  echo "Skipping llama-cpp xcframework build: macOS with Xcode is required."
+  exit 0
+fi
+
+if [ -d "$XCFRAMEWORK_DIR" ]; then
+  if [ -f "$BUILD_PROFILE_FILE" ] && [ "$(cat "$BUILD_PROFILE_FILE")" = "$EXPECTED_BUILD_PROFILE" ]; then
+    echo "llama-cpp iOS xcframework is already present at $XCFRAMEWORK_DIR"
+    exit 0
+  fi
+  echo "Rebuilding llama-cpp iOS xcframework (build profile changed or missing)."
+  rm -rf "$XCFRAMEWORK_DIR"
+fi
+
+rm -rf "$DEVICE_BUILD_DIR" "$SIM_BUILD_DIR"
+
+if ! command -v cmake >/dev/null 2>&1; then
+  echo "Error: cmake is required to build llama-cpp for iOS."
+  exit 1
+fi
+
+build_framework() {
+  local build_dir="$1"
+  local sdk="$2"
+  local architectures="$3"
+
+  rm -rf "$build_dir"
+  mkdir -p "$build_dir"
+  cmake -S "$IOS_DIR" -B "$build_dir" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_OSX_SYSROOT="$sdk" \
+    -DCMAKE_OSX_ARCHITECTURES="$architectures" \
+    -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \
+    -DCMAKE_XCODE_ATTRIBUTE_ENABLE_BITCODE=NO \
+    >&2
+
+  local jobs
+  jobs="$(sysctl -n hw.ncpu 2>/dev/null || echo 4)"
+  cmake --build "$build_dir" --config Release -- -j"$jobs" >&2
+
+  if [ -d "$build_dir/llama-cpp.framework" ]; then
+    printf '%s\n' "$build_dir/llama-cpp.framework"
+    return 0
+  fi
+
+  echo "Error: llama-cpp.framework was not produced in $build_dir" >&2
+  exit 1
+}
+
+echo "Building llama-cpp for iOS device (arm64)..."
+DEVICE_FRAMEWORK="$(build_framework "$DEVICE_BUILD_DIR" iphoneos arm64)"
+
+echo "Building llama-cpp for iOS simulator..."
+SIM_ARCHITECTURES="arm64"
+if [ "$(uname -m)" = "x86_64" ]; then
+  SIM_ARCHITECTURES="x86_64"
+fi
+SIM_FRAMEWORK="$(build_framework "$SIM_BUILD_DIR" iphonesimulator "$SIM_ARCHITECTURES")"
+
+rm -rf "$XCFRAMEWORK_DIR"
+xcodebuild -create-xcframework \
+  -framework "$DEVICE_FRAMEWORK" \
+  -framework "$SIM_FRAMEWORK" \
+  -output "$XCFRAMEWORK_DIR"
+
+if [ ! -d "$XCFRAMEWORK_DIR" ]; then
+  echo "Error: failed to create $XCFRAMEWORK_DIR"
+  exit 1
+fi
+
+mkdir -p "$FRAMEWORK_DIR"
+printf '%s' "$EXPECTED_BUILD_PROFILE" > "$BUILD_PROFILE_FILE"
+
+# Keep a flat device framework copy for CocoaPods / dlopen (arm64 slice).
+rm -rf "$FRAMEWORK_DIR/llama-cpp.framework"
+cp -R "$DEVICE_FRAMEWORK" "$FRAMEWORK_DIR/llama-cpp.framework"
+
+echo "Created llama-cpp xcframework at $XCFRAMEWORK_DIR"
+echo "Updated device framework at $FRAMEWORK_DIR/llama-cpp.framework"

package/scripts/fix-esm-extensions.cjs

@@ -0,0 +1,45 @@
+/**
+ * TypeScript emits extensionless relative imports under moduleResolution:node.
+ * Native ESM (Node + Vite exclude) needs explicit .js suffixes.
+ */
+const { readdirSync, readFileSync, writeFileSync, statSync } = require('node:fs');
+const { join, extname } = require('node:path');
+
+const esmRoot = join(__dirname, '..', 'dist', 'esm');
+
+const IMPORT_RE =
+  /(from\s+|import\s*\(\s*|export\s+\*\s+from\s+|export\s+\{[^}]*\}\s+from\s+)(['"])(\.[^'"]+?)\2/g;
+
+function walk(dir, out = []) {
+  for (const name of readdirSync(dir)) {
+    const p = join(dir, name);
+    const st = statSync(p);
+    if (st.isDirectory()) walk(p, out);
+    else if (extname(p) === '.js') out.push(p);
+  }
+  return out;
+}
+
+function rewrite(source) {
+  return source.replace(IMPORT_RE, (full, prefix, quote, spec) => {
+    if (/\.(js|mjs|cjs|json|wasm)(\?|$)/.test(spec)) return full;
+    return `${prefix}${quote}${spec}.js${quote}`;
+  });
+}
+
+if (!statSync(esmRoot, { throwIfNoEntry: false })?.isDirectory()) {
+  console.warn('[llama-cpp-pro] fix-esm-extensions: dist/esm missing');
+  process.exit(0);
+}
+
+let changed = 0;
+for (const file of walk(esmRoot)) {
+  const before = readFileSync(file, 'utf8');
+  const after = rewrite(before);
+  if (after !== before) {
+    writeFileSync(file, after);
+    changed += 1;
+  }
+}
+
+console.log(`[llama-cpp-pro] fixed ESM import extensions in ${changed} file(s)`);

package/scripts/prepare-js-dist.cjs

@@ -0,0 +1,28 @@
+/**
+ * Runs on npm pack/install (prepare). Builds JS dist when missing; no-ops otherwise.
+ */
+const { existsSync } = require('node:fs');
+const { join } = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const root = join(__dirname, '..');
+const marker = join(root, 'dist', 'esm', 'index.js');
+
+if (existsSync(marker)) {
+  process.exit(0);
+}
+
+const tsc = join(root, 'node_modules', '.bin', 'tsc');
+if (!existsSync(tsc)) {
+  console.warn(
+    '[llama-cpp-pro] dist/ missing and local build tools not installed yet. Run: npm run build:js',
+  );
+  process.exit(0);
+}
+
+const r = spawnSync('npm', ['run', 'build:js'], {
+  cwd: root,
+  stdio: 'inherit',
+  shell: process.platform === 'win32',
+});
+process.exit(r.status ?? 1);

package/scripts/stage-desktop-resources.cjs

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+/**
+ * Stage sidecar binaries, ggml GPU plugins, and WASM assets into extraResources/
+ * for electron-builder packaging.
+ *
+ * Usage: node scripts/stage-desktop-resources.cjs [--platform darwin|linux|win32]
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const root = path.resolve(__dirname, '..');
+const srcBin = path.join(root, 'sidecar', 'bin');
+const dstRoot = path.join(root, 'extraResources', 'sidecar');
+
+const platformArg = process.argv.find((a) => a.startsWith('--platform='));
+const onlyPlatform = platformArg ? platformArg.split('=')[1] : null;
+
+function mkdirp(p) {
+  fs.mkdirSync(p, { recursive: true });
+}
+
+function copyFile(src, dst) {
+  if (!fs.existsSync(src)) return false;
+  mkdirp(path.dirname(dst));
+  fs.copyFileSync(src, dst);
+  return true;
+}
+
+function copyDir(src, dst, filter) {
+  if (!fs.existsSync(src)) return 0;
+  mkdirp(dst);
+  let n = 0;
+  for (const name of fs.readdirSync(src)) {
+    const sp = path.join(src, name);
+    const dp = path.join(dst, name);
+    const st = fs.statSync(sp);
+    if (st.isDirectory()) {
+      n += copyDir(sp, dp, filter);
+    } else if (!filter || filter(name)) {
+      fs.copyFileSync(sp, dp);
+      n += 1;
+    }
+  }
+  return n;
+}
+
+const BIN_PATTERNS = [
+  /^darwin-(arm64|x64)$/,
+  /^linux-x64(-(rocm|cuda|vulkan|openblas))?$/,
+  /^win32-x64(-(rocm|cuda|vulkan|openblas))?\.exe$/,
+];
+
+function shouldCopyBin(name) {
+  return BIN_PATTERNS.some((re) => re.test(name));
+}
+
+function stageSidecarBins() {
+  if (!fs.existsSync(srcBin)) {
+    console.error(`Missing ${srcBin} — run npm run build:sidecar first`);
+    process.exit(1);
+  }
+  mkdirp(dstRoot);
+  let copied = 0;
+  for (const name of fs.readdirSync(srcBin)) {
+    if (!shouldCopyBin(name)) continue;
+    if (onlyPlatform && !name.startsWith(onlyPlatform)) continue;
+    const ok = copyFile(path.join(srcBin, name), path.join(dstRoot, name));
+    if (ok) {
+      copied += 1;
+      console.log(`  staged ${name}`);
+    }
+  }
+  return copied;
+}
+
+function stageGgmlPlugins() {
+  const srcPlugins = path.join(srcBin, 'ggml-plugins');
+  if (!fs.existsSync(srcPlugins)) return 0;
+  return copyDir(srcPlugins, path.join(dstRoot, 'ggml-plugins'), (n) =>
+    /\.(so|dll|dylib)/i.test(n) || !n.includes('.'),
+  );
+}
+
+function stageRuntimeLibs() {
+  const srcLibs = path.join(srcBin, 'runtime-libs');
+  if (!fs.existsSync(srcLibs)) return 0;
+  return copyDir(srcLibs, path.join(dstRoot, 'runtime-libs'), (n) =>
+    /\.(so|dll|dylib)/i.test(n) || !n.includes('.'),
+  );
+}
+
+function stageWasm() {
+  const srcWasm = path.join(root, 'dist', 'wasm');
+  const dstWasm = path.join(root, 'extraResources', 'llama-wasm');
+  if (!fs.existsSync(srcWasm)) {
+    console.warn('  wasm: dist/wasm not found — run npm run build:pwa');
+    return 0;
+  }
+  return copyDir(srcWasm, dstWasm);
+}
+
+console.log('Staging desktop resources...');
+const bins = stageSidecarBins();
+const plugins = stageGgmlPlugins();
+const libs = stageRuntimeLibs();
+const wasm = stageWasm();
+
+console.log(`Done: ${bins} sidecar binary(ies), ${plugins} plugin file(s), ${libs} runtime lib(s), ${wasm} wasm file(s)`);
+
+if (bins === 0) {
+  console.error('No sidecar binaries staged.');
+  process.exit(1);
+}

package/sidecar/CMakeLists.txt

@@ -0,0 +1,192 @@
+# =============================================================================
+# llama-cpp-sidecar — Native accelerated desktop sidecar binary
+#
+# Builds a standalone executable with the embedded cap-native-server HTTP API.
+# GPU backends (Vulkan, CUDA, ROCm/HIP) are loaded dynamically at runtime from
+# libggml-*.so / ggml-*.dll placed next to the binary (see cmake/ggml-backends.cmake).
+#
+# Usage:
+#   cmake -B build -DCMAKE_BUILD_TYPE=Release -DSIDECAR_VARIANT=<variant>
+#   cmake --build build --config Release
+#
+# Variants (set via -DSIDECAR_VARIANT):
+#   vulkan-openblas    — CPU + dynamic Vulkan   [default win/linux x64]
+#   vulkan             — dynamic Vulkan only
+#   openblas           — CPU-optimized (OpenBLAS when available)
+#   cuda               — dynamic CUDA (NVIDIA)
+#   rocm               — dynamic ROCm/HIP (AMD)
+#   metal              — Apple Metal GPU (macOS)
+#   metal-coreml       — Metal + CoreML ANE (macOS arm64)
+#   cpu                — CPU-only universal fallback
+# =============================================================================
+
+cmake_minimum_required(VERSION 3.16)
+project(llama-cpp-sidecar VERSION 0.2.0 LANGUAGES C CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/desktop-sources.cmake)
+
+# ---------------------------------------------------------------------------
+# Platform / architecture detection
+# ---------------------------------------------------------------------------
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+    set(SIDECAR_PLATFORM "win32")
+elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    set(SIDECAR_PLATFORM "darwin")
+elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    set(SIDECAR_PLATFORM "linux")
+else()
+    set(SIDECAR_PLATFORM "${CMAKE_SYSTEM_NAME}")
+endif()
+
+set(_SIDECAR_DETECTED_PROC "${CMAKE_SYSTEM_PROCESSOR}")
+if(CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_OSX_ARCHITECTURES STREQUAL "")
+    set(_SIDECAR_DETECTED_PROC "${CMAKE_OSX_ARCHITECTURES}")
+endif()
+
+if(_SIDECAR_DETECTED_PROC MATCHES "^(aarch64|arm64|ARM64)$")
+    set(SIDECAR_ARCH "arm64")
+elseif(_SIDECAR_DETECTED_PROC MATCHES "^(x86_64|AMD64|amd64)$")
+    set(SIDECAR_ARCH "x64")
+else()
+    set(SIDECAR_ARCH "${_SIDECAR_DETECTED_PROC}")
+endif()
+
+# ---------------------------------------------------------------------------
+# Variant selection
+# ---------------------------------------------------------------------------
+
+set(SIDECAR_VARIANT "" CACHE STRING
+    "Build variant: vulkan-openblas, cuda, rocm, metal, metal-coreml, cpu, openblas, vulkan")
+
+if(NOT SIDECAR_VARIANT)
+    if(SIDECAR_PLATFORM STREQUAL "darwin" AND SIDECAR_ARCH STREQUAL "arm64")
+        set(SIDECAR_VARIANT "metal-coreml")
+    elseif(SIDECAR_PLATFORM STREQUAL "darwin")
+        set(SIDECAR_VARIANT "metal")
+    else()
+        set(SIDECAR_VARIANT "vulkan-openblas")
+    endif()
+endif()
+
+set(SIDECAR_NEEDS_SUFFIX ON)
+if(SIDECAR_VARIANT STREQUAL "vulkan-openblas" AND NOT SIDECAR_PLATFORM STREQUAL "darwin")
+    set(SIDECAR_NEEDS_SUFFIX OFF)
+elseif(SIDECAR_VARIANT STREQUAL "metal-coreml" AND SIDECAR_PLATFORM STREQUAL "darwin" AND SIDECAR_ARCH STREQUAL "arm64")
+    set(SIDECAR_NEEDS_SUFFIX OFF)
+elseif(SIDECAR_VARIANT STREQUAL "metal" AND SIDECAR_PLATFORM STREQUAL "darwin")
+    set(SIDECAR_NEEDS_SUFFIX OFF)
+endif()
+
+if(SIDECAR_NEEDS_SUFFIX)
+    set(SIDECAR_BINARY_NAME "${SIDECAR_PLATFORM}-${SIDECAR_ARCH}-${SIDECAR_VARIANT}")
+else()
+    set(SIDECAR_BINARY_NAME "${SIDECAR_PLATFORM}-${SIDECAR_ARCH}")
+endif()
+
+message(STATUS "Sidecar target: ${SIDECAR_BINARY_NAME}")
+message(STATUS "Platform: ${SIDECAR_PLATFORM}  Arch: ${SIDECAR_ARCH}  Variant: ${SIDECAR_VARIANT}")
+
+# ---------------------------------------------------------------------------
+# Compile definitions per variant
+# ---------------------------------------------------------------------------
+
+set(SIDECAR_COMPILE_DEFS
+    -DNDEBUG
+    -DLM_GGML_USE_CPU
+)
+
+set(SIDECAR_SOURCES ${ANNADATA_DESKTOP_CORE_SOURCES})
+
+if(SIDECAR_VARIANT MATCHES "metal")
+    list(APPEND SIDECAR_SOURCES ${ANNADATA_CPP_DIR}/ggml-metal.m)
+    list(APPEND SIDECAR_COMPILE_DEFS
+        -DLM_GGML_USE_METAL
+        -DLM_GGML_METAL_EMBED_LIBRARY
+        -DLM_GGML_METAL_USE_BF16
+    )
+    if(APPLE)
+        list(APPEND SIDECAR_COMPILE_DEFS -DLM_GGML_USE_ACCELERATE)
+    endif()
+endif()
+
+# Vulkan/CUDA/ROCm are loaded dynamically via lm_ggml_backend_load_all_from_path().
+# Variant names select which plugin DLLs to bundle (see scripts/build-ggml-backends.sh).
+
+# ---------------------------------------------------------------------------
+# Metal shader embedding (macOS)
+# ---------------------------------------------------------------------------
+
+if(SIDECAR_VARIANT MATCHES "metal" AND APPLE)
+    enable_language(ASM)
+    include(${CMAKE_CURRENT_LIST_DIR}/../cmake/desktop-metal-embed.cmake)
+    list(APPEND SIDECAR_SOURCES ${METALLIB_EMBED_ASM})
+endif()
+
+# ---------------------------------------------------------------------------
+# Executable target
+# ---------------------------------------------------------------------------
+
+add_executable(${SIDECAR_BINARY_NAME}
+    ${CMAKE_CURRENT_SOURCE_DIR}/cap-sidecar-main.cpp
+    ${SIDECAR_SOURCES}
+)
+
+target_include_directories(${SIDECAR_BINARY_NAME} PRIVATE
+    ${ANNADATA_CPP_DIR}
+    ${ANNADATA_CPP_DIR}/ggml-cpu
+    ${ANNADATA_CPP_DIR}/tools/mtmd
+    ${ANNADATA_CPP_DIR}/vendor/cpp-httplib
+)
+
+target_compile_options(${SIDECAR_BINARY_NAME} PRIVATE ${SIDECAR_COMPILE_DEFS})
+
+find_package(Threads REQUIRED)
+target_link_libraries(${SIDECAR_BINARY_NAME} PRIVATE Threads::Threads)
+
+if(APPLE)
+    find_library(ACCELERATE_FRAMEWORK Accelerate)
+    find_library(METAL_FRAMEWORK Metal)
+    find_library(FOUNDATION_FRAMEWORK Foundation)
+    find_library(METALKIT_FRAMEWORK MetalKit)
+    target_link_libraries(${SIDECAR_BINARY_NAME} PRIVATE
+        ${ACCELERATE_FRAMEWORK}
+        ${METAL_FRAMEWORK}
+        ${FOUNDATION_FRAMEWORK}
+        ${METALKIT_FRAMEWORK}
+    )
+    # Honor target arch (allows arm64→x86_64 cross-builds via CMAKE_OSX_ARCHITECTURES)
+    if(SIDECAR_ARCH STREQUAL "arm64")
+        set_target_properties(${SIDECAR_BINARY_NAME} PROPERTIES OSX_ARCHITECTURES "arm64")
+    elseif(SIDECAR_ARCH STREQUAL "x64")
+        set_target_properties(${SIDECAR_BINARY_NAME} PROPERTIES OSX_ARCHITECTURES "x86_64")
+    endif()
+endif()
+
+if(WIN32)
+    target_link_libraries(${SIDECAR_BINARY_NAME} PRIVATE ws2_32)
+    if(MSVC)
+        set_property(TARGET ${SIDECAR_BINARY_NAME} PROPERTY
+            MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+    endif()
+endif()
+
+if(UNIX AND NOT APPLE)
+    target_link_options(${SIDECAR_BINARY_NAME} PRIVATE -static-libgcc -static-libstdc++)
+endif()
+
+set_target_properties(${SIDECAR_BINARY_NAME} PROPERTIES
+    RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin"
+    OUTPUT_NAME "${SIDECAR_BINARY_NAME}"
+)
+
+# Optional: build ggml backend plugins from upstream llama.cpp
+set(LLAMA_CPP_UPSTREAM "" CACHE PATH "Path to upstream llama.cpp for ggml GPU backend plugins")
+if(LLAMA_CPP_UPSTREAM AND EXISTS "${LLAMA_CPP_UPSTREAM}/CMakeLists.txt")
+    include(${CMAKE_CURRENT_LIST_DIR}/../cmake/ggml-backends.cmake)
+endif()
+
+install(TARGETS ${SIDECAR_BINARY_NAME} RUNTIME DESTINATION "sidecar")

package/sidecar/cap-sidecar-main.cpp

@@ -0,0 +1,68 @@
+/**
+ * Standalone desktop sidecar entry point.
+ *
+ * Loads optional ggml GPU backend plugins (libggml-vulkan.so, libggml-cuda.so, …)
+ * from the executable directory, then runs cap_llama_server_main until SIGINT/SIGTERM.
+ */
+
+#include "cap-native-server.h"
+#include "ggml-backend.h"
+#include "llama.h"
+
+#include <atomic>
+#include <chrono>
+#include <csignal>
+#include <cstring>
+#include <filesystem>
+#include <thread>
+
+namespace {
+
+std::atomic<bool> g_shutdown{false};
+
+void on_signal(int) {
+    g_shutdown.store(true);
+}
+
+void install_signal_handlers() {
+    std::signal(SIGINT, on_signal);
+    std::signal(SIGTERM, on_signal);
+#ifndef _WIN32
+    std::signal(SIGHUP, on_signal);
+#endif
+}
+
+const char * resolve_backend_dir(int argc, char ** argv) {
+    for (int i = 1; i < argc - 1; i++) {
+        if (argv[i] && std::strcmp(argv[i], "--backend-dir") == 0 && argv[i + 1]) {
+            return argv[i + 1];
+        }
+    }
+    return nullptr;
+}
+
+void load_dynamic_backends(int argc, char ** argv) {
+    const char * dir = resolve_backend_dir(argc, argv);
+    lm_ggml_backend_load_all_from_path(dir);
+}
+
+} // namespace
+
+int main(int argc, char ** argv) {
+    install_signal_handlers();
+    llama_backend_init();
+    load_dynamic_backends(argc, argv);
+
+    if (cap_llama_server_main(argc, argv) != 0) {
+        llama_backend_free();
+        return 1;
+    }
+
+    while (!g_shutdown.load() && cap_llama_server_is_running()) {
+        std::this_thread::sleep_for(std::chrono::milliseconds(200));
+    }
+
+    cap_llama_server_stop();
+    llama_backend_free();
+    return 0;
+}