@fugood/llama.node 0.6.3 → 1.0.0-beta.1

package/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.15)
 cmake_policy(SET CMP0091 NEW)
 cmake_policy(SET CMP0042 NEW)
 
-project (llama-node)
+project (index)
 
 set(CMAKE_CXX_STANDARD 17)
 
@@ -34,13 +34,19 @@ string(REPLACE "arm" "arm" ARCH ${ARCH})
 string(REPLACE "arm64x" "arm64" ARCH ${ARCH})
 string(REPLACE "aarch64" "arm64" ARCH ${ARCH})
 
+option(TO_PACKAGE "Build as package" OFF)
+
 if(DEFINED VARIANT)
   set(VARIANT -${VARIANT})
 else()
   set(VARIANT "")
 endif()
 
-set(PLATFORM_BINARY_DIR ${CMAKE_SOURCE_DIR}/bin/${PLATFORM}${VARIANT}/${ARCH})
+if (TO_PACKAGE)
+  set(PLATFORM_BINARY_DIR ${CMAKE_SOURCE_DIR}/packages/node-llama-${PLATFORM}-${ARCH}${VARIANT})
+else()
+  set(PLATFORM_BINARY_DIR ${CMAKE_SOURCE_DIR}/build/Release)
+endif()
 
 message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
 message(STATUS "Platform: ${PLATFORM}")
@@ -57,8 +63,14 @@ endif()
 # Improve speed
 if(CMAKE_BUILD_TYPE STREQUAL "Release")
   if (MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ob2 /Oi /Ot /Oy /GL")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O2 /Ob2 /Oi /Ot /Oy /GL")
+    if (NOT GGML_VULKAN)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ob2 /Oi /Ot /Oy /GL")
+      set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O2 /Ob2 /Oi /Ot /Oy /GL")
+      set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} /LTCG")
+    else()
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O1 /Ob1 /bigobj")
+      set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O1 /Ob1 /bigobj")
+    endif()
   else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -funroll-loops -flto")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -funroll-loops -flto")
@@ -74,6 +86,10 @@ endif()
 # flags: -fPIC
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
+if (MINGW)
+  add_definitions(-D_WIN32_WINNT=0x0601)
+endif()
+
 # VULKAN_SDK
 if (VULKAN_SDK)
   set(ENV{VULKAN_SDK} ${VULKAN_SDK})
@@ -119,15 +135,15 @@ file(
     "src/DecodeAudioTokenWorker.h"
     "src/tts_utils.cpp"
     "src/tts_utils.h"
+    "src/anyascii.h"
+    "src/anyascii.c"
 )
 
-if (CLANG AND CMAKE_SYSTEM_NAME STREQUAL "Windows")
+if (NOT MSVC AND CMAKE_SYSTEM_NAME STREQUAL "Windows")
   file(GLOB WIN_DYNAMIC_LOAD_SRC "src/win_dynamic_load.c")
 
   add_library(win_dynamic_load ${WIN_DYNAMIC_LOAD_SRC})
-  if (NOT MSVC)
-    set_target_properties(win_dynamic_load PROPERTIES COMPILE_FLAGS "-Wno-implicit-function-declaration")
-  endif()
+  set_target_properties(win_dynamic_load PROPERTIES COMPILE_FLAGS "-Wno-implicit-function-declaration")
   
   unset(CMAKE_JS_SRC)
   unset(CMAKE_JS_LIB)
@@ -145,40 +161,34 @@ target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB} llama ggml common mtmd ${C
 
 add_custom_target(copy_assets ALL DEPENDS ${PROJECT_NAME})
 
-add_custom_command(
-  TARGET copy_assets
-  COMMAND ${CMAKE_COMMAND} -E remove_directory ${PLATFORM_BINARY_DIR}
-  COMMENT "Cleaning bin folder"
-)
-
-if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET)
-  # Generate node.lib
-  execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS})
-  # copy target to bin folder
-  get_filename_component(CMAKE_JS_NODELIB_TARGET_NAME ${CMAKE_JS_NODELIB_TARGET} NAME)
+if (TO_PACKAGE)
   add_custom_command(TARGET copy_assets
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_JS_NODELIB_TARGET} ${PLATFORM_BINARY_DIR}/${CMAKE_JS_NODELIB_TARGET_NAME}
-    COMMENT "Copying to bin folder"
+    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${PROJECT_NAME}> ${PLATFORM_BINARY_DIR}/$<TARGET_FILE_NAME:${PROJECT_NAME}>
+    COMMENT "Deploy as package"
   )
 endif()
 
-# copy target to bin folder
-add_custom_command(TARGET copy_assets
-  COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${PROJECT_NAME}> ${PLATFORM_BINARY_DIR}/$<TARGET_FILE_NAME:${PROJECT_NAME}>
-  COMMENT "Copying to bin folder"
-)
+if (TO_PACKAGE)
+  set(METAL_LIB_TARGET_PATH ${PLATFORM_BINARY_DIR})
+else()
+  set(METAL_LIB_TARGET_PATH ${CMAKE_BINARY_DIR}/bin/default.metallib)
+endif()
+
+if(CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET)
+  execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS})
+endif()
 
-if (LLAMA_METAL)
-  # copy ${CMAKE_BINARY_DIR}/bin/default.metallib
+if (GGML_METAL AND NOT GGML_METAL_EMBED_LIBRARY)
+  # copy ${CMAKE_BINARY_DIR}/bin/default.metallib  
   add_custom_command(
     TARGET copy_assets
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/bin/default.metallib ${PLATFORM_BINARY_DIR}/default.metallib
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/bin/default.metallib ${METAL_LIB_TARGET_PATH}
     COMMENT "Copying default.metallib to bin folder"
   )
   add_dependencies(copy_assets ggml-metal)
 endif()
 
-if (LLAMA_CLBLAST)
+if (GGML_CLBLAST AND TO_PACKAGE)
   find_package(CLBlast)
   if (CLBlast_FOUND)
     message(STATUS "CLBlast found: ${CLBlast_DIR}")

package/README.md

@@ -17,6 +17,7 @@ An another Node binding of [llama.cpp](https://github.com/ggerganov/llama.cpp) t
 - Windows (x86_64 and arm64)
   - CPU
   - GPU acceleration via Vulkan
+  - GPU acceleration via CUDA (x86_64)
 - Linux (x86_64 and arm64)
   - CPU
   - GPU acceleration via Vulkan
@@ -63,7 +64,9 @@ console.log('Result:', text)
 
 - [x] `default`: General usage, not support GPU except macOS (Metal)
 - [x] `vulkan`: Support GPU Vulkan (Windows/Linux), but some scenario might unstable
-- [x] `cuda`: Support GPU CUDA (Linux), but only for limited capability (x86_64: 8.9, arm64: 8.7)
+- [x] `cuda`: Support GPU CUDA (Windows/Linux), but only for limited capability
+  > Linux: (x86_64: 8.9, arm64: 8.7)
+  > Windows: x86_64 - 12.0
 
 ## License
 

package/lib/binding.js

@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -33,30 +43,32 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadModule = void 0;
-const path = __importStar(require("path"));
-const setupEnv = (variant) => {
-    var _a, _b;
-    const postfix = variant ? `-${variant}` : '';
-    const binPath = path.resolve(__dirname, `../bin/${process.platform}${postfix}/${process.arch}/`);
-    const systemPathEnv = (_b = (_a = process.env.PATH) !== null && _a !== void 0 ? _a : process.env.Path) !== null && _b !== void 0 ? _b : '';
-    if (!systemPathEnv.includes(binPath)) {
-        if (process.platform === 'win32') {
-            process.env.Path = `${binPath};${systemPathEnv}`;
-        }
-        else {
-            process.env.PATH = `${binPath}:${systemPathEnv}`;
-        }
-    }
+const getPlatformPackageName = (variant) => {
+    const platform = process.platform;
+    const arch = process.arch;
+    const variantSuffix = variant && variant !== 'default' ? `-${variant}` : '';
+    return `@fugood/node-llama-${platform}-${arch}${variantSuffix}`;
 };
-const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
+const loadPlatformPackage = (packageName) => __awaiter(void 0, void 0, void 0, function* () {
     try {
-        if (variant && variant !== 'default') {
-            setupEnv(variant);
-            return (yield Promise.resolve(`${`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s))));
-        }
+        return yield Promise.resolve(`${packageName}`).then(s => __importStar(require(s)));
+    }
+    catch (error) {
+        return null;
+    }
+});
+const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
+    let module = yield loadPlatformPackage(getPlatformPackageName(variant));
+    if (module) {
+        return module;
+    }
+    module = yield loadPlatformPackage(getPlatformPackageName());
+    if (module) {
+        console.warn(`Not found package for variant "${variant}", fallback to default`);
+        return module;
     }
-    catch (_a) { } // ignore errors and try the common path
-    setupEnv();
-    return (yield Promise.resolve(`${`../bin/${process.platform}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s))));
+    console.warn(`Not found package for your platform, fallback to local build`);
+    // @ts-ignore
+    return (yield Promise.resolve().then(() => __importStar(require('../build/Release/index.node'))));
 });
 exports.loadModule = loadModule;

package/lib/binding.ts

@@ -268,33 +268,34 @@ export interface Module {
 
 export type LibVariant = 'default' | 'vulkan' | 'cuda'
 
-const setupEnv = (variant?: string) => {
-  const postfix = variant ? `-${variant}` : ''
-  const binPath = path.resolve(
-    __dirname,
-    `../bin/${process.platform}${postfix}/${process.arch}/`,
-  )
-  const systemPathEnv = process.env.PATH ?? process.env.Path ?? ''
-  if (!systemPathEnv.includes(binPath)) {
-    if (process.platform === 'win32') {
-      process.env.Path = `${binPath};${systemPathEnv}`
-    } else {
-      process.env.PATH = `${binPath}:${systemPathEnv}`
-    }
+const getPlatformPackageName = (variant?: LibVariant): string => {
+  const platform = process.platform
+  const arch = process.arch
+  const variantSuffix = variant && variant !== 'default' ? `-${variant}` : ''
+  return `@fugood/node-llama-${platform}-${arch}${variantSuffix}`
+}
+
+const loadPlatformPackage = async (packageName: string): Promise<Module | null> => {
+  try {
+    return await import(packageName) as Module
+  } catch (error) {
+    return null
   }
 }
 
 export const loadModule = async (variant?: LibVariant): Promise<Module> => {
-  try {
-    if (variant && variant !== 'default') {
-      setupEnv(variant)
-      return (await import(
-        `../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`
-      )) as Module
-    }
-  } catch {} // ignore errors and try the common path
-  setupEnv()
-  return (await import(
-    `../bin/${process.platform}/${process.arch}/llama-node.node`
-  )) as Module
+  let module = await loadPlatformPackage(getPlatformPackageName(variant))
+  if (module) {
+    return module
+  }
+
+  module = await loadPlatformPackage(getPlatformPackageName())
+  if (module) {
+    console.warn(`Not found package for variant "${variant}", fallback to default`)
+    return module
+  }
+
+  console.warn(`Not found package for your platform, fallback to local build`)
+  // @ts-ignore
+  return (await import('../build/Release/index.node')) as Module
 }

package/package.json

@@ -1,20 +1,23 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.6.3",
+  "version": "1.0.0-beta.1",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
+    "postinstall": "node scripts/build.js",
     "pretest": "node scripts/download-test-models.js",
     "test": "jest",
     "build": "tsc",
-    "prepack": "yarn build",
+    "prepack": "npm run build",
     "prebuild-native": "node scripts/generate_win_dynamic_load.js 6",
     "build-native": "cmake-js compile",
     "clean": "rimraf build",
     "prepare": "husky",
     "commitlint": "commitlint --edit",
-    "release": "release-it"
+    "release": "release-it",
+    "update-packages": "node scripts/update-packages.js",
+    "publish-if-need": "node scripts/publish-if-need.js"
   },
   "repository": {
     "type": "git",
@@ -35,7 +38,8 @@
   },
   "homepage": "https://github.com/mybigday/llama.node#readme",
   "publishConfig": {
-    "registry": "https://registry.npmjs.org"
+    "registry": "https://registry.npmjs.org",
+    "access": "public"
   },
   "binary": {
     "napi_versions": [
@@ -43,12 +47,46 @@
     ]
   },
   "files": [
-    "bin/**/*",
-    "src/**/*.{c,cc,cpp,h,hh,hpp,txt,cmake}",
+    "scripts/build.js",
+    "scripts/llama.cpp.patch",
+    "src/*.{cc,c,h,hpp}",
+    "src/DecodeAudioTokenWorker.cpp",
+    "src/DetokenizeWorker.cpp",
+    "src/DisposeWorker.cpp",
+    "src/EmbeddingWorker.cpp",
+    "src/LlamaCompletionWorker.cpp",
+    "src/LlamaContext.cpp",
+    "src/LoadSessionWorker.cpp",
+    "src/SaveSessionWorker.cpp",
+    "src/TokenizeWorker.cpp",
+    "src/tts_utils.cpp",
+    "src/llama.cpp/{common,src,include}/**/*.{h,hpp,cpp,cc,c}",
+    "src/llama.cpp/ggml/include/*.h",
+    "src/llama.cpp/ggml/src/ggml-cpu/**/*.{h,hpp,cpp,cc,c}",
     "lib/*.js",
     "lib/*.ts",
     "CMakeLists.txt"
   ],
+  "optionalDependencies": {
+    "@fugood/node-llama-linux-x64": "1.0.0-beta.1",
+    "@fugood/node-llama-linux-x64-vulkan": "1.0.0-beta.1",
+    "@fugood/node-llama-linux-x64-cuda": "1.0.0-beta.1",
+    "@fugood/node-llama-linux-arm64": "1.0.0-beta.1",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.0.0-beta.1",
+    "@fugood/node-llama-linux-arm64-cuda": "1.0.0-beta.1",
+    "@fugood/node-llama-win32-x64": "1.0.0-beta.1",
+    "@fugood/node-llama-win32-x64-vulkan": "1.0.0-beta.1",
+    "@fugood/node-llama-win32-x64-cuda": "1.0.0-beta.1",
+    "@fugood/node-llama-win32-arm64": "1.0.0-beta.1",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.0.0-beta.1",
+    "@fugood/node-llama-win32-arm64-cuda": "1.0.0-beta.1",
+    "@fugood/node-llama-darwin-x64": "1.0.0-beta.1",
+    "@fugood/node-llama-darwin-x64-vulkan": "1.0.0-beta.1",
+    "@fugood/node-llama-darwin-x64-cuda": "1.0.0-beta.1",
+    "@fugood/node-llama-darwin-arm64": "1.0.0-beta.1",
+    "@fugood/node-llama-darwin-arm64-vulkan": "1.0.0-beta.1",
+    "@fugood/node-llama-darwin-arm64-cuda": "1.0.0-beta.1"
+  },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",
     "@babel/preset-typescript": "^7.24.1",
@@ -88,4 +126,4 @@
     "singleQuote": true,
     "printWidth": 80
   }
-}
+}

package/scripts/build.js

@@ -0,0 +1,47 @@
+const fs = require('fs');
+const path = require('path');
+
+const validAccelerators = process.platform === 'darwin' ? [] : ['vulkan', 'cuda'];
+
+let isBuildFromSource = process.env.npm_config_build_from_source === 'true';
+
+let accelerator = process.env.npm_config_accelerator || '';
+
+const checkPaths = [
+  path.resolve(
+    __dirname,
+    `../node-llama-${process.platform}-${process.arch}${accelerator ? `-${accelerator}` : ''}`
+  ),
+  path.resolve(__dirname, `../build/Release/index.node`),
+];
+
+if (!isBuildFromSource && !checkPaths.some(path => fs.existsSync(path))) {
+  console.warn('Not found prebuild package, please build from source');
+  isBuildFromSource = true;
+}
+
+if (accelerator && !validAccelerators.includes(accelerator)) {
+  throw new Error(`Invalid accelerator: ${accelerator}`);
+}
+
+if (isBuildFromSource) {
+  console.log('Build from source is enabled');
+}
+
+let BuildSystem;
+try {
+  ({ BuildSystem } = require('cmake-js'));
+} catch (error) {
+  console.error('cmake-js is not installed, please install it');
+  process.exit(1);
+}
+
+const buildSystem = new BuildSystem({
+  directory: path.resolve(__dirname, '../'),
+  arch: process.arch,
+  preferClang: true,
+  out: path.resolve(__dirname, '../build'),
+  extraCMakeArgs: [accelerator && `--CDVARIANT=${accelerator}`].filter(Boolean),
+});
+
+buildSystem.build();

package/scripts/llama.cpp.patch

@@ -0,0 +1,109 @@
+diff --git a/src/llama.cpp/common/chat.cpp b/src/llama.cpp/common/chat.cpp
+index f138c7bc..e177fe92 100644
+--- a/src/llama.cpp/common/chat.cpp
++++ b/src/llama.cpp/common/chat.cpp
+@@ -1,8 +1,6 @@
+ #include "chat.h"
+ #include "json-schema-to-grammar.h"
+ #include "log.h"
+-#include "minja/chat-template.hpp"
+-#include "minja/minja.hpp"
+ 
+ #include <optional>
+ 
+@@ -15,14 +13,6 @@ static std::string format_time(const std::chrono::system_clock::time_point & now
+     return res;
+ }
+ 
+-typedef minja::chat_template common_chat_template;
+-
+-struct common_chat_templates {
+-    bool has_explicit_template; // Model had builtin template or template overridde was specified.
+-    std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
+-    std::unique_ptr<common_chat_template> template_tool_use;
+-};
+-
+ struct templates_params {
+     json messages;
+     json tools;
+diff --git a/src/llama.cpp/common/chat.h b/src/llama.cpp/common/chat.h
+index d26a09c2..cb92721a 100644
+--- a/src/llama.cpp/common/chat.h
++++ b/src/llama.cpp/common/chat.h
+@@ -6,8 +6,16 @@
+ #include <chrono>
+ #include <string>
+ #include <vector>
++#include "minja/chat-template.hpp"
++#include "minja/minja.hpp"
+ 
+-struct common_chat_templates;
++typedef minja::chat_template common_chat_template;
++
++struct common_chat_templates {
++    bool has_explicit_template; // Model had builtin template or template overridde was specified.
++    std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
++    std::unique_ptr<common_chat_template> template_tool_use;
++};
+ 
+ struct common_chat_tool_call {
+     std::string name;
+diff --git a/src/llama.cpp/common/common.cpp b/src/llama.cpp/common/common.cpp
+index 94f545f8..a55df8aa 100644
+--- a/src/llama.cpp/common/common.cpp
++++ b/src/llama.cpp/common/common.cpp
+@@ -1062,6 +1062,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
+         mparams.n_gpu_layers = params.n_gpu_layers;
+     }
+ 
++    mparams.vocab_only      = params.vocab_only;
+     mparams.main_gpu        = params.main_gpu;
+     mparams.split_mode      = params.split_mode;
+     mparams.tensor_split    = params.tensor_split;
+diff --git a/src/llama.cpp/common/common.h b/src/llama.cpp/common/common.h
+index 0a9dc059..996afcd8 100644
+--- a/src/llama.cpp/common/common.h
++++ b/src/llama.cpp/common/common.h
+@@ -217,6 +217,7 @@ enum common_reasoning_format {
+ };
+ 
+ struct common_params {
++    bool vocab_only               = false;
+     int32_t n_predict             =    -1; // new tokens to predict
+     int32_t n_ctx                 =  4096; // context size
+     int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
+diff --git a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
+index 9a3085be..8218cc16 100644
+--- a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
++++ b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
+@@ -90,7 +90,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
+         message(STATUS "ARM detected")
+ 
+         if (MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang")
+-            message(FATAL_ERROR "MSVC is not supported for ARM, use clang")
++            list(APPEND ARCH_FLAGS /arch:armv8.7)
+         else()
+             check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)
+             if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
+diff --git a/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
+index 662f1377..f9f99698 100644
+--- a/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
++++ b/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
+@@ -122,7 +122,7 @@ if (Vulkan_FOUND)
+     endif()
+ 
+     # Set up toolchain for host compilation whether cross-compiling or not
+-    if (CMAKE_CROSSCOMPILING)
++    if (CMAKE_CROSSCOMPILING OR NOT CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR)
+         if (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN)
+             set(HOST_CMAKE_TOOLCHAIN_FILE ${GGML_VULKAN_SHADERS_GEN_TOOLCHAIN})
+         else()
+@@ -144,7 +144,7 @@ if (Vulkan_FOUND)
+     include(ExternalProject)
+ 
+     # Add toolchain file if cross-compiling
+-    if (CMAKE_CROSSCOMPILING)
++    if (CMAKE_CROSSCOMPILING OR NOT CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR)
+         list(APPEND VULKAN_SHADER_GEN_CMAKE_ARGS -DCMAKE_TOOLCHAIN_FILE=${HOST_CMAKE_TOOLCHAIN_FILE})
+         message(STATUS "vulkan-shaders-gen toolchain file: ${HOST_CMAKE_TOOLCHAIN_FILE}")
+     endif()