node-llama-cpp 3.0.0-beta.1 → 3.0.0-beta.11

package/llama/addon.cpp

@@ -9,21 +9,104 @@
 #include "llama.h"
 #include "napi.h"
 
-std::string addon_model_token_to_piece(const struct llama_model * model, llama_token token) {
+#ifdef GPU_INFO_USE_CUBLAS
+#  include "gpuInfo/cuda-gpu-info.h"
+#endif
+#ifdef GPU_INFO_USE_METAL
+#  include "gpuInfo/metal-gpu-info.h"
+#endif
+
+
+struct addon_logger_log {
+    public:
+        const int logLevelNumber;
+        const std::stringstream* stringStream;
+};
+
+static void addonLlamaCppLogCallback(ggml_log_level level, const char* text, void* user_data);
+
+using AddonThreadSafeLogCallbackFunctionContext = Napi::Reference<Napi::Value>;
+void addonCallJsLogCallback(
+    Napi::Env env, Napi::Function callback, AddonThreadSafeLogCallbackFunctionContext* context, addon_logger_log* data
+);
+using AddonThreadSafeLogCallbackFunction =
+    Napi::TypedThreadSafeFunction<AddonThreadSafeLogCallbackFunctionContext, addon_logger_log, addonCallJsLogCallback>;
+
+AddonThreadSafeLogCallbackFunction addonThreadSafeLoggerCallback;
+bool addonJsLoggerCallbackSet = false;
+int addonLoggerLogLevel = 5;
+
+std::string addon_model_token_to_piece(const struct llama_model* model, llama_token token) {
     std::vector<char> result(8, 0);
     const int n_tokens = llama_token_to_piece(model, token, result.data(), result.size());
     if (n_tokens < 0) {
         result.resize(-n_tokens);
         int check = llama_token_to_piece(model, token, result.data(), result.size());
         GGML_ASSERT(check == -n_tokens);
-    }
-    else {
+    } else {
         result.resize(n_tokens);
     }
 
     return std::string(result.data(), result.size());
 }
 
+#ifdef GPU_INFO_USE_CUBLAS
+void lodCudaError(const char* message) {
+    addonLlamaCppLogCallback(GGML_LOG_LEVEL_ERROR, (std::string("CUDA error: ") + std::string(message)).c_str(), nullptr);
+}
+#endif
+
+Napi::Value getGpuVramInfo(const Napi::CallbackInfo& info) {
+    uint64_t total = 0;
+    uint64_t used = 0;
+
+#ifdef GPU_INFO_USE_CUBLAS
+    size_t cudaDeviceTotal = 0;
+    size_t cudaDeviceUsed = 0;
+    bool cudeGetInfoSuccess = gpuInfoGetTotalCudaDevicesInfo(&cudaDeviceTotal, &cudaDeviceUsed, lodCudaError);
+
+    if (cudeGetInfoSuccess) {
+        total += cudaDeviceTotal;
+        used += cudaDeviceUsed;
+    }
+#endif
+
+#ifdef GPU_INFO_USE_METAL
+    uint64_t metalDeviceTotal = 0;
+    uint64_t metalDeviceUsed = 0;
+    get_metal_gpu_info(&metalDeviceTotal, &metalDeviceUsed);
+
+    total += metalDeviceTotal;
+    used += metalDeviceUsed;
+#endif
+
+    Napi::Object result = Napi::Object::New(info.Env());
+    result.Set("total", Napi::Number::From(info.Env(), total));
+    result.Set("used", Napi::Number::From(info.Env(), used));
+
+    return result;
+}
+
+static Napi::Value getNapiToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
+    auto tokenType = llama_token_get_type(model, token);
+
+    if (tokenType == LLAMA_TOKEN_TYPE_UNDEFINED || tokenType == LLAMA_TOKEN_TYPE_UNKNOWN) {
+        return Napi::Number::From(info.Env(), -1);
+    }
+
+    return Napi::Number::From(info.Env(), token);
+}
+
+static Napi::Value getNapiControlToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
+    auto tokenType = llama_token_get_type(model, token);
+
+    if (tokenType != LLAMA_TOKEN_TYPE_CONTROL) {
+        return Napi::Number::From(info.Env(), -1);
+    }
+
+    return Napi::Number::From(info.Env(), token);
+}
+
 class AddonModel : public Napi::ObjectWrap<AddonModel> {
     public:
         llama_model_params model_params;
@@ -56,7 +139,6 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 }
             }
 
-            llama_backend_init(false);
             model = llama_load_model_from_file(modelPath.c_str(), model_params);
 
             if (model == NULL) {
@@ -95,8 +177,9 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
             }
 
             std::string text = info[0].As<Napi::String>().Utf8Value();
+            bool specialTokens = info[1].As<Napi::Boolean>().Value();
 
-            std::vector<llama_token> tokens = llama_tokenize(model, text, true, true);
+            std::vector<llama_token> tokens = llama_tokenize(model, text, false, specialTokens);
 
             Napi::Uint32Array result = Napi::Uint32Array::New(info.Env(), tokens.size());
             for (size_t i = 0; i < tokens.size(); ++i) {
@@ -139,6 +222,15 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
             return Napi::Number::From(info.Env(), llama_n_ctx_train(model));
         }
 
+        Napi::Value GetEmbeddingVectorSize(const Napi::CallbackInfo& info) {
+            if (disposed) {
+                Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
+                return info.Env().Undefined();
+            }
+
+            return Napi::Number::From(info.Env(), llama_n_embd(model));
+        }
+
         Napi::Value GetTotalSize(const Napi::CallbackInfo& info) {
             if (disposed) {
                 Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
@@ -162,7 +254,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
                 return info.Env().Undefined();
             }
-            
+
             char model_desc[128];
             int actual_length = llama_model_desc(model, model_desc, sizeof(model_desc));
 
@@ -175,7 +267,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_bos(model));
+            return getNapiControlToken(info, model, llama_token_bos(model));
         }
         Napi::Value TokenEos(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -183,7 +275,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_eos(model));
+            return getNapiControlToken(info, model, llama_token_eos(model));
         }
         Napi::Value TokenNl(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -191,7 +283,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_nl(model));
+            return getNapiToken(info, model, llama_token_nl(model));
         }
         Napi::Value PrefixToken(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -199,7 +291,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_prefix(model));
+            return getNapiControlToken(info, model, llama_token_prefix(model));
         }
         Napi::Value MiddleToken(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -207,7 +299,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_middle(model));
+            return getNapiControlToken(info, model, llama_token_middle(model));
         }
         Napi::Value SuffixToken(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -215,7 +307,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_suffix(model));
+            return getNapiControlToken(info, model, llama_token_suffix(model));
         }
         Napi::Value EotToken(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -223,7 +315,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                 return info.Env().Undefined();
             }
 
-            return Napi::Number::From(info.Env(), llama_token_eot(model));
+            return getNapiControlToken(info, model, llama_token_eot(model));
         }
         Napi::Value GetTokenString(const Napi::CallbackInfo& info) {
             if (disposed) {
@@ -244,6 +336,29 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
             return Napi::String::New(info.Env(), ss.str());
         }
 
+        Napi::Value GetTokenType(const Napi::CallbackInfo& info) {
+            if (disposed) {
+                Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
+                return info.Env().Undefined();
+            }
+
+            if (info[0].IsNumber() == false) {
+                return Napi::Number::From(info.Env(), int32_t(LLAMA_TOKEN_TYPE_UNDEFINED));
+            }
+
+            int token = info[0].As<Napi::Number>().Int32Value();
+            auto tokenType = llama_token_get_type(model, token);
+
+            return Napi::Number::From(info.Env(), int32_t(tokenType));
+        }
+        Napi::Value ShouldPrependBosToken(const Napi::CallbackInfo& info) {
+            const int addBos = llama_add_bos_token(model);
+
+            bool shouldPrependBos = addBos != -1 ? bool(addBos) : (llama_vocab_type(model) == LLAMA_VOCAB_TYPE_SPM);
+
+            return Napi::Boolean::New(info.Env(), shouldPrependBos);
+        }
+
         static void init(Napi::Object exports) {
             exports.Set(
                 "AddonModel",
@@ -254,6 +369,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                         InstanceMethod("tokenize", &AddonModel::Tokenize),
                         InstanceMethod("detokenize", &AddonModel::Detokenize),
                         InstanceMethod("getTrainContextSize", &AddonModel::GetTrainContextSize),
+                        InstanceMethod("getEmbeddingVectorSize", &AddonModel::GetEmbeddingVectorSize),
                         InstanceMethod("getTotalSize", &AddonModel::GetTotalSize),
                         InstanceMethod("getTotalParameters", &AddonModel::GetTotalParameters),
                         InstanceMethod("getModelDescription", &AddonModel::GetModelDescription),
@@ -265,7 +381,9 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
                         InstanceMethod("suffixToken", &AddonModel::SuffixToken),
                         InstanceMethod("eotToken", &AddonModel::EotToken),
                         InstanceMethod("getTokenString", &AddonModel::GetTokenString),
-                        InstanceMethod("dispose", &AddonModel::Dispose)
+                        InstanceMethod("getTokenType", &AddonModel::GetTokenType),
+                        InstanceMethod("shouldPrependBosToken", &AddonModel::ShouldPrependBosToken),
+                        InstanceMethod("dispose", &AddonModel::Dispose),
                     }
                 )
             );
@@ -352,29 +470,23 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
             context_params.seed = -1;
             context_params.n_ctx = 4096;
             context_params.n_threads = 6;
-            context_params.n_threads_batch == -1 ? context_params.n_threads : context_params.n_threads_batch;
+            context_params.n_threads_batch = context_params.n_threads;
 
             if (info.Length() > 1 && info[1].IsObject()) {
                 Napi::Object options = info[1].As<Napi::Object>();
 
-                if (options.Has("seed")) {
-                    context_params.seed = options.Get("seed").As<Napi::Number>().Int32Value();
+                if (options.Has("noSeed")) {
+                    context_params.seed = time(NULL);
+                } else if (options.Has("seed")) {
+                    context_params.seed = options.Get("seed").As<Napi::Number>().Uint32Value();
                 }
 
                 if (options.Has("contextSize")) {
-                    context_params.n_ctx = options.Get("contextSize").As<Napi::Number>().Int32Value();
+                    context_params.n_ctx = options.Get("contextSize").As<Napi::Number>().Uint32Value();
                 }
 
                 if (options.Has("batchSize")) {
-                    context_params.n_batch = options.Get("batchSize").As<Napi::Number>().Int32Value();
-                }
-
-                if (options.Has("f16Kv")) {
-                    context_params.f16_kv = options.Get("f16Kv").As<Napi::Boolean>().Value();
-                }
-
-                if (options.Has("logitsAll")) {
-                    context_params.logits_all = options.Get("logitsAll").As<Napi::Boolean>().Value();
+                    context_params.n_batch = options.Get("batchSize").As<Napi::Number>().Uint32Value();
                 }
 
                 if (options.Has("embedding")) {
@@ -382,8 +494,11 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
                 }
 
                 if (options.Has("threads")) {
-                    context_params.n_threads = options.Get("threads").As<Napi::Number>().Int32Value();
-                    context_params.n_threads_batch == -1 ? context_params.n_threads : context_params.n_threads_batch;
+                    const auto n_threads = options.Get("threads").As<Napi::Number>().Uint32Value();
+                    const auto resolved_n_threads = n_threads == 0 ? std::thread::hardware_concurrency() : n_threads;
+
+                    context_params.n_threads = resolved_n_threads;
+                    context_params.n_threads_batch = resolved_n_threads;
                 }
             }
 
@@ -533,6 +648,41 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
         Napi::Value DecodeBatch(const Napi::CallbackInfo& info);
         Napi::Value SampleToken(const Napi::CallbackInfo& info);
 
+        Napi::Value AcceptGrammarEvaluationStateToken(const Napi::CallbackInfo& info) {
+            AddonGrammarEvaluationState* grammar_evaluation_state =
+                Napi::ObjectWrap<AddonGrammarEvaluationState>::Unwrap(info[0].As<Napi::Object>());
+            llama_token tokenId = info[1].As<Napi::Number>().Int32Value();
+
+            if ((grammar_evaluation_state)->grammar != nullptr) {
+                llama_grammar_accept_token(ctx, (grammar_evaluation_state)->grammar, tokenId);
+            }
+
+            return info.Env().Undefined();
+        }
+
+        Napi::Value GetEmbedding(const Napi::CallbackInfo& info) {
+            if (disposed) {
+                Napi::Error::New(info.Env(), "Context is disposed").ThrowAsJavaScriptException();
+                return info.Env().Undefined();
+            }
+
+            const int n_embd = llama_n_embd(model->model);
+            const auto* embeddings = llama_get_embeddings(ctx);
+
+            Napi::Float64Array result = Napi::Float64Array::New(info.Env(), n_embd);
+            for (size_t i = 0; i < n_embd; ++i) {
+                result[i] = embeddings[i];
+            }
+
+            return result;
+        }
+
+        Napi::Value PrintTimings(const Napi::CallbackInfo& info) {
+            llama_print_timings(ctx);
+            llama_reset_timings(ctx);
+            return info.Env().Undefined();
+        }
+
         static void init(Napi::Object exports) {
             exports.Set(
                 "AddonContext",
@@ -548,7 +698,10 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
                         InstanceMethod("shiftSequenceTokenCells", &AddonContext::ShiftSequenceTokenCells),
                         InstanceMethod("decodeBatch", &AddonContext::DecodeBatch),
                         InstanceMethod("sampleToken", &AddonContext::SampleToken),
-                        InstanceMethod("dispose", &AddonContext::Dispose)
+                        InstanceMethod("acceptGrammarEvaluationStateToken", &AddonContext::AcceptGrammarEvaluationStateToken),
+                        InstanceMethod("getEmbedding", &AddonContext::GetEmbedding),
+                        InstanceMethod("printTimings", &AddonContext::PrintTimings),
+                        InstanceMethod("dispose", &AddonContext::Dispose),
                     }
                 )
             );
@@ -610,6 +763,7 @@ class AddonContextSampleTokenWorker : Napi::AsyncWorker, Napi::Promise::Deferred
         bool use_grammar = false;
         llama_token result;
         float temperature = 0.0f;
+        float min_p = 0;
         int32_t top_k = 40;
         float top_p = 0.95f;
         float repeat_penalty = 1.10f;  // 1.0 = disabled
@@ -633,6 +787,10 @@ class AddonContextSampleTokenWorker : Napi::AsyncWorker, Napi::Promise::Deferred
                     temperature = options.Get("temperature").As<Napi::Number>().FloatValue();
                 }
 
+                if (options.Has("minP")) {
+                    min_p = options.Get("minP").As<Napi::Number>().FloatValue();
+                }
+
                 if (options.Has("topK")) {
                     top_k = options.Get("topK").As<Napi::Number>().Int32Value();
                 }
@@ -734,6 +892,7 @@ class AddonContextSampleTokenWorker : Napi::AsyncWorker, Napi::Promise::Deferred
                 llama_sample_tail_free(ctx->ctx, &candidates_p, tfs_z, min_keep);
                 llama_sample_typical(ctx->ctx, &candidates_p, typical_p, min_keep);
                 llama_sample_top_p(ctx->ctx, &candidates_p, resolved_top_p, min_keep);
+                llama_sample_min_p(ctx->ctx, &candidates_p, min_p, min_keep);
                 llama_sample_temp(ctx->ctx, &candidates_p, temperature);
                 new_token_id = llama_sample_token(ctx->ctx, &candidates_p);
             }
@@ -764,15 +923,144 @@ Napi::Value systemInfo(const Napi::CallbackInfo& info) {
     return Napi::String::From(info.Env(), llama_print_system_info());
 }
 
+int addonGetGgmlLogLevelNumber(ggml_log_level level) {
+    switch (level) {
+        case GGML_LOG_LEVEL_ERROR: return 2;
+        case GGML_LOG_LEVEL_WARN: return 3;
+        case GGML_LOG_LEVEL_INFO: return 4;
+        case GGML_LOG_LEVEL_DEBUG: return 5;
+    }
+
+    return 1;
+}
+
+void addonCallJsLogCallback(
+    Napi::Env env, Napi::Function callback, AddonThreadSafeLogCallbackFunctionContext* context, addon_logger_log* data
+) {
+    bool called = false;
+
+    if (env != nullptr && callback != nullptr && addonJsLoggerCallbackSet) {
+        try {
+            callback.Call({
+                Napi::Number::New(env, data->logLevelNumber),
+                Napi::String::New(env, data->stringStream->str()),
+            });
+            called = true;
+        } catch (const Napi::Error& e) {
+            called = false;
+        }
+    }
+    
+    if (!called && data != nullptr) {
+        if (data->logLevelNumber == 2) {
+            fputs(data->stringStream->str().c_str(), stderr);
+            fflush(stderr);
+        } else {
+            fputs(data->stringStream->str().c_str(), stdout);
+            fflush(stdout);
+        }
+    }
+
+    if (data != nullptr) {
+        delete data->stringStream;
+        delete data;
+    }
+}
+
+static void addonLlamaCppLogCallback(ggml_log_level level, const char* text, void* user_data) {
+    int logLevelNumber = addonGetGgmlLogLevelNumber(level);
+
+    if (logLevelNumber > addonLoggerLogLevel) {
+        return;
+    }
+
+    if (addonJsLoggerCallbackSet) {
+        std::stringstream* stringStream = new std::stringstream();
+        if (text != nullptr) {
+            *stringStream << text;
+        }
+
+        addon_logger_log* data = new addon_logger_log {
+            logLevelNumber,
+            stringStream,
+        };
+
+        auto status = addonThreadSafeLoggerCallback.NonBlockingCall(data);
+
+        if (status == napi_ok) {
+            return;
+        }
+    }
+
+    if (level == 2) {
+        fputs(text, stderr);
+        fflush(stderr);
+    } else {
+        fputs(text, stdout);
+        fflush(stdout);
+    }
+}
+
+Napi::Value setLogger(const Napi::CallbackInfo& info) {
+    if (info.Length() < 1 || !info[0].IsFunction()) {
+        if (addonJsLoggerCallbackSet) {
+            addonJsLoggerCallbackSet = false;
+            addonThreadSafeLoggerCallback.Release();
+        }
+
+        return info.Env().Undefined();
+    }
+
+    auto addonLoggerJSCallback = info[0].As<Napi::Function>();
+    AddonThreadSafeLogCallbackFunctionContext* context = new Napi::Reference<Napi::Value>(Napi::Persistent(info.This()));
+    addonThreadSafeLoggerCallback = AddonThreadSafeLogCallbackFunction::New(
+        info.Env(),
+        addonLoggerJSCallback,
+        "loggerCallback",
+        0,
+        1,
+        context,
+        [](Napi::Env, void*, AddonThreadSafeLogCallbackFunctionContext* ctx) {
+            addonJsLoggerCallbackSet = false;
+
+            delete ctx;
+        }
+    );
+    addonJsLoggerCallbackSet = true;
+
+    // prevent blocking the main node process from exiting due to active resources
+    addonThreadSafeLoggerCallback.Unref(info.Env());
+
+    return info.Env().Undefined();
+}
+
+Napi::Value setLoggerLogLevel(const Napi::CallbackInfo& info) {
+    if (info.Length() < 1 || !info[0].IsNumber()) {
+        addonLoggerLogLevel = 5;
+
+        return info.Env().Undefined();
+    }
+
+    addonLoggerLogLevel = info[0].As<Napi::Number>().Int32Value();
+
+    return info.Env().Undefined();
+}
+
 Napi::Object registerCallback(Napi::Env env, Napi::Object exports) {
-    llama_backend_init(false);
+    llama_backend_init();
     exports.DefineProperties({
         Napi::PropertyDescriptor::Function("systemInfo", systemInfo),
+        Napi::PropertyDescriptor::Function("setLogger", setLogger),
+        Napi::PropertyDescriptor::Function("setLoggerLogLevel", setLoggerLogLevel),
+        Napi::PropertyDescriptor::Function("getGpuVramInfo", getGpuVramInfo),
     });
     AddonModel::init(exports);
     AddonGrammar::init(exports);
     AddonGrammarEvaluationState::init(exports);
     AddonContext::init(exports);
+
+    llama_log_set(addonLlamaCppLogCallback, nullptr);
+
     return exports;
 }
 

package/llama/binariesGithubRelease.json

@@ -1,3 +1,3 @@
 {
-    "release": "b1567"
+    "release": "b2174"
 }

package/llama/gpuInfo/cuda-gpu-info.cu

@@ -0,0 +1,99 @@
+#include <stddef.h>
+
+#if defined(GPU_INFO_USE_HIPBLAS)
+#include <hip/hip_runtime.h>
+#include <hipblas/hipblas.h>
+#define cudaGetDevice hipGetDevice
+#define cudaGetDeviceCount hipGetDeviceCount
+#define cudaGetErrorString hipGetErrorString
+#define cudaMemGetInfo hipMemGetInfo
+#define cudaSetDevice hipSetDevice
+#define cudaSuccess hipSuccess
+#else
+#include <cuda_runtime.h>
+#include <cuda.h>
+#endif
+
+
+typedef void (*gpuInfoErrorLogCallback_t)(const char* message);
+
+bool gpuInfoSetCudaDevice(const int device, gpuInfoErrorLogCallback_t errorLogCallback) {
+    int current_device;
+    auto getDeviceResult = cudaGetDevice(&current_device);
+
+    if (getDeviceResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(getDeviceResult));
+        return false;
+    }
+
+    if (device == current_device) {
+        return true;
+    }
+
+    const auto setDeviceResult = cudaSetDevice(device);
+
+    if (setDeviceResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(setDeviceResult));
+        return false;
+    }
+
+    return true;
+}
+
+bool gpuInfoGetCudaDeviceInfo(int device, size_t * total, size_t * used, gpuInfoErrorLogCallback_t errorLogCallback) {
+    gpuInfoSetCudaDevice(device, errorLogCallback);
+
+    size_t freeMem;
+    size_t totalMem;
+    auto getMemInfoResult = cudaMemGetInfo(&freeMem, &totalMem);
+
+    if (getMemInfoResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(getMemInfoResult));
+        return false;
+    }
+
+    *total = totalMem;
+    *used = totalMem - freeMem;
+
+    return true;
+}
+
+int gpuInfoGetCudaDeviceCount(gpuInfoErrorLogCallback_t errorLogCallback) {
+    int deviceCount;
+    auto getDeviceCountResult = cudaGetDeviceCount(&deviceCount);
+
+    if (getDeviceCountResult != cudaSuccess) {
+        errorLogCallback(cudaGetErrorString(getDeviceCountResult));
+        return -1;
+    }
+
+    return deviceCount;
+}
+
+bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoErrorLogCallback_t errorLogCallback) {
+    int deviceCount = gpuInfoGetCudaDeviceCount(errorLogCallback);
+
+    if (deviceCount < 0) {
+        return false;
+    }
+
+    size_t usedMem = 0;
+    size_t totalMem = 0;
+
+    for (int i = 0; i < deviceCount; i++) {
+        size_t deviceUsedMem;
+        size_t deviceTotalMem;
+
+        if (!gpuInfoGetCudaDeviceInfo(i, &deviceTotalMem, &deviceUsedMem, errorLogCallback)) {
+            return false;
+        }
+
+        usedMem += deviceUsedMem;
+        totalMem += deviceTotalMem;
+    }
+
+    *total = totalMem;
+    *used = usedMem;
+
+    return true;
+}

package/llama/gpuInfo/cuda-gpu-info.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <stddef.h>
+
+typedef void (*gpuInfoErrorLogCallback_t)(const char* message);
+
+bool gpuInfoGetTotalCudaDevicesInfo(size_t * total, size_t * used, gpuInfoErrorLogCallback_t errorLogCallback);

package/llama/gpuInfo/metal-gpu-info.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include <stdint.h>
+
+void get_metal_gpu_info(uint64_t * total, uint64_t * used);

package/llama/gpuInfo/metal-gpu-info.mm

@@ -0,0 +1,17 @@
+#include <stdint.h>
+#import <Metal/Metal.h>
+
+void get_metal_gpu_info(uint64_t * total, uint64_t * used) {
+    id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+
+    if (device) {
+        *total = device.recommendedMaxWorkingSetSize;
+        *used = device.currentAllocatedSize;
+    } else {
+        *total = 0;
+        *used = 0;
+    }
+
+    [device release];
+    device = nil;
+}

package/llama/llama.cpp.info.json

@@ -0,0 +1,4 @@
+{
+    "tag": "b2174",
+    "llamaCppGithubRepo": "ggerganov/llama.cpp"
+}

package/llamaBins/linux-arm64/.buildMetadata.json

@@ -0,0 +1 @@
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"arm64","computeLayers":{"metal":false,"cuda":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2174"}}}

package/llamaBins/linux-armv7l/.buildMetadata.json

@@ -0,0 +1 @@
+{"buildOptions":{"customCmakeOptions":{},"progressLogs":true,"platform":"linux","arch":"armv7l","computeLayers":{"metal":false,"cuda":false},"llamaCpp":{"repo":"ggerganov/llama.cpp","release":"b2174"}}}