@fugood/llama.node 1.4.4 → 1.4.6

package/lib/binding.ts

@@ -309,6 +309,45 @@ export type BackendDeviceInfo = {
   metadata?: Record<string, any>
 }
 
+export type BenchResult = {
+  /** Maximum KV cache size */
+  nKvMax: number
+  /** Batch size */
+  nBatch: number
+  /** Micro-batch size */
+  nUBatch: number
+  /** Flash attention type (0=disabled, 1=enabled, 2=auto) */
+  flashAttn: number
+  /** Whether prompt processing is shared */
+  isPpShared: boolean
+  /** Number of GPU layers */
+  nGpuLayers: number
+  /** Number of threads */
+  nThreads: number
+  /** Number of threads for batch processing */
+  nThreadsBatch: number
+  /** Prompt processing tokens count */
+  pp: number
+  /** Text generation tokens count */
+  tg: number
+  /** Parallel level */
+  pl: number
+  /** KV cache used */
+  nKv: number
+  /** Time for prompt processing (ms) */
+  tPp: number
+  /** Speed of prompt processing (tokens/sec) */
+  speedPp: number
+  /** Time for text generation (ms) */
+  tTg: number
+  /** Speed of text generation (tokens/sec) */
+  speedTg: number
+  /** Total time (ms) */
+  t: number
+  /** Overall speed (tokens/sec) */
+  speed: number
+}
+
 export type ModelInfo = {
   desc: string
   nEmbd: number
@@ -573,6 +612,16 @@ export interface LlamaContext {
    */
   clearCache(clearData?: boolean): void
 
+  /**
+   * Run a benchmark to measure model performance
+   * @param pp Number of tokens to process for prompt processing benchmark
+   * @param tg Number of tokens to generate for text generation benchmark
+   * @param pl Parallel level (number of sequences)
+   * @param nr Number of repetitions
+   * @returns Benchmark results
+   */
+  bench(pp: number, tg: number, pl: number, nr: number): Promise<BenchResult>
+
   // static
   loadModelInfo(path: string, skip: string[]): Promise<GGUFModelInfo>
   toggleNativeLog(

package/lib/index.js

@@ -204,6 +204,19 @@ class LlamaContextWrapper {
     clearCache(clearData) {
         this.ctx.clearCache(clearData);
     }
+    /**
+     * Run a benchmark to measure model performance
+     * @param pp Number of tokens to process for prompt processing benchmark
+     * @param tg Number of tokens to generate for text generation benchmark
+     * @param pl Parallel level (number of sequences)
+     * @param nr Number of repetitions
+     * @returns Benchmark results including timing and speed metrics
+     */
+    bench(pp, tg, pl, nr) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.ctx.bench(pp, tg, pl, nr);
+        });
+    }
 }
 const loadModel = (options, onProgress) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;

package/lib/index.ts

@@ -16,6 +16,7 @@ import type {
   JinjaFormattedChatResult,
   Tool,
   GGUFModelInfo,
+  BenchResult,
 } from './binding'
 import { BUILD_NUMBER, BUILD_COMMIT } from './version'
 import { LlamaParallelAPI } from './parallel'
@@ -309,6 +310,18 @@ class LlamaContextWrapper {
   clearCache(clearData?: boolean): void {
     this.ctx.clearCache(clearData)
   }
+
+  /**
+   * Run a benchmark to measure model performance
+   * @param pp Number of tokens to process for prompt processing benchmark
+   * @param tg Number of tokens to generate for text generation benchmark
+   * @param pl Parallel level (number of sequences)
+   * @param nr Number of repetitions
+   * @returns Benchmark results including timing and speed metrics
+   */
+  async bench(pp: number, tg: number, pl: number, nr: number): Promise<BenchResult> {
+    return this.ctx.bench(pp, tg, pl, nr)
+  }
 }
 
 export const loadModel = async (

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.4.4",
+  "version": "1.4.6",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,20 +72,20 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-darwin-arm64": "1.4.4",
-    "@fugood/node-llama-darwin-x64": "1.4.4",
-    "@fugood/node-llama-linux-arm64": "1.4.4",
-    "@fugood/node-llama-linux-arm64-cuda": "1.4.4",
-    "@fugood/node-llama-linux-arm64-snapdragon": "1.4.4",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.4.4",
-    "@fugood/node-llama-linux-x64": "1.4.4",
-    "@fugood/node-llama-linux-x64-cuda": "1.4.4",
-    "@fugood/node-llama-linux-x64-vulkan": "1.4.4",
-    "@fugood/node-llama-win32-arm64": "1.4.4",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.4.4",
-    "@fugood/node-llama-win32-x64": "1.4.4",
-    "@fugood/node-llama-win32-x64-cuda": "1.4.4",
-    "@fugood/node-llama-win32-x64-vulkan": "1.4.4"
+    "@fugood/node-llama-darwin-arm64": "1.4.6",
+    "@fugood/node-llama-darwin-x64": "1.4.6",
+    "@fugood/node-llama-linux-arm64": "1.4.6",
+    "@fugood/node-llama-linux-arm64-cuda": "1.4.6",
+    "@fugood/node-llama-linux-arm64-snapdragon": "1.4.6",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.4.6",
+    "@fugood/node-llama-linux-x64": "1.4.6",
+    "@fugood/node-llama-linux-x64-cuda": "1.4.6",
+    "@fugood/node-llama-linux-x64-vulkan": "1.4.6",
+    "@fugood/node-llama-win32-arm64": "1.4.6",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.4.6",
+    "@fugood/node-llama-win32-x64": "1.4.6",
+    "@fugood/node-llama-win32-x64-cuda": "1.4.6",
+    "@fugood/node-llama-win32-x64-vulkan": "1.4.6"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/src/LlamaContext.cpp

@@ -203,6 +203,9 @@ void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
            static_cast<napi_property_attributes>(napi_enumerable)),
        InstanceMethod<&LlamaContext::ClearCache>(
            "clearCache",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::Bench>(
+           "bench",
            static_cast<napi_property_attributes>(napi_enumerable))});
   Napi::FunctionReference *constructor = new Napi::FunctionReference();
   *constructor = Napi::Persistent(func);
@@ -1529,3 +1532,69 @@ void LlamaContext::ClearCache(const Napi::CallbackInfo &info) {
 
   _rn_ctx->clearCache(clear_data);
 }
+
+// bench(pp: number, tg: number, pl: number, nr: number): Promise<BenchResult>
+Napi::Value LlamaContext::Bench(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+
+  if (info.Length() < 4) {
+    Napi::TypeError::New(env, "Expected 4 arguments: pp, tg, pl, nr")
+        .ThrowAsJavaScriptException();
+    return env.Undefined();
+  }
+
+  if (!_rn_ctx) {
+    Napi::TypeError::New(env, "Context is disposed").ThrowAsJavaScriptException();
+    return env.Undefined();
+  }
+
+  if (!_rn_ctx->completion) {
+    Napi::TypeError::New(env, "Completion context not initialized")
+        .ThrowAsJavaScriptException();
+    return env.Undefined();
+  }
+
+  int pp = info[0].ToNumber().Int32Value();
+  int tg = info[1].ToNumber().Int32Value();
+  int pl = info[2].ToNumber().Int32Value();
+  int nr = info[3].ToNumber().Int32Value();
+
+  std::string result;
+  try {
+    result = _rn_ctx->completion->bench(pp, tg, pl, nr);
+  } catch (const std::exception &e) {
+    Napi::Error::New(env, e.what()).ThrowAsJavaScriptException();
+    return env.Undefined();
+  }
+
+  // Parse the JSON result and return as object
+  try {
+    auto parsed = json::parse(result);
+    Napi::Object benchResult = Napi::Object::New(env);
+
+    benchResult.Set("nKvMax", Napi::Number::New(env, parsed["n_kv_max"].get<int>()));
+    benchResult.Set("nBatch", Napi::Number::New(env, parsed["n_batch"].get<int>()));
+    benchResult.Set("nUBatch", Napi::Number::New(env, parsed["n_ubatch"].get<int>()));
+    benchResult.Set("flashAttn", Napi::Number::New(env, parsed["flash_attn"].get<int>()));
+    benchResult.Set("isPpShared", Napi::Boolean::New(env, parsed["is_pp_shared"].get<int>() != 0));
+    benchResult.Set("nGpuLayers", Napi::Number::New(env, parsed["n_gpu_layers"].get<int>()));
+    benchResult.Set("nThreads", Napi::Number::New(env, parsed["n_threads"].get<int>()));
+    benchResult.Set("nThreadsBatch", Napi::Number::New(env, parsed["n_threads_batch"].get<int>()));
+    benchResult.Set("pp", Napi::Number::New(env, parsed["pp"].get<int>()));
+    benchResult.Set("tg", Napi::Number::New(env, parsed["tg"].get<int>()));
+    benchResult.Set("pl", Napi::Number::New(env, parsed["pl"].get<int>()));
+    benchResult.Set("nKv", Napi::Number::New(env, parsed["n_kv"].get<int>()));
+    benchResult.Set("tPp", Napi::Number::New(env, parsed["t_pp"].get<double>()));
+    benchResult.Set("speedPp", Napi::Number::New(env, parsed["speed_pp"].get<double>()));
+    benchResult.Set("tTg", Napi::Number::New(env, parsed["t_tg"].get<double>()));
+    benchResult.Set("speedTg", Napi::Number::New(env, parsed["speed_tg"].get<double>()));
+    benchResult.Set("t", Napi::Number::New(env, parsed["t"].get<double>()));
+    benchResult.Set("speed", Napi::Number::New(env, parsed["speed"].get<double>()));
+
+    return benchResult;
+  } catch (const std::exception &e) {
+    Napi::Error::New(env, std::string("Failed to parse benchmark result: ") + e.what())
+        .ThrowAsJavaScriptException();
+    return env.Undefined();
+  }
+}

package/src/LlamaContext.h

@@ -72,6 +72,9 @@ private:
   // Cache management
   void ClearCache(const Napi::CallbackInfo &info);
 
+  // Benchmarking
+  Napi::Value Bench(const Napi::CallbackInfo &info);
+
   std::string _info;
   std::vector<std::string> _used_devices;
   Napi::Object _meta;