@fugood/llama.node 0.3.4 → 0.3.5

package/CMakeLists.txt

@@ -7,7 +7,7 @@ project (llama-node)
 set(CMAKE_CXX_STANDARD 17)
 
 execute_process(COMMAND
-  git apply ${CMAKE_CURRENT_SOURCE_DIR}/scripts/ggml-cpu-CMakeLists.txt.patch
+  git apply ${CMAKE_CURRENT_SOURCE_DIR}/scripts/llama.cpp.patch
   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 

package/lib/binding.ts

@@ -16,6 +16,7 @@ export type LlamaModelOptions = {
   n_gpu_layers?: number
   use_mlock?: boolean
   use_mmap?: boolean
+  vocab_only?: boolean
 }
 
 export type LlamaCompletionOptions = {
@@ -53,6 +54,16 @@ export type LlamaCompletionResult = {
   tokens_predicted: number
   tokens_evaluated: number
   truncated: boolean
+  timings: {
+    prompt_n: number
+    prompt_ms: number
+    prompt_per_token_ms: number
+    prompt_per_second: number
+    predicted_n: number
+    predicted_ms: number
+    predicted_per_token_ms: number
+    predicted_per_second: number
+  }
 }
 
 export type LlamaCompletionToken = {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "description": "Llama.cpp for Node.js",
   "main": "lib/index.js",
   "scripts": {

package/src/LlamaCompletionWorker.cpp

@@ -159,6 +159,22 @@ void LlamaCompletionWorker::OnOK() {
              Napi::Boolean::New(Napi::AsyncWorker::Env(), _result.truncated));
   result.Set("text",
              Napi::String::New(Napi::AsyncWorker::Env(), _result.text.c_str()));
+
+  auto ctx = _sess->context();
+  const auto timings_token = llama_perf_context(ctx);
+
+  auto timingsResult = Napi::Object::New(Napi::AsyncWorker::Env());
+  timingsResult.Set("prompt_n", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.n_p_eval));
+  timingsResult.Set("prompt_ms", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.t_p_eval_ms));
+  timingsResult.Set("prompt_per_token_ms", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.t_p_eval_ms / timings_token.n_p_eval));
+  timingsResult.Set("prompt_per_second", Napi::Number::New(Napi::AsyncWorker::Env(), 1e3 / timings_token.t_p_eval_ms * timings_token.n_p_eval));
+  timingsResult.Set("predicted_n", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.n_eval));
+  timingsResult.Set("predicted_ms", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.t_eval_ms));
+  timingsResult.Set("predicted_per_token_ms", Napi::Number::New(Napi::AsyncWorker::Env(), timings_token.t_eval_ms / timings_token.n_eval));
+  timingsResult.Set("predicted_per_second", Napi::Number::New(Napi::AsyncWorker::Env(), 1e3 / timings_token.t_eval_ms * timings_token.n_eval));
+
+  result.Set("timings", timingsResult);
+  
   Napi::Promise::Deferred::Resolve(result);
 }
 

package/src/LlamaContext.cpp

@@ -76,6 +76,11 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
     Napi::TypeError::New(env, "Model is required").ThrowAsJavaScriptException();
   }
 
+  params.vocab_only = get_option<bool>(options, "vocab_only", false);
+  if (params.vocab_only) {
+    params.warmup = false;
+  }
+
   params.n_ctx = get_option<int32_t>(options, "n_ctx", 512);
   params.n_batch = get_option<int32_t>(options, "n_batch", 2048);
   params.embedding = get_option<bool>(options, "embedding", false);