@fugood/llama.node 0.3.10 → 0.3.12

package/lib/binding.ts

@@ -150,6 +150,7 @@ export interface LlamaContext {
   getLoadedLoraAdapters(): { path: string; scaled: number }[]
   // static
   loadModelInfo(path: string, skip: string[]): Promise<Object>
+  toggleNativeLog(enable: boolean, callback: (level: string, text: string) => void): void
 }
 
 export interface Module {

package/lib/index.js

@@ -23,14 +23,39 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = void 0;
+exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = void 0;
+exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
 __exportStar(require("./binding"), exports);
 const mods = {};
+const logListeners = [];
+const logCallback = (level, text) => {
+    logListeners.forEach((listener) => listener(level, text));
+};
+let logEnabled = false;
+const refreshNativeLogSetup = () => {
+    Object.entries(mods).forEach(([, mod]) => {
+        mod.LlamaContext.toggleNativeLog(logEnabled, logCallback);
+    });
+};
+const toggleNativeLog = (enable) => __awaiter(void 0, void 0, void 0, function* () {
+    logEnabled = enable;
+    refreshNativeLogSetup();
+});
+exports.toggleNativeLog = toggleNativeLog;
+function addNativeLogListener(listener) {
+    logListeners.push(listener);
+    return {
+        remove: () => {
+            logListeners.splice(logListeners.indexOf(listener), 1);
+        },
+    };
+}
 const loadModel = (options) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;
     const variant = (_a = options.lib_variant) !== null && _a !== void 0 ? _a : 'default';
     (_b = mods[variant]) !== null && _b !== void 0 ? _b : (mods[variant] = yield (0, binding_1.loadModule)(options.lib_variant));
+    refreshNativeLogSetup();
     return new mods[variant].LlamaContext(options);
 });
 exports.loadModel = loadModel;
@@ -46,6 +71,7 @@ const loadLlamaModelInfo = (path) => __awaiter(void 0, void 0, void 0, function*
     var _a;
     const variant = 'default';
     (_a = mods[variant]) !== null && _a !== void 0 ? _a : (mods[variant] = yield (0, binding_1.loadModule)(variant));
+    refreshNativeLogSetup();
     return mods[variant].LlamaContext.loadModelInfo(path, modelInfoSkip);
 });
 exports.loadLlamaModelInfo = loadLlamaModelInfo;

package/lib/index.ts

@@ -9,11 +9,42 @@ export interface LlamaModelOptionsExtended extends LlamaModelOptions {
 
 const mods: { [key: string]: Module } = {}
 
+const logListeners: Array<(level: string, text: string) => void> = []
+
+const logCallback = (level: string, text: string) => {
+  logListeners.forEach((listener) => listener(level, text))
+}
+
+let logEnabled = false
+
+const refreshNativeLogSetup = () => {
+  Object.entries(mods).forEach(([, mod]) => {
+    mod.LlamaContext.toggleNativeLog(logEnabled, logCallback)
+  })
+}
+
+export const toggleNativeLog = async (enable: boolean) => {
+  logEnabled = enable
+  refreshNativeLogSetup()
+}
+
+export function addNativeLogListener(
+  listener: (level: string, text: string) => void,
+): { remove: () => void } {
+  logListeners.push(listener)
+  return {
+    remove: () => {
+      logListeners.splice(logListeners.indexOf(listener), 1)
+    },
+  }
+}
+
 export const loadModel = async (
   options: LlamaModelOptionsExtended,
 ): Promise<LlamaContext> => {
   const variant = options.lib_variant ?? 'default'
   mods[variant] ??= await loadModule(options.lib_variant)
+  refreshNativeLogSetup()
   return new mods[variant].LlamaContext(options)
 }
 
@@ -30,5 +61,6 @@ const modelInfoSkip = [
 export const loadLlamaModelInfo = async (path: string): Promise<Object> => {
   const variant = 'default'
   mods[variant] ??= await loadModule(variant)
+  refreshNativeLogSetup()
   return mods[variant].LlamaContext.loadModelInfo(path, modelInfoSkip)
 }

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.3.10",
-  "description": "Llama.cpp for Node.js",
+  "version": "0.3.12",
+  "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
     "test": "jest",
@@ -22,7 +22,9 @@
     "llama",
     "llm",
     "ai",
-    "genai"
+    "genai",
+    "Local LLM",
+    "llama.cpp"
   ],
   "author": "Hans <hans.chen@bricks.tools>",
   "license": "MIT",

package/src/LlamaContext.cpp

@@ -120,6 +120,9 @@ void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
            "release", static_cast<napi_property_attributes>(napi_enumerable)),
        StaticMethod<&LlamaContext::ModelInfo>(
            "loadModelInfo",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       StaticMethod<&LlamaContext::ToggleNativeLog>(
+           "toggleNativeLog",
            static_cast<napi_property_attributes>(napi_enumerable))});
   Napi::FunctionReference *constructor = new Napi::FunctionReference();
   *constructor = Napi::Persistent(func);
@@ -278,6 +281,46 @@ bool validateModelChatTemplate(const struct llama_model * model, const bool use_
   return common_chat_verify_template(tmpl, use_jinja);
 }
 
+static Napi::FunctionReference _log_callback;
+
+// toggleNativeLog(enable: boolean, callback: (log: string) => void): void
+void LlamaContext::ToggleNativeLog(const Napi::CallbackInfo &info) {
+  bool enable = info[0].ToBoolean().Value();
+  if (enable) {
+    _log_callback.Reset(info[1].As<Napi::Function>());
+    
+    llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
+      llama_log_callback_default(level, text, user_data);
+
+      std::string level_str = "";
+      if (level == GGML_LOG_LEVEL_ERROR) {
+        level_str = "error";
+      } else if (level == GGML_LOG_LEVEL_INFO) {
+        level_str = "info";
+      } else if (level == GGML_LOG_LEVEL_WARN) {
+        level_str = "warn";
+      }
+
+      if (_log_callback.IsEmpty()) {
+        return;
+      }
+      try {
+        Napi::Env env = _log_callback.Env();
+        Napi::HandleScope scope(env);
+        _log_callback.Call({
+          Napi::String::New(env, level_str),
+          Napi::String::New(env, text)
+        });
+      } catch (const std::exception &e) {
+        // printf("Error calling log callback: %s\n", e.what());
+      }
+    }, nullptr);
+  } else {
+    _log_callback.Reset();
+    llama_log_set(llama_log_callback_default, nullptr);
+  }
+}
+
 // getModelInfo(): object
 Napi::Value LlamaContext::GetModelInfo(const Napi::CallbackInfo &info) {
   char desc[1024];
@@ -312,8 +355,7 @@ Napi::Value LlamaContext::GetModelInfo(const Napi::CallbackInfo &info) {
   defaultCaps.Set("parallelToolCalls", _templates.template_default->original_caps().supports_parallel_tool_calls);
   defaultCaps.Set("toolCallId", _templates.template_default->original_caps().supports_tool_call_id);
   minja.Set("defaultCaps", defaultCaps);
-  Napi::Object toolUse = Napi::Object::New(info.Env());
-  toolUse.Set("toolUse", validateModelChatTemplate(model, true, "tool_use"));
+  minja.Set("toolUse", validateModelChatTemplate(model, true, "tool_use"));
   if (_templates.template_tool_use) {
     Napi::Object toolUseCaps = Napi::Object::New(info.Env());
     toolUseCaps.Set("tools", _templates.template_tool_use->original_caps().supports_tools);
@@ -322,13 +364,15 @@ Napi::Value LlamaContext::GetModelInfo(const Napi::CallbackInfo &info) {
     toolUseCaps.Set("systemRole", _templates.template_tool_use->original_caps().supports_system_role);
     toolUseCaps.Set("parallelToolCalls", _templates.template_tool_use->original_caps().supports_parallel_tool_calls);
     toolUseCaps.Set("toolCallId", _templates.template_tool_use->original_caps().supports_tool_call_id);
-    toolUse.Set("toolUseCaps", toolUseCaps);
+    minja.Set("toolUseCaps", toolUseCaps);
   }
-  minja.Set("toolUse", toolUse);
   chatTemplates.Set("minja", minja);
   details.Set("chatTemplates", chatTemplates);
 
   details.Set("metadata", metadata);
+
+  // Deprecated: use chatTemplates.llamaChat instead
+  details.Set("isChatTemplateSupported", validateModelChatTemplate(_sess->model(), false, ""));
   return details;
 }
 

package/src/LlamaContext.h

@@ -5,6 +5,7 @@ class LlamaCompletionWorker;
 class LlamaContext : public Napi::ObjectWrap<LlamaContext> {
 public:
   LlamaContext(const Napi::CallbackInfo &info);
+  static void ToggleNativeLog(const Napi::CallbackInfo &info);
   static Napi::Value ModelInfo(const Napi::CallbackInfo& info);
   static void Init(Napi::Env env, Napi::Object &exports);