@fugood/llama.node 0.0.1-alpha.1 → 0.0.1-alpha.2

package/README.md

@@ -7,16 +7,16 @@ Node binding of [llama.cpp](https://github.com/ggerganov/llama.cpp).
 ## Installation
 
 ```sh
-npm install llama.node
+npm install @fugood/llama.node
 ```
 
 ## Usage
 
 ```js
-import { loadModel } from 'llama.node'
+import { loadModel } from '@fugood/llama.node'
 
 // Initial a Llama context with the model (may take a while)
-const context = loadModel({
+const context = await loadModel({
   model: 'path/to/gguf/model',
   use_mlock: true,
   n_ctx: 2048,

package/lib/binding.js

@@ -1,13 +1,45 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadModule = void 0;
-const loadModule = (variant) => {
+const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         if (variant) {
-            return require(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`);
+            return yield Promise.resolve(`${`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
         }
     }
     catch (_a) { } // ignore errors and try the common path
-    return require(`../bin/${process.platform}/${process.arch}/llama-node.node`);
-};
+    return yield Promise.resolve(`${`../bin/${process.platform}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
+});
 exports.loadModule = loadModule;

package/lib/binding.ts

@@ -41,17 +41,18 @@ export interface LlamaContext {
   stopCompletion(): void
   saveSession(path: string): Promise<void>
   loadSession(path: string): Promise<void>
+  release(): Promise<void>
 }
 
 export interface Module {
   LlamaContext: LlamaContext
 }
 
-export const loadModule = (variant?: string): Module => {
+export const loadModule = async (variant?: string): Promise<Module> => {
   try {
     if (variant) {
-      return require(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`) as Module
+      return await import(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`) as Module
     }
   } catch {} // ignore errors and try the common path
-  return require(`../bin/${process.platform}/${process.arch}/llama-node.node`) as Module
+  return await import(`../bin/${process.platform}/${process.arch}/llama-node.node`) as Module
 }

package/lib/index.js

@@ -13,12 +13,22 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadModel = void 0;
 const binding_1 = require("./binding");
 __exportStar(require("./binding"), exports);
-const loadModel = (options) => {
-    const { LlamaContext } = (0, binding_1.loadModule)(options.lib_variant);
-    return new LlamaContext(options);
-};
+let module = null;
+const loadModel = (options) => __awaiter(void 0, void 0, void 0, function* () {
+    module !== null && module !== void 0 ? module : (module = yield (0, binding_1.loadModule)(options.lib_variant));
+    return new module.LlamaContext(options);
+});
 exports.loadModel = loadModel;

package/lib/index.ts

@@ -1,5 +1,5 @@
 import { loadModule, LlamaModelOptions } from './binding'
-import type { LlamaContext } from './binding'
+import type { Module, LlamaContext } from './binding'
 
 export * from './binding'
 
@@ -7,7 +7,9 @@ export interface LlamaModelOptionsExtended extends LlamaModelOptions {
   lib_variant?: string
 }
 
-export const loadModel = (options: LlamaModelOptionsExtended): LlamaContext => {
-    const { LlamaContext } = loadModule(options.lib_variant)
-    return new LlamaContext(options)
+let module: Module | null = null
+
+export const loadModel = async (options: LlamaModelOptionsExtended): Promise<LlamaContext> => {
+    module ??= await loadModule(options.lib_variant)
+    return new module.LlamaContext(options)
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.0.1-alpha.1",
+  "version": "0.0.1-alpha.2",
   "description": "Llama.cpp for Node.js",
   "main": "lib/index.js",
   "scripts": {
@@ -30,7 +30,7 @@
   },
   "homepage": "https://github.com/mybigday/llama.node#readme",
   "publishConfig": {
-    "registry": "https://registry.npmjs.org"        
+    "registry": "https://registry.npmjs.org"
   },
   "binary": {
     "napi_versions": [

package/src/addons.cpp

@@ -23,6 +23,34 @@ size_t common_part(const std::vector<llama_token> &a,
   return i;
 }
 
+template <typename T>
+constexpr T get_option(const Napi::Object &options, const std::string &name,
+                       const T default_value) {
+  if (options.Has(name) && !options.Get(name).IsUndefined() &&
+      !options.Get(name).IsNull()) {
+    if constexpr (std::is_same<T, std::string>::value) {
+      return options.Get(name).ToString().operator T();
+    } else if constexpr (std::is_same<T, int32_t>::value ||
+                         std::is_same<T, uint32_t>::value ||
+                         std::is_same<T, float>::value ||
+                         std::is_same<T, double>::value) {
+      return options.Get(name).ToNumber().operator T();
+    } else if constexpr (std::is_same<T, bool>::value) {
+      return options.Get(name).ToBoolean().operator T();
+    } else {
+      static_assert(std::is_same<T, std::string>::value ||
+                        std::is_same<T, int32_t>::value ||
+                        std::is_same<T, uint32_t>::value ||
+                        std::is_same<T, float>::value ||
+                        std::is_same<T, double>::value ||
+                        std::is_same<T, bool>::value,
+                    "Unsupported type");
+    }
+  } else {
+    return default_value;
+  }
+}
+
 class LlamaCompletionWorker;
 
 class LlamaContext : public Napi::ObjectWrap<LlamaContext> {
@@ -37,37 +65,21 @@ public:
     }
     auto options = info[0].As<Napi::Object>();
 
-    if (options.Has("model")) {
-      params.model = options.Get("model").ToString();
-    }
-    if (options.Has("embedding")) {
-      params.embedding = options.Get("embedding").ToBoolean();
-    }
-    if (options.Has("n_ctx")) {
-      params.n_ctx = options.Get("n_ctx").ToNumber();
-    }
-    if (options.Has("n_batch")) {
-      params.n_batch = options.Get("n_batch").ToNumber();
-    }
-    if (options.Has("n_threads")) {
-      params.n_threads = options.Get("n_threads").ToNumber();
-    }
-    if (options.Has("n_gpu_layers")) {
-      params.n_gpu_layers = options.Get("n_gpu_layers").ToNumber();
-    }
-    if (options.Has("use_mlock")) {
-      params.use_mlock = options.Get("use_mlock").ToBoolean();
-    }
-    if (options.Has("use_mmap")) {
-      params.use_mmap = options.Get("use_mmap").ToBoolean();
-    }
-    if (options.Has("numa")) {
-      int numa = options.Get("numa").ToNumber();
-      params.numa = static_cast<ggml_numa_strategy>(numa);
-    }
-    if (options.Has("seed")) {
-      params.seed = options.Get("seed").ToNumber();
+    params.model = get_option<std::string>(options, "model", "");
+    if (params.model.empty()) {
+      Napi::TypeError::New(env, "Model is required")
+          .ThrowAsJavaScriptException();
     }
+    params.embedding = get_option<bool>(options, "embedding", false);
+    params.n_ctx = get_option<int32_t>(options, "n_ctx", 512);
+    params.n_batch = get_option<int32_t>(options, "n_batch", 2048);
+    params.n_threads =
+        get_option<int32_t>(options, "n_threads", get_math_cpu_count() / 2);
+    params.n_gpu_layers = get_option<int32_t>(options, "n_gpu_layers", -1);
+    params.use_mlock = get_option<bool>(options, "use_mlock", false);
+    params.use_mmap = get_option<bool>(options, "use_mmap", true);
+    params.numa = static_cast<ggml_numa_strategy>(
+        get_option<uint32_t>(options, "numa", 0));
 
     llama_backend_init();
     llama_numa_init(params.numa);
@@ -99,6 +111,9 @@ public:
              static_cast<napi_property_attributes>(napi_enumerable)),
          InstanceMethod<&LlamaContext::LoadSession>(
              "loadSession",
+             static_cast<napi_property_attributes>(napi_enumerable)),
+         InstanceMethod<&LlamaContext::Release>(
+             "release",
              static_cast<napi_property_attributes>(napi_enumerable))});
     Napi::FunctionReference *constructor = new Napi::FunctionReference();
     *constructor = Napi::Persistent(func);
@@ -127,12 +142,21 @@ public:
 
   std::mutex &getMutex() { return mutex; }
 
+  void Dispose() {
+    std::lock_guard<std::mutex> lock(mutex);
+    compl_worker = nullptr;
+    ctx.reset();
+    tokens.reset();
+    model.reset();
+  }
+
 private:
   Napi::Value GetSystemInfo(const Napi::CallbackInfo &info);
   Napi::Value Completion(const Napi::CallbackInfo &info);
   void StopCompletion(const Napi::CallbackInfo &info);
   Napi::Value SaveSession(const Napi::CallbackInfo &info);
   Napi::Value LoadSession(const Napi::CallbackInfo &info);
+  Napi::Value Release(const Napi::CallbackInfo &info);
 
   gpt_params params;
   LlamaCppModel model{nullptr, llama_free_model};
@@ -270,7 +294,6 @@ protected:
       tokens_predicted += 1;
       n_input = 1;
       if (_has_callback) {
-        // _cb.Call({ Napi::String::New(AsyncWorker::Env(), token) });
         const char *c_token = strdup(token.c_str());
         _tsfn.BlockingCall(c_token, [](Napi::Env env, Napi::Function jsCallback,
                                        const char *value) {
@@ -378,6 +401,26 @@ protected:
   void OnError(const Napi::Error &err) { Reject(err.Value()); }
 };
 
+class DisposeWorker : public Napi::AsyncWorker, public Napi::Promise::Deferred {
+public:
+  DisposeWorker(Napi::Env env, LlamaContext *ctx)
+      : AsyncWorker(env), Deferred(env), ctx_(ctx) {
+    ctx_->Ref();
+  }
+
+  ~DisposeWorker() { ctx_->Unref(); }
+
+protected:
+  void Execute() override { ctx_->Dispose(); }
+
+  void OnOK() override { Resolve(AsyncWorker::Env().Undefined()); }
+
+  void OnError(const Napi::Error &err) override { Reject(err.Value()); }
+
+private:
+  LlamaContext *ctx_;
+};
+
 // getSystemInfo(): string
 Napi::Value LlamaContext::GetSystemInfo(const Napi::CallbackInfo &info) {
   return Napi::String::New(info.Env(), get_system_info(params).c_str());
@@ -396,63 +439,41 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   auto options = info[0].As<Napi::Object>();
 
   gpt_params params;
-  if (options.Has("prompt")) {
-    params.prompt = options.Get("prompt").ToString();
-  } else {
+  params.prompt = get_option<std::string>(options, "prompt", "");
+  if (params.prompt.empty()) {
     Napi::TypeError::New(env, "Prompt is required")
         .ThrowAsJavaScriptException();
   }
-  params.n_predict =
-      options.Has("n_predict") ? options.Get("n_predict").ToNumber() : -1;
-  params.sparams.temp = options.Has("temperature")
-                            ? options.Get("temperature").ToNumber()
-                            : 0.80f;
-  params.sparams.top_k =
-      options.Has("top_k") ? options.Get("top_k").ToNumber() : 40;
-  params.sparams.top_p =
-      options.Has("top_p") ? options.Get("top_p").ToNumber() : 0.95f;
-  params.sparams.min_p =
-      options.Has("min_p") ? options.Get("min_p").ToNumber() : 0.05f;
-  params.sparams.tfs_z =
-      options.Has("tfs_z") ? options.Get("tfs_z").ToNumber() : 1.00f;
-  params.sparams.mirostat =
-      options.Has("mirostat") ? options.Get("mirostat").ToNumber() : 0;
-  params.sparams.mirostat_tau = options.Has("mirostat_tau")
-                                    ? options.Get("mirostat_tau").ToNumber()
-                                    : 5.00f;
-  params.sparams.mirostat_eta = options.Has("mirostat_eta")
-                                    ? options.Get("mirostat_eta").ToNumber()
-                                    : 0.10f;
-  params.sparams.penalty_last_n = options.Has("penalty_last_n")
-                                      ? options.Get("penalty_last_n").ToNumber()
-                                      : 64;
-  params.sparams.penalty_repeat = options.Has("penalty_repeat")
-                                      ? options.Get("penalty_repeat").ToNumber()
-                                      : 1.00f;
-  params.sparams.penalty_freq = options.Has("penalty_freq")
-                                    ? options.Get("penalty_freq").ToNumber()
-                                    : 0.00f;
+  params.n_predict = get_option<int32_t>(options, "n_predict", -1);
+  params.sparams.temp = get_option<float>(options, "temperature", 0.80f);
+  params.sparams.top_k = get_option<int32_t>(options, "top_k", 40);
+  params.sparams.top_p = get_option<float>(options, "top_p", 0.95f);
+  params.sparams.min_p = get_option<float>(options, "min_p", 0.05f);
+  params.sparams.tfs_z = get_option<float>(options, "tfs_z", 1.00f);
+  params.sparams.mirostat = get_option<int32_t>(options, "mirostat", 0.00f);
+  params.sparams.mirostat_tau =
+      get_option<float>(options, "mirostat_tau", 5.00f);
+  params.sparams.mirostat_eta =
+      get_option<float>(options, "mirostat_eta", 0.10f);
+  params.sparams.penalty_last_n =
+      get_option<int32_t>(options, "penalty_last_n", 64);
+  params.sparams.penalty_repeat =
+      get_option<float>(options, "penalty_repeat", 1.00f);
+  params.sparams.penalty_freq =
+      get_option<float>(options, "penalty_freq", 0.00f);
   params.sparams.penalty_present =
-      options.Has("penalty_present") ? options.Get("penalty_present").ToNumber()
-                                     : 0.00f;
-  params.sparams.penalize_nl = options.Has("penalize_nl")
-                                   ? options.Get("penalize_nl").ToBoolean()
-                                   : false;
-  params.sparams.typical_p =
-      options.Has("typical_p") ? options.Get("typical_p").ToNumber() : 1.00f;
-  params.ignore_eos =
-      options.Has("ignore_eos") ? options.Get("ignore_eos").ToBoolean() : false;
-  params.sparams.grammar = options.Has("grammar")
-                               ? options.Get("grammar").ToString().Utf8Value()
-                               : "";
-  params.n_keep = options.Has("n_keep") ? options.Get("n_keep").ToNumber() : 0;
-  params.seed =
-      options.Has("seed") ? options.Get("seed").ToNumber() : LLAMA_DEFAULT_SEED;
+      get_option<float>(options, "penalty_present", 0.00f);
+  params.sparams.penalize_nl = get_option<bool>(options, "penalize_nl", false);
+  params.sparams.typical_p = get_option<float>(options, "typical_p", 1.00f);
+  params.ignore_eos = get_option<float>(options, "ignore_eos", false);
+  params.sparams.grammar = get_option<std::string>(options, "grammar", "");
+  params.n_keep = get_option<int32_t>(options, "n_keep", 0);
+  params.seed = get_option<int32_t>(options, "seed", LLAMA_DEFAULT_SEED);
   std::vector<std::string> stop_words;
-  if (options.Has("stop")) {
+  if (options.Has("stop") && options.Get("stop").IsArray()) {
     auto stop_words_array = options.Get("stop").As<Napi::Array>();
     for (size_t i = 0; i < stop_words_array.Length(); i++) {
-      stop_words.push_back(stop_words_array.Get(i).ToString());
+      stop_words.push_back(stop_words_array.Get(i).ToString().Utf8Value());
     }
   }
 
@@ -498,6 +519,16 @@ Napi::Value LlamaContext::LoadSession(const Napi::CallbackInfo &info) {
   return worker->Promise();
 }
 
+// release(): Promise<void>
+Napi::Value LlamaContext::Release(const Napi::CallbackInfo &info) {
+  if (compl_worker != nullptr) {
+    compl_worker->Stop();
+  }
+  auto *worker = new DisposeWorker(info.Env(), this);
+  worker->Queue();
+  return worker->Promise();
+}
+
 Napi::Object Init(Napi::Env env, Napi::Object exports) {
   LlamaContext::Export(env, exports);
   return exports;