@fugood/llama.node 0.0.1-alpha.2 → 0.0.1-alpha.4

package/CMakeLists.txt

@@ -61,7 +61,21 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(LLAMA_STATIC ON CACHE BOOL "Build llama as static library")
 add_subdirectory("src/llama.cpp")
 
-file(GLOB SOURCE_FILES "src/addons.cpp")
+file(
+  GLOB SOURCE_FILES
+    "src/addons.cc"
+    "src/common.hpp"
+    "src/DisposeWorker.cpp"
+    "src/DisposeWorker.h"
+    "src/LlamaCompletionWorker.cpp"
+    "src/LlamaCompletionWorker.h"
+    "src/LlamaContext.cpp"
+    "src/LlamaContext.h"
+    "src/LoadSessionWorker.cpp"
+    "src/LoadSessionWorker.h"
+    "src/SaveSessionWorker.cpp"
+    "src/SaveSessionWorker.h"
+)
 
 add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC})
 set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
@@ -83,3 +97,15 @@ add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
   COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${PROJECT_NAME}> ${PLATFORM_BINARY_DIR}/$<TARGET_FILE_NAME:${PROJECT_NAME}>
   COMMENT "Copying to bin folder"
 )
+
+if (LLAMA_METAL)
+  # copy ${CMAKE_BINARY_DIR}/bin/default.metallib
+  add_custom_target(copy_metallib)
+  add_custom_command(
+    TARGET copy_metallib
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/bin/default.metallib ${PLATFORM_BINARY_DIR}/default.metallib
+    COMMENT "Copying default.metallib to bin folder"
+  )
+  add_dependencies(copy_metallib ggml-metal)
+  add_dependencies(${PROJECT_NAME} copy_metallib)
+endif()

package/README.md

@@ -20,7 +20,7 @@ const context = await loadModel({
   model: 'path/to/gguf/model',
   use_mlock: true,
   n_ctx: 2048,
-  n_gpu_layers: 1, // > 0: enable Metal on iOS
+  n_gpu_layers: 1, // > 0: enable GPU
   // embedding: true, // use embedding
 })
 

package/lib/binding.js

@@ -33,13 +33,30 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadModule = void 0;
+const path = __importStar(require("path"));
+const setupEnv = (variant) => {
+    var _a, _b;
+    const postfix = variant ? `-${variant}` : '';
+    const binPath = path.resolve(__dirname, `../bin/${process.platform}${postfix}/${process.arch}/`);
+    const systemPathEnv = (_b = (_a = process.env.PATH) !== null && _a !== void 0 ? _a : process.env.Path) !== null && _b !== void 0 ? _b : '';
+    if (!systemPathEnv.includes(binPath)) {
+        if (process.platform === 'win32') {
+            process.env.Path = `${binPath};${systemPathEnv}`;
+        }
+        else {
+            process.env.PATH = `${binPath}:${systemPathEnv}`;
+        }
+    }
+};
 const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         if (variant) {
+            setupEnv(variant);
             return yield Promise.resolve(`${`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
         }
     }
     catch (_a) { } // ignore errors and try the common path
+    setupEnv();
     return yield Promise.resolve(`${`../bin/${process.platform}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
 });
 exports.loadModule = loadModule;

package/lib/binding.ts

@@ -1,3 +1,5 @@
+import * as path from 'path'
+
 export type LlamaModelOptions = {
   model: string
   embedding?: boolean
@@ -48,11 +50,26 @@ export interface Module {
   LlamaContext: LlamaContext
 }
 
+const setupEnv = (variant?: string) => {
+  const postfix = variant ? `-${variant}` : ''
+  const binPath = path.resolve(__dirname, `../bin/${process.platform}${postfix}/${process.arch}/`)
+  const systemPathEnv = process.env.PATH ?? process.env.Path ?? ''
+  if (!systemPathEnv.includes(binPath)) {
+    if (process.platform === 'win32') {
+      process.env.Path = `${binPath};${systemPathEnv}`
+    } else {
+      process.env.PATH = `${binPath}:${systemPathEnv}`
+    }
+  }
+}
+
 export const loadModule = async (variant?: string): Promise<Module> => {
   try {
     if (variant) {
+      setupEnv(variant)
       return await import(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`) as Module
     }
   } catch {} // ignore errors and try the common path
+  setupEnv()
   return await import(`../bin/${process.platform}/${process.arch}/llama-node.node`) as Module
 }

package/lib/index.js

@@ -26,9 +26,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadModel = void 0;
 const binding_1 = require("./binding");
 __exportStar(require("./binding"), exports);
-let module = null;
+const mods = {};
 const loadModel = (options) => __awaiter(void 0, void 0, void 0, function* () {
-    module !== null && module !== void 0 ? module : (module = yield (0, binding_1.loadModule)(options.lib_variant));
-    return new module.LlamaContext(options);
+    var _a, _b;
+    const variant = (_a = options.lib_variant) !== null && _a !== void 0 ? _a : 'default';
+    (_b = mods[variant]) !== null && _b !== void 0 ? _b : (mods[variant] = yield (0, binding_1.loadModule)(options.lib_variant));
+    return new mods[variant].LlamaContext(options);
 });
 exports.loadModel = loadModel;

package/lib/index.ts

@@ -7,9 +7,10 @@ export interface LlamaModelOptionsExtended extends LlamaModelOptions {
   lib_variant?: string
 }
 
-let module: Module | null = null
+const mods: { [key: string]: Module } = {}
 
 export const loadModel = async (options: LlamaModelOptionsExtended): Promise<LlamaContext> => {
-    module ??= await loadModule(options.lib_variant)
-    return new module.LlamaContext(options)
+  const variant = options.lib_variant ?? 'default'
+  mods[variant] ??= await loadModule(options.lib_variant)
+  return new mods[variant].LlamaContext(options)
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.0.1-alpha.2",
+  "version": "0.0.1-alpha.4",
   "description": "Llama.cpp for Node.js",
   "main": "lib/index.js",
   "scripts": {
@@ -57,7 +57,8 @@
     "husky": "^9.0.11",
     "jest": "^29.7.0",
     "rimraf": "^5.0.5",
-    "typescript": "^5.4.5"
+    "typescript": "^5.4.5",
+    "wait-for-expect": "^3.0.2"
   },
   "dependencies": {
     "node-addon-api": "^8.0.0"

package/src/DisposeWorker.cpp

@@ -0,0 +1,11 @@
+#include "DisposeWorker.h"
+
+DisposeWorker::DisposeWorker(const Napi::CallbackInfo &info,
+                             LlamaSessionPtr sess)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), sess_(std::move(sess)) {}
+
+void DisposeWorker::Execute() { sess_->dispose(); }
+
+void DisposeWorker::OnOK() { Resolve(AsyncWorker::Env().Undefined()); }
+
+void DisposeWorker::OnError(const Napi::Error &err) { Reject(err.Value()); }

package/src/DisposeWorker.h

@@ -0,0 +1,14 @@
+#include "common.hpp"
+
+class DisposeWorker : public Napi::AsyncWorker, public Napi::Promise::Deferred {
+public:
+  DisposeWorker(const Napi::CallbackInfo &info, LlamaSessionPtr sess);
+
+protected:
+  void Execute();
+  void OnOK();
+  void OnError(const Napi::Error &err);
+
+private:
+  LlamaSessionPtr sess_;
+};

package/src/LlamaCompletionWorker.cpp

@@ -0,0 +1,163 @@
+#include "LlamaCompletionWorker.h"
+#include "LlamaContext.h"
+
+size_t common_part(const std::vector<llama_token> &a,
+                   const std::vector<llama_token> &b) {
+  size_t i = 0;
+  while (i < a.size() && i < b.size() && a[i] == b[i]) {
+    i++;
+  }
+  return i;
+}
+
+size_t findStoppingStrings(const std::string &text,
+                           const size_t last_token_size,
+                           const std::vector<std::string> &stop_words) {
+  size_t stop_pos = std::string::npos;
+
+  for (const std::string &word : stop_words) {
+    size_t pos;
+
+    const size_t tmp = word.size() + last_token_size;
+    const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;
+
+    pos = text.find(word, from_pos);
+
+    if (pos != std::string::npos &&
+        (stop_pos == std::string::npos || pos < stop_pos)) {
+      stop_pos = pos;
+    }
+  }
+
+  return stop_pos;
+}
+
+LlamaCompletionWorker::LlamaCompletionWorker(
+    const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+    Napi::Function callback, gpt_params params,
+    std::vector<std::string> stop_words)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
+      _params(params), _stop_words(stop_words) {
+  if (!callback.IsEmpty()) {
+    _tsfn = Napi::ThreadSafeFunction::New(info.Env(), callback,
+                                          "LlamaCompletionCallback", 0, 1);
+    _has_callback = true;
+  }
+}
+
+LlamaCompletionWorker::~LlamaCompletionWorker() {
+  if (_has_callback) {
+    _tsfn.Release();
+  }
+}
+
+void LlamaCompletionWorker::Execute() {
+  _sess->get_mutex().lock();
+  const auto t_main_start = ggml_time_us();
+  const size_t n_ctx = _params.n_ctx;
+  const auto n_keep = _params.n_keep;
+  size_t n_cur = 0;
+  size_t n_input = 0;
+  const auto model = llama_get_model(_sess->context());
+  const bool add_bos = llama_should_add_bos_token(model);
+  auto ctx = _sess->context();
+
+  llama_set_rng_seed(ctx, _params.seed);
+
+  LlamaCppSampling sampling{llama_sampling_init(_params.sparams),
+                            llama_sampling_free};
+
+  std::vector<llama_token> prompt_tokens =
+      ::llama_tokenize(ctx, _params.prompt, add_bos);
+  n_input = prompt_tokens.size();
+  if (_sess->tokens_ptr()->size() > 0) {
+    n_cur = common_part(*(_sess->tokens_ptr()), prompt_tokens);
+    if (n_cur == n_input) {
+      --n_cur;
+    }
+    n_input -= n_cur;
+    llama_kv_cache_seq_rm(ctx, 0, n_cur, -1);
+  }
+  _sess->set_tokens(std::move(prompt_tokens));
+
+  const int max_len = _params.n_predict < 0 ? 0 : _params.n_predict;
+  _sess->tokens_ptr()->reserve(_sess->tokens_ptr()->size() + max_len);
+
+  auto embd = _sess->tokens_ptr();
+  for (int i = 0; i < max_len || _stop; i++) {
+    // check if we need to remove some tokens
+    if (embd->size() >= _params.n_ctx) {
+      const int n_left = n_cur - n_keep - 1;
+      const int n_discard = n_left / 2;
+
+      llama_kv_cache_seq_rm(ctx, 0, n_keep + 1, n_keep + n_discard + 1);
+      llama_kv_cache_seq_add(ctx, 0, n_keep + 1 + n_discard, n_cur, -n_discard);
+
+      // shift the tokens
+      embd->insert(embd->begin() + n_keep + 1,
+                   embd->begin() + n_keep + 1 + n_discard, embd->end());
+      embd->resize(embd->size() - n_discard);
+
+      n_cur -= n_discard;
+      _result.truncated = true;
+    }
+    int ret = llama_decode(
+        ctx, llama_batch_get_one(embd->data() + n_cur, n_input, n_cur, 0));
+    if (ret < 0) {
+      SetError("Failed to decode token, code: " + std::to_string(ret));
+      break;
+    }
+    // sample the next token
+    const llama_token new_token_id =
+        llama_sampling_sample(sampling.get(), ctx, nullptr);
+    // prepare the next batch
+    embd->emplace_back(new_token_id);
+    auto token = llama_token_to_piece(ctx, new_token_id);
+    _result.text += token;
+    n_cur += n_input;
+    _result.tokens_evaluated += n_input;
+    _result.tokens_predicted += 1;
+    n_input = 1;
+    if (_has_callback) {
+      const char *c_token = strdup(token.c_str());
+      _tsfn.BlockingCall(c_token, [](Napi::Env env, Napi::Function jsCallback,
+                                     const char *value) {
+        auto obj = Napi::Object::New(env);
+        obj.Set("token", Napi::String::New(env, value));
+        delete value;
+        jsCallback.Call({obj});
+      });
+    }
+    // is it an end of generation?
+    if (llama_token_is_eog(model, new_token_id)) {
+      break;
+    }
+    // check for stop words
+    if (!_stop_words.empty()) {
+      const size_t stop_pos =
+          findStoppingStrings(_result.text, token.size(), _stop_words);
+      if (stop_pos != std::string::npos) {
+        break;
+      }
+    }
+  }
+  const auto t_main_end = ggml_time_us();
+  _sess->get_mutex().unlock();
+}
+
+void LlamaCompletionWorker::OnOK() {
+  auto result = Napi::Object::New(Napi::AsyncWorker::Env());
+  result.Set("tokens_evaluated", Napi::Number::New(Napi::AsyncWorker::Env(),
+                                                   _result.tokens_evaluated));
+  result.Set("tokens_predicted", Napi::Number::New(Napi::AsyncWorker::Env(),
+                                                   _result.tokens_predicted));
+  result.Set("truncated",
+             Napi::Boolean::New(Napi::AsyncWorker::Env(), _result.truncated));
+  result.Set("text",
+             Napi::String::New(Napi::AsyncWorker::Env(), _result.text.c_str()));
+  Napi::Promise::Deferred::Resolve(result);
+}
+
+void LlamaCompletionWorker::OnError(const Napi::Error &err) {
+  Napi::Promise::Deferred::Reject(err.Value());
+}

package/src/LlamaCompletionWorker.h

@@ -0,0 +1,34 @@
+#include "common.hpp"
+
+struct CompletionResult {
+  std::string text = "";
+  bool truncated = false;
+  size_t tokens_predicted = 0;
+  size_t tokens_evaluated = 0;
+};
+
+class LlamaCompletionWorker : public Napi::AsyncWorker,
+                              public Napi::Promise::Deferred {
+public:
+  LlamaCompletionWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+                        Napi::Function callback, gpt_params params,
+                        std::vector<std::string> stop_words = {});
+
+  ~LlamaCompletionWorker();
+
+  inline void Stop() { _stop = true; }
+
+protected:
+  void Execute();
+  void OnOK();
+  void OnError(const Napi::Error &err);
+
+private:
+  LlamaSessionPtr _sess;
+  gpt_params _params;
+  std::vector<std::string> _stop_words;
+  Napi::ThreadSafeFunction _tsfn;
+  bool _has_callback = false;
+  bool _stop = false;
+  CompletionResult _result;
+};

package/src/LlamaContext.cpp

@@ -0,0 +1,200 @@
+#include "LlamaContext.h"
+#include "DisposeWorker.h"
+#include "LlamaCompletionWorker.h"
+#include "LoadSessionWorker.h"
+#include "SaveSessionWorker.h"
+
+void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
+  Napi::Function func = DefineClass(
+      env, "LlamaContext",
+      {InstanceMethod<&LlamaContext::GetSystemInfo>(
+           "getSystemInfo",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::Completion>(
+           "completion",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::StopCompletion>(
+           "stopCompletion",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::SaveSession>(
+           "saveSession",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::LoadSession>(
+           "loadSession",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::Release>(
+           "release", static_cast<napi_property_attributes>(napi_enumerable))});
+  Napi::FunctionReference *constructor = new Napi::FunctionReference();
+  *constructor = Napi::Persistent(func);
+#if NAPI_VERSION > 5
+  env.SetInstanceData(constructor);
+#endif
+  exports.Set("LlamaContext", func);
+}
+
+// construct({ model, embedding, n_ctx, n_batch, n_threads, n_gpu_layers,
+// use_mlock, use_mmap }): LlamaContext throws error
+LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
+    : Napi::ObjectWrap<LlamaContext>(info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsObject()) {
+    Napi::TypeError::New(env, "Object expected").ThrowAsJavaScriptException();
+  }
+  auto options = info[0].As<Napi::Object>();
+
+  gpt_params params;
+  params.model = get_option<std::string>(options, "model", "");
+  if (params.model.empty()) {
+    Napi::TypeError::New(env, "Model is required").ThrowAsJavaScriptException();
+  }
+  params.embedding = get_option<bool>(options, "embedding", false);
+  params.n_ctx = get_option<int32_t>(options, "n_ctx", 512);
+  params.n_batch = get_option<int32_t>(options, "n_batch", 2048);
+  params.n_threads =
+      get_option<int32_t>(options, "n_threads", get_math_cpu_count() / 2);
+  params.n_gpu_layers = get_option<int32_t>(options, "n_gpu_layers", -1);
+  params.use_mlock = get_option<bool>(options, "use_mlock", false);
+  params.use_mmap = get_option<bool>(options, "use_mmap", true);
+  params.numa =
+      static_cast<ggml_numa_strategy>(get_option<uint32_t>(options, "numa", 0));
+
+  llama_backend_init();
+  llama_numa_init(params.numa);
+
+  llama_model *model;
+  llama_context *ctx;
+  std::tie(model, ctx) = llama_init_from_gpt_params(params);
+
+  if (model == nullptr || ctx == nullptr) {
+    Napi::TypeError::New(env, "Failed to load model")
+        .ThrowAsJavaScriptException();
+  }
+
+  _sess = std::make_shared<LlamaSession>(ctx, params);
+  _info = get_system_info(params);
+}
+
+// getSystemInfo(): string
+Napi::Value LlamaContext::GetSystemInfo(const Napi::CallbackInfo &info) {
+  return Napi::String::New(info.Env(), _info);
+}
+
+// completion(options: LlamaCompletionOptions, onToken?: (token: string) =>
+// void): Promise<LlamaCompletionResult>
+Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsObject()) {
+    Napi::TypeError::New(env, "Object expected").ThrowAsJavaScriptException();
+  }
+  if (info.Length() >= 2 && !info[1].IsFunction()) {
+    Napi::TypeError::New(env, "Function expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  auto options = info[0].As<Napi::Object>();
+
+  gpt_params params = _sess->params();
+  params.prompt = get_option<std::string>(options, "prompt", "");
+  if (params.prompt.empty()) {
+    Napi::TypeError::New(env, "Prompt is required")
+        .ThrowAsJavaScriptException();
+  }
+  params.n_predict = get_option<int32_t>(options, "n_predict", -1);
+  params.sparams.temp = get_option<float>(options, "temperature", 0.80f);
+  params.sparams.top_k = get_option<int32_t>(options, "top_k", 40);
+  params.sparams.top_p = get_option<float>(options, "top_p", 0.95f);
+  params.sparams.min_p = get_option<float>(options, "min_p", 0.05f);
+  params.sparams.tfs_z = get_option<float>(options, "tfs_z", 1.00f);
+  params.sparams.mirostat = get_option<int32_t>(options, "mirostat", 0.00f);
+  params.sparams.mirostat_tau =
+      get_option<float>(options, "mirostat_tau", 5.00f);
+  params.sparams.mirostat_eta =
+      get_option<float>(options, "mirostat_eta", 0.10f);
+  params.sparams.penalty_last_n =
+      get_option<int32_t>(options, "penalty_last_n", 64);
+  params.sparams.penalty_repeat =
+      get_option<float>(options, "penalty_repeat", 1.00f);
+  params.sparams.penalty_freq =
+      get_option<float>(options, "penalty_freq", 0.00f);
+  params.sparams.penalty_present =
+      get_option<float>(options, "penalty_present", 0.00f);
+  params.sparams.penalize_nl = get_option<bool>(options, "penalize_nl", false);
+  params.sparams.typical_p = get_option<float>(options, "typical_p", 1.00f);
+  params.ignore_eos = get_option<float>(options, "ignore_eos", false);
+  params.sparams.grammar = get_option<std::string>(options, "grammar", "");
+  params.n_keep = get_option<int32_t>(options, "n_keep", 0);
+  params.seed = get_option<int32_t>(options, "seed", LLAMA_DEFAULT_SEED);
+  std::vector<std::string> stop_words;
+  if (options.Has("stop") && options.Get("stop").IsArray()) {
+    auto stop_words_array = options.Get("stop").As<Napi::Array>();
+    for (size_t i = 0; i < stop_words_array.Length(); i++) {
+      stop_words.push_back(stop_words_array.Get(i).ToString().Utf8Value());
+    }
+  }
+
+  Napi::Function callback;
+  if (info.Length() >= 2) {
+    callback = info[1].As<Napi::Function>();
+  }
+
+  auto *worker =
+      new LlamaCompletionWorker(info, _sess, callback, params, stop_words);
+  worker->Queue();
+  _wip = worker;
+  return worker->Promise();
+}
+
+// stopCompletion(): void
+void LlamaContext::StopCompletion(const Napi::CallbackInfo &info) {
+  if (_wip != nullptr) {
+    _wip->Stop();
+  }
+}
+
+// saveSession(path: string): Promise<void> throws error
+Napi::Value LlamaContext::SaveSession(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsString()) {
+    Napi::TypeError::New(env, "String expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  auto *worker = new SaveSessionWorker(info, _sess);
+  worker->Queue();
+  return worker->Promise();
+}
+
+// loadSession(path: string): Promise<{ count }> throws error
+Napi::Value LlamaContext::LoadSession(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsString()) {
+    Napi::TypeError::New(env, "String expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  auto *worker = new LoadSessionWorker(info, _sess);
+  worker->Queue();
+  return worker->Promise();
+}
+
+// release(): Promise<void>
+Napi::Value LlamaContext::Release(const Napi::CallbackInfo &info) {
+  auto env = info.Env();
+  if (_wip != nullptr) {
+    _wip->Stop();
+  }
+  if (_sess == nullptr) {
+    auto promise = Napi::Promise::Deferred(env);
+    promise.Resolve(env.Undefined());
+    return promise.Promise();
+  }
+  auto *worker = new DisposeWorker(info, std::move(_sess));
+  worker->Queue();
+  return worker->Promise();
+}

package/src/LlamaContext.h

@@ -0,0 +1,21 @@
+#include "common.hpp"
+
+class LlamaCompletionWorker;
+
+class LlamaContext : public Napi::ObjectWrap<LlamaContext> {
+public:
+  LlamaContext(const Napi::CallbackInfo &info);
+  static void Init(Napi::Env env, Napi::Object &exports);
+
+private:
+  Napi::Value GetSystemInfo(const Napi::CallbackInfo &info);
+  Napi::Value Completion(const Napi::CallbackInfo &info);
+  void StopCompletion(const Napi::CallbackInfo &info);
+  Napi::Value SaveSession(const Napi::CallbackInfo &info);
+  Napi::Value LoadSession(const Napi::CallbackInfo &info);
+  Napi::Value Release(const Napi::CallbackInfo &info);
+
+  std::string _info;
+  LlamaSessionPtr _sess = nullptr;
+  LlamaCompletionWorker *_wip = nullptr;
+};

package/src/LoadSessionWorker.cpp

@@ -0,0 +1,24 @@
+#include "LoadSessionWorker.h"
+#include "LlamaContext.h"
+
+LoadSessionWorker::LoadSessionWorker(const Napi::CallbackInfo &info,
+                                     LlamaSessionPtr &sess)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _path(info[0].ToString()),
+      _sess(sess) {}
+
+void LoadSessionWorker::Execute() {
+  _sess->get_mutex().lock();
+  // reserve the maximum number of tokens for capacity
+  std::vector<llama_token> tokens;
+  tokens.reserve(_sess->params().n_ctx);
+  if (!llama_state_load_file(_sess->context(), _path.c_str(), tokens.data(),
+                             tokens.capacity(), &count)) {
+    SetError("Failed to load session");
+  }
+  _sess->set_tokens(std::move(tokens));
+  _sess->get_mutex().unlock();
+}
+
+void LoadSessionWorker::OnOK() { Resolve(AsyncWorker::Env().Undefined()); }
+
+void LoadSessionWorker::OnError(const Napi::Error &err) { Reject(err.Value()); }