@fugood/llama.node 0.0.1-alpha.1 → 0.0.1-alpha.3

package/CMakeLists.txt

@@ -61,7 +61,21 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(LLAMA_STATIC ON CACHE BOOL "Build llama as static library")
 add_subdirectory("src/llama.cpp")
 
-file(GLOB SOURCE_FILES "src/addons.cpp")
+file(
+  GLOB SOURCE_FILES
+    "src/addons.cc"
+    "src/common.hpp"
+    "src/DisposeWorker.cpp"
+    "src/DisposeWorker.h"
+    "src/LlamaCompletionWorker.cpp"
+    "src/LlamaCompletionWorker.h"
+    "src/LlamaContext.cpp"
+    "src/LlamaContext.h"
+    "src/LoadSessionWorker.cpp"
+    "src/LoadSessionWorker.h"
+    "src/SaveSessionWorker.cpp"
+    "src/SaveSessionWorker.h"
+)
 
 add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC})
 set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
@@ -83,3 +97,15 @@ add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
   COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${PROJECT_NAME}> ${PLATFORM_BINARY_DIR}/$<TARGET_FILE_NAME:${PROJECT_NAME}>
   COMMENT "Copying to bin folder"
 )
+
+if (LLAMA_METAL)
+  # copy ${CMAKE_BINARY_DIR}/bin/default.metallib
+  add_custom_target(copy_metallib)
+  add_custom_command(
+    TARGET copy_metallib
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/bin/default.metallib ${PLATFORM_BINARY_DIR}/default.metallib
+    COMMENT "Copying default.metallib to bin folder"
+  )
+  add_dependencies(copy_metallib ggml-metal)
+  add_dependencies(${PROJECT_NAME} copy_metallib)
+endif()

package/README.md

@@ -7,20 +7,20 @@ Node binding of [llama.cpp](https://github.com/ggerganov/llama.cpp).
 ## Installation
 
 ```sh
-npm install llama.node
+npm install @fugood/llama.node
 ```
 
 ## Usage
 
 ```js
-import { loadModel } from 'llama.node'
+import { loadModel } from '@fugood/llama.node'
 
 // Initial a Llama context with the model (may take a while)
-const context = loadModel({
+const context = await loadModel({
   model: 'path/to/gguf/model',
   use_mlock: true,
   n_ctx: 2048,
-  n_gpu_layers: 1, // > 0: enable Metal on iOS
+  n_gpu_layers: 1, // > 0: enable GPU
   // embedding: true, // use embedding
 })
 

package/lib/binding.js

@@ -1,13 +1,45 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadModule = void 0;
-const loadModule = (variant) => {
+const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         if (variant) {
-            return require(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`);
+            return yield Promise.resolve(`${`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
         }
     }
     catch (_a) { } // ignore errors and try the common path
-    return require(`../bin/${process.platform}/${process.arch}/llama-node.node`);
-};
+    return yield Promise.resolve(`${`../bin/${process.platform}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
+});
 exports.loadModule = loadModule;

package/lib/binding.ts

@@ -41,17 +41,18 @@ export interface LlamaContext {
   stopCompletion(): void
   saveSession(path: string): Promise<void>
   loadSession(path: string): Promise<void>
+  release(): Promise<void>
 }
 
 export interface Module {
   LlamaContext: LlamaContext
 }
 
-export const loadModule = (variant?: string): Module => {
+export const loadModule = async (variant?: string): Promise<Module> => {
   try {
     if (variant) {
-      return require(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`) as Module
+      return await import(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`) as Module
     }
   } catch {} // ignore errors and try the common path
-  return require(`../bin/${process.platform}/${process.arch}/llama-node.node`) as Module
+  return await import(`../bin/${process.platform}/${process.arch}/llama-node.node`) as Module
 }

package/lib/index.js

@@ -13,12 +13,24 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadModel = void 0;
 const binding_1 = require("./binding");
 __exportStar(require("./binding"), exports);
-const loadModel = (options) => {
-    const { LlamaContext } = (0, binding_1.loadModule)(options.lib_variant);
-    return new LlamaContext(options);
-};
+const mods = {};
+const loadModel = (options) => __awaiter(void 0, void 0, void 0, function* () {
+    var _a, _b;
+    const variant = (_a = options.lib_variant) !== null && _a !== void 0 ? _a : 'default';
+    (_b = mods[variant]) !== null && _b !== void 0 ? _b : (mods[variant] = yield (0, binding_1.loadModule)(options.lib_variant));
+    return new mods[variant].LlamaContext(options);
+});
 exports.loadModel = loadModel;

package/lib/index.ts

@@ -1,5 +1,5 @@
 import { loadModule, LlamaModelOptions } from './binding'
-import type { LlamaContext } from './binding'
+import type { Module, LlamaContext } from './binding'
 
 export * from './binding'
 
@@ -7,7 +7,10 @@ export interface LlamaModelOptionsExtended extends LlamaModelOptions {
   lib_variant?: string
 }
 
-export const loadModel = (options: LlamaModelOptionsExtended): LlamaContext => {
-    const { LlamaContext } = loadModule(options.lib_variant)
-    return new LlamaContext(options)
+const mods: { [key: string]: Module } = {}
+
+export const loadModel = async (options: LlamaModelOptionsExtended): Promise<LlamaContext> => {
+  const variant = options.lib_variant ?? 'default'
+  mods[variant] ??= await loadModule(options.lib_variant)
+  return new mods[variant].LlamaContext(options)
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.0.1-alpha.1",
+  "version": "0.0.1-alpha.3",
   "description": "Llama.cpp for Node.js",
   "main": "lib/index.js",
   "scripts": {
@@ -30,7 +30,7 @@
   },
   "homepage": "https://github.com/mybigday/llama.node#readme",
   "publishConfig": {
-    "registry": "https://registry.npmjs.org"        
+    "registry": "https://registry.npmjs.org"
   },
   "binary": {
     "napi_versions": [
@@ -57,7 +57,8 @@
     "husky": "^9.0.11",
     "jest": "^29.7.0",
     "rimraf": "^5.0.5",
-    "typescript": "^5.4.5"
+    "typescript": "^5.4.5",
+    "wait-for-expect": "^3.0.2"
   },
   "dependencies": {
     "node-addon-api": "^8.0.0"

package/src/DisposeWorker.cpp

@@ -0,0 +1,11 @@
+#include "DisposeWorker.h"
+
+DisposeWorker::DisposeWorker(const Napi::CallbackInfo &info,
+                             LlamaSessionPtr sess)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), sess_(std::move(sess)) {}
+
+void DisposeWorker::Execute() { sess_->dispose(); }
+
+void DisposeWorker::OnOK() { Resolve(AsyncWorker::Env().Undefined()); }
+
+void DisposeWorker::OnError(const Napi::Error &err) { Reject(err.Value()); }

package/src/DisposeWorker.h

@@ -0,0 +1,14 @@
+#include "common.hpp"
+
+class DisposeWorker : public Napi::AsyncWorker, public Napi::Promise::Deferred {
+public:
+  DisposeWorker(const Napi::CallbackInfo &info, LlamaSessionPtr sess);
+
+protected:
+  void Execute();
+  void OnOK();
+  void OnError(const Napi::Error &err);
+
+private:
+  LlamaSessionPtr sess_;
+};

package/src/LlamaCompletionWorker.cpp

@@ -0,0 +1,163 @@
+#include "LlamaCompletionWorker.h"
+#include "LlamaContext.h"
+
+size_t common_part(const std::vector<llama_token> &a,
+                   const std::vector<llama_token> &b) {
+  size_t i = 0;
+  while (i < a.size() && i < b.size() && a[i] == b[i]) {
+    i++;
+  }
+  return i;
+}
+
+size_t findStoppingStrings(const std::string &text,
+                           const size_t last_token_size,
+                           const std::vector<std::string> &stop_words) {
+  size_t stop_pos = std::string::npos;
+
+  for (const std::string &word : stop_words) {
+    size_t pos;
+
+    const size_t tmp = word.size() + last_token_size;
+    const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;
+
+    pos = text.find(word, from_pos);
+
+    if (pos != std::string::npos &&
+        (stop_pos == std::string::npos || pos < stop_pos)) {
+      stop_pos = pos;
+    }
+  }
+
+  return stop_pos;
+}
+
+LlamaCompletionWorker::LlamaCompletionWorker(
+    const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+    Napi::Function callback, gpt_params params,
+    std::vector<std::string> stop_words)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
+      _params(params), _stop_words(stop_words) {
+  if (!callback.IsEmpty()) {
+    _tsfn = Napi::ThreadSafeFunction::New(info.Env(), callback,
+                                          "LlamaCompletionCallback", 0, 1);
+    _has_callback = true;
+  }
+}
+
+LlamaCompletionWorker::~LlamaCompletionWorker() {
+  if (_has_callback) {
+    _tsfn.Release();
+  }
+}
+
+void LlamaCompletionWorker::Execute() {
+  _sess->get_mutex().lock();
+  const auto t_main_start = ggml_time_us();
+  const size_t n_ctx = _params.n_ctx;
+  const auto n_keep = _params.n_keep;
+  size_t n_cur = 0;
+  size_t n_input = 0;
+  const auto model = llama_get_model(_sess->context());
+  const bool add_bos = llama_should_add_bos_token(model);
+  auto ctx = _sess->context();
+
+  llama_set_rng_seed(ctx, _params.seed);
+
+  LlamaCppSampling sampling{llama_sampling_init(_params.sparams),
+                            llama_sampling_free};
+
+  std::vector<llama_token> prompt_tokens =
+      ::llama_tokenize(ctx, _params.prompt, add_bos);
+  n_input = prompt_tokens.size();
+  if (_sess->tokens_ptr()->size() > 0) {
+    n_cur = common_part(*(_sess->tokens_ptr()), prompt_tokens);
+    if (n_cur == n_input) {
+      --n_cur;
+    }
+    n_input -= n_cur;
+    llama_kv_cache_seq_rm(ctx, 0, n_cur, -1);
+  }
+  _sess->set_tokens(std::move(prompt_tokens));
+
+  const int max_len = _params.n_predict < 0 ? 0 : _params.n_predict;
+  _sess->tokens_ptr()->reserve(_sess->tokens_ptr()->size() + max_len);
+
+  auto embd = _sess->tokens_ptr();
+  for (int i = 0; i < max_len || _stop; i++) {
+    // check if we need to remove some tokens
+    if (embd->size() >= _params.n_ctx) {
+      const int n_left = n_cur - n_keep - 1;
+      const int n_discard = n_left / 2;
+
+      llama_kv_cache_seq_rm(ctx, 0, n_keep + 1, n_keep + n_discard + 1);
+      llama_kv_cache_seq_add(ctx, 0, n_keep + 1 + n_discard, n_cur, -n_discard);
+
+      // shift the tokens
+      embd->insert(embd->begin() + n_keep + 1,
+                   embd->begin() + n_keep + 1 + n_discard, embd->end());
+      embd->resize(embd->size() - n_discard);
+
+      n_cur -= n_discard;
+      _result.truncated = true;
+    }
+    int ret = llama_decode(
+        ctx, llama_batch_get_one(embd->data() + n_cur, n_input, n_cur, 0));
+    if (ret < 0) {
+      SetError("Failed to decode token, code: " + std::to_string(ret));
+      break;
+    }
+    // sample the next token
+    const llama_token new_token_id =
+        llama_sampling_sample(sampling.get(), ctx, nullptr);
+    // prepare the next batch
+    embd->emplace_back(new_token_id);
+    auto token = llama_token_to_piece(ctx, new_token_id);
+    _result.text += token;
+    n_cur += n_input;
+    _result.tokens_evaluated += n_input;
+    _result.tokens_predicted += 1;
+    n_input = 1;
+    if (_has_callback) {
+      const char *c_token = strdup(token.c_str());
+      _tsfn.BlockingCall(c_token, [](Napi::Env env, Napi::Function jsCallback,
+                                     const char *value) {
+        auto obj = Napi::Object::New(env);
+        obj.Set("token", Napi::String::New(env, value));
+        delete value;
+        jsCallback.Call({obj});
+      });
+    }
+    // is it an end of generation?
+    if (llama_token_is_eog(model, new_token_id)) {
+      break;
+    }
+    // check for stop words
+    if (!_stop_words.empty()) {
+      const size_t stop_pos =
+          findStoppingStrings(_result.text, token.size(), _stop_words);
+      if (stop_pos != std::string::npos) {
+        break;
+      }
+    }
+  }
+  const auto t_main_end = ggml_time_us();
+  _sess->get_mutex().unlock();
+}
+
+void LlamaCompletionWorker::OnOK() {
+  auto result = Napi::Object::New(Napi::AsyncWorker::Env());
+  result.Set("tokens_evaluated", Napi::Number::New(Napi::AsyncWorker::Env(),
+                                                   _result.tokens_evaluated));
+  result.Set("tokens_predicted", Napi::Number::New(Napi::AsyncWorker::Env(),
+                                                   _result.tokens_predicted));
+  result.Set("truncated",
+             Napi::Boolean::New(Napi::AsyncWorker::Env(), _result.truncated));
+  result.Set("text",
+             Napi::String::New(Napi::AsyncWorker::Env(), _result.text.c_str()));
+  Napi::Promise::Deferred::Resolve(result);
+}
+
+void LlamaCompletionWorker::OnError(const Napi::Error &err) {
+  Napi::Promise::Deferred::Reject(err.Value());
+}

package/src/LlamaCompletionWorker.h

@@ -0,0 +1,34 @@
+#include "common.hpp"
+
+struct CompletionResult {
+  std::string text = "";
+  bool truncated = false;
+  size_t tokens_predicted = 0;
+  size_t tokens_evaluated = 0;
+};
+
+class LlamaCompletionWorker : public Napi::AsyncWorker,
+                              public Napi::Promise::Deferred {
+public:
+  LlamaCompletionWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+                        Napi::Function callback, gpt_params params,
+                        std::vector<std::string> stop_words = {});
+
+  ~LlamaCompletionWorker();
+
+  inline void Stop() { _stop = true; }
+
+protected:
+  void Execute();
+  void OnOK();
+  void OnError(const Napi::Error &err);
+
+private:
+  LlamaSessionPtr _sess;
+  gpt_params _params;
+  std::vector<std::string> _stop_words;
+  Napi::ThreadSafeFunction _tsfn;
+  bool _has_callback = false;
+  bool _stop = false;
+  CompletionResult _result;
+};

package/src/LlamaContext.cpp

@@ -0,0 +1,200 @@
+#include "LlamaContext.h"
+#include "DisposeWorker.h"
+#include "LlamaCompletionWorker.h"
+#include "LoadSessionWorker.h"
+#include "SaveSessionWorker.h"
+
+void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
+  Napi::Function func = DefineClass(
+      env, "LlamaContext",
+      {InstanceMethod<&LlamaContext::GetSystemInfo>(
+           "getSystemInfo",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::Completion>(
+           "completion",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::StopCompletion>(
+           "stopCompletion",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::SaveSession>(
+           "saveSession",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::LoadSession>(
+           "loadSession",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::Release>(
+           "release", static_cast<napi_property_attributes>(napi_enumerable))});
+  Napi::FunctionReference *constructor = new Napi::FunctionReference();
+  *constructor = Napi::Persistent(func);
+#if NAPI_VERSION > 5
+  env.SetInstanceData(constructor);
+#endif
+  exports.Set("LlamaContext", func);
+}
+
+// construct({ model, embedding, n_ctx, n_batch, n_threads, n_gpu_layers,
+// use_mlock, use_mmap }): LlamaContext throws error
+LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
+    : Napi::ObjectWrap<LlamaContext>(info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsObject()) {
+    Napi::TypeError::New(env, "Object expected").ThrowAsJavaScriptException();
+  }
+  auto options = info[0].As<Napi::Object>();
+
+  gpt_params params;
+  params.model = get_option<std::string>(options, "model", "");
+  if (params.model.empty()) {
+    Napi::TypeError::New(env, "Model is required").ThrowAsJavaScriptException();
+  }
+  params.embedding = get_option<bool>(options, "embedding", false);
+  params.n_ctx = get_option<int32_t>(options, "n_ctx", 512);
+  params.n_batch = get_option<int32_t>(options, "n_batch", 2048);
+  params.n_threads =
+      get_option<int32_t>(options, "n_threads", get_math_cpu_count() / 2);
+  params.n_gpu_layers = get_option<int32_t>(options, "n_gpu_layers", -1);
+  params.use_mlock = get_option<bool>(options, "use_mlock", false);
+  params.use_mmap = get_option<bool>(options, "use_mmap", true);
+  params.numa =
+      static_cast<ggml_numa_strategy>(get_option<uint32_t>(options, "numa", 0));
+
+  llama_backend_init();
+  llama_numa_init(params.numa);
+
+  llama_model *model;
+  llama_context *ctx;
+  std::tie(model, ctx) = llama_init_from_gpt_params(params);
+
+  if (model == nullptr || ctx == nullptr) {
+    Napi::TypeError::New(env, "Failed to load model")
+        .ThrowAsJavaScriptException();
+  }
+
+  _sess = std::make_shared<LlamaSession>(ctx, params);
+  _info = get_system_info(params);
+}
+
+// getSystemInfo(): string
+Napi::Value LlamaContext::GetSystemInfo(const Napi::CallbackInfo &info) {
+  return Napi::String::New(info.Env(), _info);
+}
+
+// completion(options: LlamaCompletionOptions, onToken?: (token: string) =>
+// void): Promise<LlamaCompletionResult>
+Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsObject()) {
+    Napi::TypeError::New(env, "Object expected").ThrowAsJavaScriptException();
+  }
+  if (info.Length() >= 2 && !info[1].IsFunction()) {
+    Napi::TypeError::New(env, "Function expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  auto options = info[0].As<Napi::Object>();
+
+  gpt_params params = _sess->params();
+  params.prompt = get_option<std::string>(options, "prompt", "");
+  if (params.prompt.empty()) {
+    Napi::TypeError::New(env, "Prompt is required")
+        .ThrowAsJavaScriptException();
+  }
+  params.n_predict = get_option<int32_t>(options, "n_predict", -1);
+  params.sparams.temp = get_option<float>(options, "temperature", 0.80f);
+  params.sparams.top_k = get_option<int32_t>(options, "top_k", 40);
+  params.sparams.top_p = get_option<float>(options, "top_p", 0.95f);
+  params.sparams.min_p = get_option<float>(options, "min_p", 0.05f);
+  params.sparams.tfs_z = get_option<float>(options, "tfs_z", 1.00f);
+  params.sparams.mirostat = get_option<int32_t>(options, "mirostat", 0.00f);
+  params.sparams.mirostat_tau =
+      get_option<float>(options, "mirostat_tau", 5.00f);
+  params.sparams.mirostat_eta =
+      get_option<float>(options, "mirostat_eta", 0.10f);
+  params.sparams.penalty_last_n =
+      get_option<int32_t>(options, "penalty_last_n", 64);
+  params.sparams.penalty_repeat =
+      get_option<float>(options, "penalty_repeat", 1.00f);
+  params.sparams.penalty_freq =
+      get_option<float>(options, "penalty_freq", 0.00f);
+  params.sparams.penalty_present =
+      get_option<float>(options, "penalty_present", 0.00f);
+  params.sparams.penalize_nl = get_option<bool>(options, "penalize_nl", false);
+  params.sparams.typical_p = get_option<float>(options, "typical_p", 1.00f);
+  params.ignore_eos = get_option<float>(options, "ignore_eos", false);
+  params.sparams.grammar = get_option<std::string>(options, "grammar", "");
+  params.n_keep = get_option<int32_t>(options, "n_keep", 0);
+  params.seed = get_option<int32_t>(options, "seed", LLAMA_DEFAULT_SEED);
+  std::vector<std::string> stop_words;
+  if (options.Has("stop") && options.Get("stop").IsArray()) {
+    auto stop_words_array = options.Get("stop").As<Napi::Array>();
+    for (size_t i = 0; i < stop_words_array.Length(); i++) {
+      stop_words.push_back(stop_words_array.Get(i).ToString().Utf8Value());
+    }
+  }
+
+  Napi::Function callback;
+  if (info.Length() >= 2) {
+    callback = info[1].As<Napi::Function>();
+  }
+
+  auto *worker =
+      new LlamaCompletionWorker(info, _sess, callback, params, stop_words);
+  worker->Queue();
+  _wip = worker;
+  return worker->Promise();
+}
+
+// stopCompletion(): void
+void LlamaContext::StopCompletion(const Napi::CallbackInfo &info) {
+  if (_wip != nullptr) {
+    _wip->Stop();
+  }
+}
+
+// saveSession(path: string): Promise<void> throws error
+Napi::Value LlamaContext::SaveSession(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsString()) {
+    Napi::TypeError::New(env, "String expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  auto *worker = new SaveSessionWorker(info, _sess);
+  worker->Queue();
+  return worker->Promise();
+}
+
+// loadSession(path: string): Promise<{ count }> throws error
+Napi::Value LlamaContext::LoadSession(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsString()) {
+    Napi::TypeError::New(env, "String expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  auto *worker = new LoadSessionWorker(info, _sess);
+  worker->Queue();
+  return worker->Promise();
+}
+
+// release(): Promise<void>
+Napi::Value LlamaContext::Release(const Napi::CallbackInfo &info) {
+  auto env = info.Env();
+  if (_wip != nullptr) {
+    _wip->Stop();
+  }
+  if (_sess == nullptr) {
+    auto promise = Napi::Promise::Deferred(env);
+    promise.Resolve(env.Undefined());
+    return promise.Promise();
+  }
+  auto *worker = new DisposeWorker(info, std::move(_sess));
+  worker->Queue();
+  return worker->Promise();
+}