@fugood/llama.node 0.3.9 → 0.3.10

package/lib/binding.js

@@ -52,11 +52,11 @@ const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         if (variant && variant !== 'default') {
             setupEnv(variant);
-            return yield Promise.resolve(`${`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
+            return (yield Promise.resolve(`${`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s))));
         }
     }
     catch (_a) { } // ignore errors and try the common path
     setupEnv();
-    return yield Promise.resolve(`${`../bin/${process.platform}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
+    return (yield Promise.resolve(`${`../bin/${process.platform}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s))));
 });
 exports.loadModule = loadModule;

package/lib/binding.ts

@@ -2,11 +2,12 @@ import * as path from 'path'
 
 export type ChatMessage = {
   role: string
-  text: string
+  content: string
 }
 
 export type LlamaModelOptions = {
   model: string
+  chat_template?: string
   embedding?: boolean
   embd_normalize?: number
   pooling_type?: 'none' | 'mean' | 'cls' | 'last' | 'rank'
@@ -42,10 +43,24 @@ export type LlamaModelOptions = {
   lora_list?: { path: string; scaled: number }[]
 }
 
+export type CompletionResponseFormat = {
+  type: 'text' | 'json_object' | 'json_schema'
+  json_schema?: {
+    strict?: boolean
+    schema: object
+  }
+  schema?: object // for json_object type
+}
+
 export type LlamaCompletionOptions = {
   messages?: ChatMessage[]
+  jinja?: boolean
+  chat_template?: string
+  response_format?: CompletionResponseFormat
+  tools?: object
+  parallel_tool_calls?: boolean
+  tool_choice?: string
   prompt?: string
-  n_samples?: number
   temperature?: number
   top_k?: number
   top_p?: number
@@ -70,6 +85,9 @@ export type LlamaCompletionOptions = {
   seed?: number
   stop?: string[]
   grammar?: string
+  grammar_lazy?: boolean
+  grammar_triggers?: { word: string; at_start: boolean }[]
+  preserved_tokens?: string[]
 }
 
 export type LlamaCompletionResult = {
@@ -105,8 +123,21 @@ export interface LlamaContext {
   new (options: LlamaModelOptions): LlamaContext
   getSystemInfo(): string
   getModelInfo(): object
-  getFormattedChat(messages: ChatMessage[]): string
-  completion(options: LlamaCompletionOptions, callback?: (token: LlamaCompletionToken) => void): Promise<LlamaCompletionResult>
+  getFormattedChat(
+    messages: ChatMessage[],
+    chat_template?: string,
+    params?: {
+      jinja?: boolean
+      response_format?: CompletionResponseFormat
+      tools?: object
+      parallel_tool_calls?: object
+      tool_choice?: string
+    },
+  ): object | string
+  completion(
+    options: LlamaCompletionOptions,
+    callback?: (token: LlamaCompletionToken) => void,
+  ): Promise<LlamaCompletionResult>
   stopCompletion(): void
   tokenize(text: string): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
@@ -129,7 +160,10 @@ export type LibVariant = 'default' | 'vulkan' | 'cuda'
 
 const setupEnv = (variant?: string) => {
   const postfix = variant ? `-${variant}` : ''
-  const binPath = path.resolve(__dirname, `../bin/${process.platform}${postfix}/${process.arch}/`)
+  const binPath = path.resolve(
+    __dirname,
+    `../bin/${process.platform}${postfix}/${process.arch}/`,
+  )
   const systemPathEnv = process.env.PATH ?? process.env.Path ?? ''
   if (!systemPathEnv.includes(binPath)) {
     if (process.platform === 'win32') {
@@ -144,9 +178,13 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
   try {
     if (variant && variant !== 'default') {
       setupEnv(variant)
-      return await import(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`) as Module
+      return (await import(
+        `../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`
+      )) as Module
     }
   } catch {} // ignore errors and try the common path
   setupEnv()
-  return await import(`../bin/${process.platform}/${process.arch}/llama-node.node`) as Module
-}
+  return (await import(
+    `../bin/${process.platform}/${process.arch}/llama-node.node`
+  )) as Module
+}

package/lib/index.ts

@@ -9,7 +9,9 @@ export interface LlamaModelOptionsExtended extends LlamaModelOptions {
 
 const mods: { [key: string]: Module } = {}
 
-export const loadModel = async (options: LlamaModelOptionsExtended): Promise<LlamaContext> => {
+export const loadModel = async (
+  options: LlamaModelOptionsExtended,
+): Promise<LlamaContext> => {
   const variant = options.lib_variant ?? 'default'
   mods[variant] ??= await loadModule(options.lib_variant)
   return new mods[variant].LlamaContext(options)

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.3.9",
+  "version": "0.3.10",
   "description": "Llama.cpp for Node.js",
   "main": "lib/index.js",
   "scripts": {
@@ -77,5 +77,12 @@
     "testMatch": [
       "**/*.test.ts"
     ]
+  },
+  "prettier": {
+    "trailingComma": "all",
+    "tabWidth": 2,
+    "semi": false,
+    "singleQuote": true,
+    "printWidth": 80
   }
 }

package/src/LlamaCompletionWorker.cpp

@@ -35,9 +35,10 @@ size_t findStoppingStrings(const std::string &text,
 LlamaCompletionWorker::LlamaCompletionWorker(
     const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
     Napi::Function callback, common_params params,
-    std::vector<std::string> stop_words)
+    std::vector<std::string> stop_words,
+    int32_t chat_format)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
-      _params(params), _stop_words(stop_words) {
+      _params(params), _stop_words(stop_words), _chat_format(chat_format) {
   if (!callback.IsEmpty()) {
     _tsfn = Napi::ThreadSafeFunction::New(info.Env(), callback,
                                           "LlamaCompletionCallback", 0, 1);
@@ -152,15 +153,41 @@ void LlamaCompletionWorker::Execute() {
 }
 
 void LlamaCompletionWorker::OnOK() {
-  auto result = Napi::Object::New(Napi::AsyncWorker::Env());
-  result.Set("tokens_evaluated", Napi::Number::New(Napi::AsyncWorker::Env(),
+  auto env = Napi::AsyncWorker::Env();
+  auto result = Napi::Object::New(env);
+  result.Set("tokens_evaluated", Napi::Number::New(env,
                                                    _result.tokens_evaluated));
   result.Set("tokens_predicted", Napi::Number::New(Napi::AsyncWorker::Env(),
                                                    _result.tokens_predicted));
   result.Set("truncated",
-             Napi::Boolean::New(Napi::AsyncWorker::Env(), _result.truncated));
+             Napi::Boolean::New(env, _result.truncated));
   result.Set("text",
-             Napi::String::New(Napi::AsyncWorker::Env(), _result.text.c_str()));
+             Napi::String::New(env, _result.text.c_str()));
+
+  Napi::Array tool_calls = Napi::Array::New(Napi::AsyncWorker::Env());
+  if (!_stop) {
+    try {
+      common_chat_msg message = common_chat_parse(_result.text, static_cast<common_chat_format>(_chat_format));
+      for (size_t i = 0; i < message.tool_calls.size(); i++) {
+        const auto &tc = message.tool_calls[i];
+        Napi::Object tool_call = Napi::Object::New(env);
+        tool_call.Set("type", "function");
+        Napi::Object function = Napi::Object::New(env);
+        function.Set("name", tc.name);
+        function.Set("arguments", tc.arguments);
+        tool_call.Set("function", function);
+        if (!tc.id.empty()) {
+          tool_call.Set("id", tc.id);
+        }
+        tool_calls.Set(i, tool_call);
+      }
+    } catch (const std::exception &e) {
+      // console_log(env, "Error parsing tool calls: " + std::string(e.what()));
+    }
+  }
+  if (tool_calls.Length() > 0) {
+    result.Set("tool_calls", tool_calls);
+  }
 
   auto ctx = _sess->context();
   const auto timings_token = llama_perf_context(ctx);

package/src/LlamaCompletionWorker.h

@@ -13,7 +13,8 @@ class LlamaCompletionWorker : public Napi::AsyncWorker,
 public:
   LlamaCompletionWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
                         Napi::Function callback, common_params params,
-                        std::vector<std::string> stop_words = {});
+                        std::vector<std::string> stop_words = {},
+                        int32_t chat_format = 0);
 
   ~LlamaCompletionWorker();
 
@@ -30,6 +31,7 @@ private:
   LlamaSessionPtr _sess;
   common_params _params;
   std::vector<std::string> _stop_words;
+  int32_t _chat_format;
   Napi::ThreadSafeFunction _tsfn;
   bool _has_callback = false;
   bool _stop = false;