@fugood/llama.node 0.3.9 → 0.3.11

package/lib/binding.js

@@ -52,11 +52,11 @@ const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
     try {
         if (variant && variant !== 'default') {
             setupEnv(variant);
-            return yield Promise.resolve(`${`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
+            return (yield Promise.resolve(`${`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s))));
         }
     }
     catch (_a) { } // ignore errors and try the common path
     setupEnv();
-    return yield Promise.resolve(`${`../bin/${process.platform}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
+    return (yield Promise.resolve(`${`../bin/${process.platform}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s))));
 });
 exports.loadModule = loadModule;

package/lib/binding.ts

@@ -2,11 +2,12 @@ import * as path from 'path'
 
 export type ChatMessage = {
   role: string
-  text: string
+  content: string
 }
 
 export type LlamaModelOptions = {
   model: string
+  chat_template?: string
   embedding?: boolean
   embd_normalize?: number
   pooling_type?: 'none' | 'mean' | 'cls' | 'last' | 'rank'
@@ -42,10 +43,24 @@ export type LlamaModelOptions = {
   lora_list?: { path: string; scaled: number }[]
 }
 
+export type CompletionResponseFormat = {
+  type: 'text' | 'json_object' | 'json_schema'
+  json_schema?: {
+    strict?: boolean
+    schema: object
+  }
+  schema?: object // for json_object type
+}
+
 export type LlamaCompletionOptions = {
   messages?: ChatMessage[]
+  jinja?: boolean
+  chat_template?: string
+  response_format?: CompletionResponseFormat
+  tools?: object
+  parallel_tool_calls?: boolean
+  tool_choice?: string
   prompt?: string
-  n_samples?: number
   temperature?: number
   top_k?: number
   top_p?: number
@@ -70,6 +85,9 @@ export type LlamaCompletionOptions = {
   seed?: number
   stop?: string[]
   grammar?: string
+  grammar_lazy?: boolean
+  grammar_triggers?: { word: string; at_start: boolean }[]
+  preserved_tokens?: string[]
 }
 
 export type LlamaCompletionResult = {
@@ -105,8 +123,21 @@ export interface LlamaContext {
   new (options: LlamaModelOptions): LlamaContext
   getSystemInfo(): string
   getModelInfo(): object
-  getFormattedChat(messages: ChatMessage[]): string
-  completion(options: LlamaCompletionOptions, callback?: (token: LlamaCompletionToken) => void): Promise<LlamaCompletionResult>
+  getFormattedChat(
+    messages: ChatMessage[],
+    chat_template?: string,
+    params?: {
+      jinja?: boolean
+      response_format?: CompletionResponseFormat
+      tools?: object
+      parallel_tool_calls?: object
+      tool_choice?: string
+    },
+  ): object | string
+  completion(
+    options: LlamaCompletionOptions,
+    callback?: (token: LlamaCompletionToken) => void,
+  ): Promise<LlamaCompletionResult>
   stopCompletion(): void
   tokenize(text: string): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
@@ -119,6 +150,7 @@ export interface LlamaContext {
   getLoadedLoraAdapters(): { path: string; scaled: number }[]
   // static
   loadModelInfo(path: string, skip: string[]): Promise<Object>
+  toggleNativeLog(enable: boolean, callback: (level: string, text: string) => void): void
 }
 
 export interface Module {
@@ -129,7 +161,10 @@ export type LibVariant = 'default' | 'vulkan' | 'cuda'
 
 const setupEnv = (variant?: string) => {
   const postfix = variant ? `-${variant}` : ''
-  const binPath = path.resolve(__dirname, `../bin/${process.platform}${postfix}/${process.arch}/`)
+  const binPath = path.resolve(
+    __dirname,
+    `../bin/${process.platform}${postfix}/${process.arch}/`,
+  )
   const systemPathEnv = process.env.PATH ?? process.env.Path ?? ''
   if (!systemPathEnv.includes(binPath)) {
     if (process.platform === 'win32') {
@@ -144,9 +179,13 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
   try {
     if (variant && variant !== 'default') {
       setupEnv(variant)
-      return await import(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`) as Module
+      return (await import(
+        `../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`
+      )) as Module
     }
   } catch {} // ignore errors and try the common path
   setupEnv()
-  return await import(`../bin/${process.platform}/${process.arch}/llama-node.node`) as Module
-}
+  return (await import(
+    `../bin/${process.platform}/${process.arch}/llama-node.node`
+  )) as Module
+}

package/lib/index.js

@@ -23,7 +23,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = void 0;
+exports.toggleNativeLog = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = void 0;
+exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
 __exportStar(require("./binding"), exports);
 const mods = {};
@@ -49,3 +50,22 @@ const loadLlamaModelInfo = (path) => __awaiter(void 0, void 0, void 0, function*
     return mods[variant].LlamaContext.loadModelInfo(path, modelInfoSkip);
 });
 exports.loadLlamaModelInfo = loadLlamaModelInfo;
+const logListeners = [];
+const logCallback = (level, text) => {
+    logListeners.forEach((listener) => listener(level, text));
+};
+const toggleNativeLog = (enable, options) => __awaiter(void 0, void 0, void 0, function* () {
+    var _a, _b;
+    const v = (_a = options === null || options === void 0 ? void 0 : options.variant) !== null && _a !== void 0 ? _a : 'default';
+    (_b = mods[v]) !== null && _b !== void 0 ? _b : (mods[v] = yield (0, binding_1.loadModule)(v));
+    return mods[v].LlamaContext.toggleNativeLog(enable, logCallback);
+});
+exports.toggleNativeLog = toggleNativeLog;
+function addNativeLogListener(listener) {
+    logListeners.push(listener);
+    return {
+        remove: () => {
+            logListeners.splice(logListeners.indexOf(listener), 1);
+        },
+    };
+}

package/lib/index.ts

@@ -9,7 +9,9 @@ export interface LlamaModelOptionsExtended extends LlamaModelOptions {
 
 const mods: { [key: string]: Module } = {}
 
-export const loadModel = async (options: LlamaModelOptionsExtended): Promise<LlamaContext> => {
+export const loadModel = async (
+  options: LlamaModelOptionsExtended,
+): Promise<LlamaContext> => {
   const variant = options.lib_variant ?? 'default'
   mods[variant] ??= await loadModule(options.lib_variant)
   return new mods[variant].LlamaContext(options)
@@ -30,3 +32,31 @@ export const loadLlamaModelInfo = async (path: string): Promise<Object> => {
   mods[variant] ??= await loadModule(variant)
   return mods[variant].LlamaContext.loadModelInfo(path, modelInfoSkip)
 }
+
+const logListeners: Array<(level: string, text: string) => void> = []
+
+const logCallback = (level: string, text: string) => {
+  logListeners.forEach((listener) => listener(level, text))
+}
+
+export const toggleNativeLog = async (
+  enable: boolean,
+  options?: {
+    variant?: LibVariant
+  },
+) => {
+  const v = options?.variant ?? 'default'
+  mods[v] ??= await loadModule(v)
+  return mods[v].LlamaContext.toggleNativeLog(enable, logCallback)
+}
+
+export function addNativeLogListener(
+  listener: (level: string, text: string) => void,
+): { remove: () => void } {
+  logListeners.push(listener)
+  return {
+    remove: () => {
+      logListeners.splice(logListeners.indexOf(listener), 1)
+    },
+  }
+}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.3.9",
-  "description": "Llama.cpp for Node.js",
+  "version": "0.3.11",
+  "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
     "test": "jest",
@@ -22,7 +22,9 @@
     "llama",
     "llm",
     "ai",
-    "genai"
+    "genai",
+    "Local LLM",
+    "llama.cpp"
   ],
   "author": "Hans <hans.chen@bricks.tools>",
   "license": "MIT",
@@ -77,5 +79,12 @@
     "testMatch": [
       "**/*.test.ts"
     ]
+  },
+  "prettier": {
+    "trailingComma": "all",
+    "tabWidth": 2,
+    "semi": false,
+    "singleQuote": true,
+    "printWidth": 80
   }
 }

package/src/LlamaCompletionWorker.cpp

@@ -35,9 +35,10 @@ size_t findStoppingStrings(const std::string &text,
 LlamaCompletionWorker::LlamaCompletionWorker(
     const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
     Napi::Function callback, common_params params,
-    std::vector<std::string> stop_words)
+    std::vector<std::string> stop_words,
+    int32_t chat_format)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
-      _params(params), _stop_words(stop_words) {
+      _params(params), _stop_words(stop_words), _chat_format(chat_format) {
   if (!callback.IsEmpty()) {
     _tsfn = Napi::ThreadSafeFunction::New(info.Env(), callback,
                                           "LlamaCompletionCallback", 0, 1);
@@ -152,15 +153,41 @@ void LlamaCompletionWorker::Execute() {
 }
 
 void LlamaCompletionWorker::OnOK() {
-  auto result = Napi::Object::New(Napi::AsyncWorker::Env());
-  result.Set("tokens_evaluated", Napi::Number::New(Napi::AsyncWorker::Env(),
+  auto env = Napi::AsyncWorker::Env();
+  auto result = Napi::Object::New(env);
+  result.Set("tokens_evaluated", Napi::Number::New(env,
                                                    _result.tokens_evaluated));
   result.Set("tokens_predicted", Napi::Number::New(Napi::AsyncWorker::Env(),
                                                    _result.tokens_predicted));
   result.Set("truncated",
-             Napi::Boolean::New(Napi::AsyncWorker::Env(), _result.truncated));
+             Napi::Boolean::New(env, _result.truncated));
   result.Set("text",
-             Napi::String::New(Napi::AsyncWorker::Env(), _result.text.c_str()));
+             Napi::String::New(env, _result.text.c_str()));
+
+  Napi::Array tool_calls = Napi::Array::New(Napi::AsyncWorker::Env());
+  if (!_stop) {
+    try {
+      common_chat_msg message = common_chat_parse(_result.text, static_cast<common_chat_format>(_chat_format));
+      for (size_t i = 0; i < message.tool_calls.size(); i++) {
+        const auto &tc = message.tool_calls[i];
+        Napi::Object tool_call = Napi::Object::New(env);
+        tool_call.Set("type", "function");
+        Napi::Object function = Napi::Object::New(env);
+        function.Set("name", tc.name);
+        function.Set("arguments", tc.arguments);
+        tool_call.Set("function", function);
+        if (!tc.id.empty()) {
+          tool_call.Set("id", tc.id);
+        }
+        tool_calls.Set(i, tool_call);
+      }
+    } catch (const std::exception &e) {
+      // console_log(env, "Error parsing tool calls: " + std::string(e.what()));
+    }
+  }
+  if (tool_calls.Length() > 0) {
+    result.Set("tool_calls", tool_calls);
+  }
 
   auto ctx = _sess->context();
   const auto timings_token = llama_perf_context(ctx);

package/src/LlamaCompletionWorker.h

@@ -13,7 +13,8 @@ class LlamaCompletionWorker : public Napi::AsyncWorker,
 public:
   LlamaCompletionWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
                         Napi::Function callback, common_params params,
-                        std::vector<std::string> stop_words = {});
+                        std::vector<std::string> stop_words = {},
+                        int32_t chat_format = 0);
 
   ~LlamaCompletionWorker();
 
@@ -30,6 +31,7 @@ private:
   LlamaSessionPtr _sess;
   common_params _params;
   std::vector<std::string> _stop_words;
+  int32_t _chat_format;
   Napi::ThreadSafeFunction _tsfn;
   bool _has_callback = false;
   bool _stop = false;