@fugood/llama.node 1.3.0-rc.2 → 1.3.0-rc.5

package/README.md

@@ -37,11 +37,9 @@ import { loadModel } from '@fugood/llama.node'
 // Initial a Llama context with the model (may take a while)
 const context = await loadModel({
   model: 'path/to/gguf/model',
-  use_mlock: true,
   n_ctx: 2048,
-  n_gpu_layers: 1, // > 0: enable GPU
-  // embedding: true, // use embedding
-  // lib_variant: 'opencl', // Change backend
+  n_gpu_layers: 99, // > 0: enable GPU
+  // lib_variant: 'vulkan', // Change backend
 })
 
 // Do completion

package/lib/binding.js

@@ -42,7 +42,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadModule = void 0;
+exports.isLibVariantAvailable = exports.loadModule = void 0;
 const getPlatformPackageName = (variant) => {
     const platform = process.platform;
     const arch = process.arch;
@@ -72,3 +72,21 @@ const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
     return (yield Promise.resolve().then(() => __importStar(require('../build/Release/index.node'))));
 });
 exports.loadModule = loadModule;
+const isLibVariantAvailable = (variant) => __awaiter(void 0, void 0, void 0, function* () {
+    if (variant && variant !== 'default') {
+        const module = yield loadPlatformPackage(getPlatformPackageName(variant));
+        return module != null;
+    }
+    const defaultModule = yield loadPlatformPackage(getPlatformPackageName());
+    if (defaultModule)
+        return true;
+    try {
+        // @ts-ignore
+        yield Promise.resolve().then(() => __importStar(require('../build/Release/index.node')));
+        return true;
+    }
+    catch (error) {
+        return false;
+    }
+});
+exports.isLibVariantAvailable = isLibVariantAvailable;

package/lib/binding.ts

@@ -375,7 +375,7 @@ export type ToolCall = {
 }
 
 export interface LlamaContext {
-  new (options: LlamaModelOptions): LlamaContext
+  new (options: LlamaModelOptions, onProgress?: (progress: number) => void): LlamaContext
   getSystemInfo(): string
   getModelInfo(): ModelInfo
   getFormattedChat(
@@ -587,3 +587,21 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
   // @ts-ignore
   return (await import('../build/Release/index.node')) as Module
 }
+
+export const isLibVariantAvailable = async (variant?: LibVariant): Promise<boolean> => {
+  if (variant && variant !== 'default') {
+    const module = await loadPlatformPackage(getPlatformPackageName(variant))
+    return module != null
+  }
+
+  const defaultModule = await loadPlatformPackage(getPlatformPackageName())
+  if (defaultModule) return true
+
+  try {
+    // @ts-ignore
+    await import('../build/Release/index.node')
+    return true
+  } catch (error) {
+    return false
+  }
+}

package/lib/index.js

@@ -23,14 +23,14 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BuildInfo = exports.getBackendDevicesInfo = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.MTMD_DEFAULT_MEDIA_MARKER = exports.LlamaParallelAPI = void 0;
+exports.BuildInfo = exports.getBackendDevicesInfo = exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.LlamaParallelAPI = void 0;
 exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
 const version_1 = require("./version");
 const parallel_1 = require("./parallel");
 Object.defineProperty(exports, "LlamaParallelAPI", { enumerable: true, get: function () { return parallel_1.LlamaParallelAPI; } });
+const utils_1 = require("./utils");
 __exportStar(require("./binding"), exports);
-exports.MTMD_DEFAULT_MEDIA_MARKER = '<__media__>';
 const mods = {};
 const logListeners = [];
 const logCallback = (level, text) => {
@@ -83,60 +83,9 @@ class LlamaContextWrapper {
     isLlamaChatSupported() {
         return !!this.ctx.getModelInfo().chatTemplates.llamaChat;
     }
-    _formatMediaChat(messages) {
-        if (!messages)
-            return {
-                messages,
-                has_media: false,
-            };
-        const mediaPaths = [];
-        return {
-            messages: messages.map((msg) => {
-                if (Array.isArray(msg.content)) {
-                    const content = msg.content.map((part) => {
-                        var _a;
-                        // Handle multimodal content
-                        if (part.type === 'image_url') {
-                            let path = ((_a = part.image_url) === null || _a === void 0 ? void 0 : _a.url) || '';
-                            mediaPaths.push(path);
-                            return {
-                                type: 'text',
-                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
-                            };
-                        }
-                        else if (part.type === 'input_audio') {
-                            const { input_audio: audio } = part;
-                            if (!audio)
-                                throw new Error('input_audio is required');
-                            const { format } = audio;
-                            if (format != 'wav' && format != 'mp3') {
-                                throw new Error(`Unsupported audio format: ${format}`);
-                            }
-                            if (audio.url) {
-                                const path = audio.url.replace(/file:\/\//, '');
-                                mediaPaths.push(path);
-                            }
-                            else if (audio.data) {
-                                mediaPaths.push(audio.data);
-                            }
-                            return {
-                                type: 'text',
-                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
-                            };
-                        }
-                        return part;
-                    });
-                    return Object.assign(Object.assign({}, msg), { content });
-                }
-                return msg;
-            }),
-            has_media: mediaPaths.length > 0,
-            media_paths: mediaPaths,
-        };
-    }
     getFormattedChat(messages, template, params) {
         var _a;
-        const { messages: chat, has_media, media_paths, } = this._formatMediaChat(messages);
+        const { messages: chat, has_media, media_paths, } = (0, utils_1.formatMediaChat)(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
         if (template)
@@ -170,7 +119,7 @@ class LlamaContextWrapper {
             media_paths }, jinjaResult);
     }
     completion(options, callback) {
-        const { messages, media_paths = options.media_paths } = this._formatMediaChat(options.messages);
+        const { messages, media_paths = options.media_paths } = (0, utils_1.formatMediaChat)(options.messages);
         return this.ctx.completion(Object.assign(Object.assign({}, options), { messages, media_paths: options.media_paths || media_paths }), callback || (() => { }));
     }
     stopCompletion() {
@@ -244,12 +193,12 @@ class LlamaContextWrapper {
         return this.ctx.decodeAudioTokens(tokens);
     }
 }
-const loadModel = (options) => __awaiter(void 0, void 0, void 0, function* () {
+const loadModel = (options, onProgress) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;
     const variant = (_a = options.lib_variant) !== null && _a !== void 0 ? _a : 'default';
     (_b = mods[variant]) !== null && _b !== void 0 ? _b : (mods[variant] = yield (0, binding_1.loadModule)(options.lib_variant));
     refreshNativeLogSetup();
-    const nativeCtx = new mods[variant].LlamaContext(options);
+    const nativeCtx = new mods[variant].LlamaContext(options, onProgress);
     return new LlamaContextWrapper(nativeCtx);
 });
 exports.loadModel = loadModel;

package/lib/index.ts

@@ -19,12 +19,11 @@ import type {
 } from './binding'
 import { BUILD_NUMBER, BUILD_COMMIT } from './version'
 import { LlamaParallelAPI } from './parallel'
+import { formatMediaChat } from './utils'
 
 export * from './binding'
 export { LlamaParallelAPI }
 
-export const MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'
-
 export interface LlamaModelOptionsExtended extends LlamaModelOptions {
   lib_variant?: LibVariant
 }
@@ -104,63 +103,6 @@ class LlamaContextWrapper {
     return !!this.ctx.getModelInfo().chatTemplates.llamaChat
   }
 
-  _formatMediaChat(messages: ChatMessage[] | undefined): {
-    messages: ChatMessage[] | undefined
-    has_media: boolean
-    media_paths?: string[]
-  } {
-    if (!messages)
-      return {
-        messages,
-        has_media: false,
-      }
-    const mediaPaths: string[] = []
-    return {
-      messages: messages.map((msg) => {
-        if (Array.isArray(msg.content)) {
-          const content = msg.content.map((part) => {
-            // Handle multimodal content
-            if (part.type === 'image_url') {
-              let path = part.image_url?.url || ''
-              mediaPaths.push(path)
-              return {
-                type: 'text',
-                text: MTMD_DEFAULT_MEDIA_MARKER,
-              }
-            } else if (part.type === 'input_audio') {
-              const { input_audio: audio } = part
-              if (!audio) throw new Error('input_audio is required')
-
-              const { format } = audio
-              if (format != 'wav' && format != 'mp3') {
-                throw new Error(`Unsupported audio format: ${format}`)
-              }
-              if (audio.url) {
-                const path = audio.url.replace(/file:\/\//, '')
-                mediaPaths.push(path)
-              } else if (audio.data) {
-                mediaPaths.push(audio.data)
-              }
-              return {
-                type: 'text',
-                text: MTMD_DEFAULT_MEDIA_MARKER,
-              }
-            }
-            return part
-          })
-
-          return {
-            ...msg,
-            content,
-          }
-        }
-        return msg
-      }),
-      has_media: mediaPaths.length > 0,
-      media_paths: mediaPaths,
-    }
-  }
-
   getFormattedChat(
     messages: ChatMessage[],
     template?: string,
@@ -180,7 +122,7 @@ class LlamaContextWrapper {
       messages: chat,
       has_media,
       media_paths,
-    } = this._formatMediaChat(messages)
+    } = formatMediaChat(messages)
 
     const useJinja = this.isJinjaSupported() && params?.jinja
     let tmpl
@@ -228,7 +170,7 @@ class LlamaContextWrapper {
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult> {
     const { messages, media_paths = options.media_paths } =
-      this._formatMediaChat(options.messages)
+      formatMediaChat(options.messages)
     return this.ctx.completion(
       {
         ...options,
@@ -357,12 +299,13 @@ class LlamaContextWrapper {
 
 export const loadModel = async (
   options: LlamaModelOptionsExtended,
+  onProgress?: (progress: number) => void,
 ): Promise<LlamaContextWrapper> => {
   const variant = options.lib_variant ?? 'default'
   mods[variant] ??= await loadModule(options.lib_variant)
   refreshNativeLogSetup()
 
-  const nativeCtx = new mods[variant].LlamaContext(options)
+  const nativeCtx = new mods[variant].LlamaContext(options, onProgress)
   return new LlamaContextWrapper(nativeCtx)
 }
 

package/lib/parallel.js

@@ -10,6 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LlamaParallelAPI = void 0;
+const utils_1 = require("./utils");
 class LlamaParallelAPI {
     constructor(context) {
         this.enabled = false;
@@ -87,8 +88,9 @@ class LlamaParallelAPI {
                     }
                 }
                 : undefined;
+            const { messages, media_paths = options.media_paths } = (0, utils_1.formatMediaChat)(options.messages);
             // Queue the completion immediately (this is synchronous!)
-            const { requestId } = this.context.queueCompletion(options, tokenCallback ||
+            const { requestId } = this.context.queueCompletion(Object.assign(Object.assign({}, options), { messages, media_paths: media_paths }), tokenCallback ||
                 ((error, result) => {
                     if (error) {
                         const pendingReq = this.pendingRequests.get(result === null || result === void 0 ? void 0 : result.requestId);

package/lib/parallel.ts

@@ -5,6 +5,7 @@ import type {
   LlamaCompletionToken,
   RerankParams,
 } from './binding'
+import { formatMediaChat } from './utils'
 
 export class LlamaParallelAPI {
   private context: LlamaContext
@@ -109,9 +110,16 @@ export class LlamaParallelAPI {
         }
       : undefined
 
+    const { messages, media_paths = options.media_paths } = formatMediaChat(
+      options.messages,
+    )
     // Queue the completion immediately (this is synchronous!)
     const { requestId } = this.context.queueCompletion(
-      options,
+      {
+        ...options,
+        messages,
+        media_paths: media_paths,
+      },
       tokenCallback ||
         ((error, result) => {
           if (error) {

package/lib/utils.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatMediaChat = exports.MTMD_DEFAULT_MEDIA_MARKER = void 0;
+exports.MTMD_DEFAULT_MEDIA_MARKER = '<__media__>';
+const formatMediaChat = (messages) => {
+    if (!messages)
+        return {
+            messages,
+            has_media: false,
+        };
+    const mediaPaths = [];
+    return {
+        messages: messages.map((msg) => {
+            if (Array.isArray(msg.content)) {
+                const content = msg.content.map((part) => {
+                    var _a;
+                    // Handle multimodal content
+                    if (part.type === 'image_url') {
+                        let path = ((_a = part.image_url) === null || _a === void 0 ? void 0 : _a.url) || '';
+                        mediaPaths.push(path);
+                        return {
+                            type: 'text',
+                            text: exports.MTMD_DEFAULT_MEDIA_MARKER,
+                        };
+                    }
+                    else if (part.type === 'input_audio') {
+                        const { input_audio: audio } = part;
+                        if (!audio)
+                            throw new Error('input_audio is required');
+                        const { format } = audio;
+                        if (format != 'wav' && format != 'mp3') {
+                            throw new Error(`Unsupported audio format: ${format}`);
+                        }
+                        if (audio.url) {
+                            const path = audio.url.replace(/file:\/\//, '');
+                            mediaPaths.push(path);
+                        }
+                        else if (audio.data) {
+                            mediaPaths.push(audio.data);
+                        }
+                        return {
+                            type: 'text',
+                            text: exports.MTMD_DEFAULT_MEDIA_MARKER,
+                        };
+                    }
+                    return part;
+                });
+                return Object.assign(Object.assign({}, msg), { content });
+            }
+            return msg;
+        }),
+        has_media: mediaPaths.length > 0,
+        media_paths: mediaPaths,
+    };
+};
+exports.formatMediaChat = formatMediaChat;

package/lib/utils.ts

@@ -0,0 +1,63 @@
+
+import type {
+  ChatMessage,
+} from './binding'
+
+export const MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'
+
+export const formatMediaChat = (messages: ChatMessage[] | undefined): {
+  messages: ChatMessage[] | undefined
+  has_media: boolean
+  media_paths?: string[]
+} => {
+  if (!messages)
+    return {
+      messages,
+      has_media: false,
+    }
+  const mediaPaths: string[] = []
+  return {
+    messages: messages.map((msg) => {
+      if (Array.isArray(msg.content)) {
+        const content = msg.content.map((part) => {
+          // Handle multimodal content
+          if (part.type === 'image_url') {
+            let path = part.image_url?.url || ''
+            mediaPaths.push(path)
+            return {
+              type: 'text',
+              text: MTMD_DEFAULT_MEDIA_MARKER,
+            }
+          } else if (part.type === 'input_audio') {
+            const { input_audio: audio } = part
+            if (!audio) throw new Error('input_audio is required')
+
+            const { format } = audio
+            if (format != 'wav' && format != 'mp3') {
+              throw new Error(`Unsupported audio format: ${format}`)
+            }
+            if (audio.url) {
+              const path = audio.url.replace(/file:\/\//, '')
+              mediaPaths.push(path)
+            } else if (audio.data) {
+              mediaPaths.push(audio.data)
+            }
+            return {
+              type: 'text',
+              text: MTMD_DEFAULT_MEDIA_MARKER,
+            }
+          }
+          return part
+        })
+
+        return {
+          ...msg,
+          content,
+        }
+      }
+      return msg
+    }),
+    has_media: mediaPaths.length > 0,
+    media_paths: mediaPaths,
+  }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.3.0-rc.2",
+  "version": "1.3.0-rc.5",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,19 +72,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.3.0-rc.2",
-    "@fugood/node-llama-linux-x64-vulkan": "1.3.0-rc.2",
-    "@fugood/node-llama-linux-x64-cuda": "1.3.0-rc.2",
-    "@fugood/node-llama-linux-arm64": "1.3.0-rc.2",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.3.0-rc.2",
-    "@fugood/node-llama-linux-arm64-cuda": "1.3.0-rc.2",
-    "@fugood/node-llama-win32-x64": "1.3.0-rc.2",
-    "@fugood/node-llama-win32-x64-vulkan": "1.3.0-rc.2",
-    "@fugood/node-llama-win32-x64-cuda": "1.3.0-rc.2",
-    "@fugood/node-llama-win32-arm64": "1.3.0-rc.2",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.3.0-rc.2",
-    "@fugood/node-llama-darwin-x64": "1.3.0-rc.2",
-    "@fugood/node-llama-darwin-arm64": "1.3.0-rc.2"
+    "@fugood/node-llama-linux-x64": "1.3.0-rc.5",
+    "@fugood/node-llama-linux-x64-vulkan": "1.3.0-rc.5",
+    "@fugood/node-llama-linux-x64-cuda": "1.3.0-rc.5",
+    "@fugood/node-llama-linux-arm64": "1.3.0-rc.5",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.3.0-rc.5",
+    "@fugood/node-llama-linux-arm64-cuda": "1.3.0-rc.5",
+    "@fugood/node-llama-win32-x64": "1.3.0-rc.5",
+    "@fugood/node-llama-win32-x64-vulkan": "1.3.0-rc.5",
+    "@fugood/node-llama-win32-x64-cuda": "1.3.0-rc.5",
+    "@fugood/node-llama-win32-arm64": "1.3.0-rc.5",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.3.0-rc.5",
+    "@fugood/node-llama-darwin-x64": "1.3.0-rc.5",
+    "@fugood/node-llama-darwin-arm64": "1.3.0-rc.5"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/src/LlamaContext.cpp

@@ -221,7 +221,7 @@ static int32_t pooling_type_from_str(const std::string &s) {
 }
 
 // construct({ model, embedding, n_ctx, n_batch, n_threads, n_gpu_layers,
-// use_mlock, use_mmap }): LlamaContext throws error
+// use_mlock, use_mmap }, onProgress?: (progress: number) => void): LlamaContext throws error
 LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
     : Napi::ObjectWrap<LlamaContext>(info) {
   Napi::Env env = info.Env();
@@ -230,6 +230,16 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   }
   auto options = info[0].As<Napi::Object>();
 
+  // Check if progress callback is provided
+  bool has_progress_callback = info.Length() >= 2 && info[1].IsFunction();
+  if (has_progress_callback) {
+    _progress_tsfn = Napi::ThreadSafeFunction::New(
+        env, info[1].As<Napi::Function>(), "Model Loading Progress", 0, 1,
+        [](Napi::Env) {
+          // Finalizer callback
+        });
+  }
+
   common_params params;
   params.model.path = get_option<std::string>(options, "model", "");
   if (params.model.path.empty()) {
@@ -323,12 +333,55 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
 
   // Use rn-llama context instead of direct session
   _rn_ctx = new llama_rn_context();
+  _rn_ctx->is_load_interrupted = false;
+  _rn_ctx->loading_progress = 0;
+
+  // Set up progress callback if provided
+  if (has_progress_callback) {
+    params.load_progress_callback = [](float progress, void *user_data) {
+      LlamaContext *self = static_cast<LlamaContext *>(user_data);
+      unsigned int percentage = static_cast<unsigned int>(100 * progress);
+
+      // Only call callback if progress increased
+      if (percentage > self->_rn_ctx->loading_progress) {
+        self->_rn_ctx->loading_progress = percentage;
+
+        // Create a heap-allocated copy of the percentage
+        auto *data = new unsigned int(percentage);
+
+        // Queue callback to be executed on the JavaScript thread
+        auto status = self->_progress_tsfn.NonBlockingCall(
+            data, [](Napi::Env env, Napi::Function jsCallback, unsigned int *data) {
+              jsCallback.Call({Napi::Number::New(env, *data)});
+              delete data;
+            });
+
+        // If the call failed, clean up the data
+        if (status != napi_ok) {
+          delete data;
+        }
+      }
+
+      // Return true to continue loading, false to interrupt
+      return !self->_rn_ctx->is_load_interrupted;
+    };
+    params.load_progress_callback_user_data = this;
+  }
+
   if (!_rn_ctx->loadModel(params)) {
+    if (has_progress_callback) {
+      _progress_tsfn.Release();
+    }
     delete _rn_ctx;
     _rn_ctx = nullptr;
     Napi::TypeError::New(env, "Failed to load model").ThrowAsJavaScriptException();
   }
 
+  // Release progress callback after model is loaded
+  if (has_progress_callback) {
+    _progress_tsfn.Release();
+  }
+
   // Handle LoRA adapters through rn-llama
   if (!lora.empty()) {
     _rn_ctx->applyLoraAdapters(lora);
@@ -343,6 +396,11 @@ LlamaContext::~LlamaContext() {
     _context_valid->store(false);
   }
 
+  // Interrupt model loading if in progress
+  if (_rn_ctx) {
+    _rn_ctx->is_load_interrupted = true;
+  }
+
   // The DisposeWorker is responsible for cleanup of _rn_ctx
   // If _rn_ctx is still not null here, it means disposal was not properly initiated
   if (_rn_ctx) {

package/src/LlamaContext.h

@@ -78,4 +78,7 @@ private:
   // Validity flag for async callbacks to prevent use-after-free
   // Shared pointer ensures callbacks can safely check if context is still alive
   std::shared_ptr<std::atomic<bool>> _context_valid;
+
+  // Progress callback support for model loading
+  Napi::ThreadSafeFunction _progress_tsfn;
 };