@fugood/llama.node 0.4.7 → 0.5.0

package/lib/binding.ts

@@ -6,6 +6,11 @@ export type MessagePart = {
   text?: string,
   image_url?: {
     url?: string
+  },
+  input_audio?: {
+    format: string
+    data?: string
+    url?: string
   }
 }
 
@@ -103,12 +108,12 @@ export type LlamaCompletionOptions = {
   grammar_triggers?: { type: number; word: string; at_start: boolean }[]
   preserved_tokens?: string[]
   /**
-   * Path(s) to image file(s) to process before generating text.
-   * When provided, the image(s) will be processed and added to the context.
+   * Path(s) to media file(s) to process before generating text.
+   * When provided, the media will be processed and added to the context.
    * Requires multimodal support to be enabled via initMultimodal.
    * Supports both file paths and base64 data URLs.
    */
-  image_paths?: string | string[]
+  media_paths?: string | string[]
 }
 
 export type LlamaCompletionResult = {
@@ -137,10 +142,10 @@ export type LlamaCompletionToken = {
 
 export type TokenizeResult = {
   tokens: Int32Array
-  has_image: boolean
+  has_media: boolean
   bitmap_hashes: string[]
   chunk_pos: number[]
-  chunk_pos_images: number[]
+  chunk_pos_media: number[]
 }
 
 export type EmbeddingResult = {
@@ -167,7 +172,7 @@ export interface LlamaContext {
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult>
   stopCompletion(): void
-  tokenize(text: string, image_paths?: string[]): Promise<TokenizeResult>
+  tokenize(text: string, media_paths?: string[]): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
   embedding(text: string): Promise<EmbeddingResult>
   saveSession(path: string): Promise<void>
@@ -189,6 +194,15 @@ export interface LlamaContext {
    */
   isMultimodalEnabled(): Promise<boolean>
 
+  /**
+   * Get multimodal support capabilities
+   * @returns Promise resolving to an object with vision and audio support
+   */
+  getMultimodalSupport(): Promise<{
+    vision: boolean
+    audio: boolean
+  }>
+
   /**
    * Release multimodal support
    */

package/lib/index.js

@@ -23,10 +23,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = void 0;
+exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.MTMD_DEFAULT_MEDIA_MARKER = void 0;
 exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
 __exportStar(require("./binding"), exports);
+exports.MTMD_DEFAULT_MEDIA_MARKER = '<__media__>';
 const mods = {};
 const logListeners = [];
 const logCallback = (level, text) => {
@@ -78,13 +79,13 @@ class LlamaContextWrapper {
     isLlamaChatSupported() {
         return !!this.ctx.getModelInfo().chatTemplates.llamaChat;
     }
-    _formatImageChat(messages) {
+    _formatMediaChat(messages) {
         if (!messages)
             return {
                 messages,
-                has_image: false,
+                has_media: false,
             };
-        const imagePaths = [];
+        const mediaPaths = [];
         return {
             messages: messages.map((msg) => {
                 if (Array.isArray(msg.content)) {
@@ -93,10 +94,30 @@ class LlamaContextWrapper {
                         // Handle multimodal content
                         if (part.type === 'image_url') {
                             let path = ((_a = part.image_url) === null || _a === void 0 ? void 0 : _a.url) || '';
-                            imagePaths.push(path);
+                            mediaPaths.push(path);
                             return {
                                 type: 'text',
-                                text: '<__image__>',
+                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
+                            };
+                        }
+                        else if (part.type === 'input_audio') {
+                            const { input_audio: audio } = part;
+                            if (!audio)
+                                throw new Error('input_audio is required');
+                            const { format } = audio;
+                            if (format != 'wav' && format != 'mp3') {
+                                throw new Error(`Unsupported audio format: ${format}`);
+                            }
+                            if (audio.url) {
+                                const path = audio.url.replace(/file:\/\//, '');
+                                mediaPaths.push(path);
+                            }
+                            else if (audio.data) {
+                                mediaPaths.push(audio.data);
+                            }
+                            return {
+                                type: 'text',
+                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
                             };
                         }
                         return part;
@@ -105,12 +126,12 @@ class LlamaContextWrapper {
                 }
                 return msg;
             }),
-            has_image: imagePaths.length > 0,
-            image_paths: imagePaths,
+            has_media: mediaPaths.length > 0,
+            media_paths: mediaPaths,
         };
     }
     getFormattedChat(messages, template, params) {
-        const { messages: chat, has_image, image_paths, } = this._formatImageChat(messages);
+        const { messages: chat, has_media, media_paths, } = this._formatMediaChat(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
         if (template)
@@ -127,25 +148,25 @@ class LlamaContextWrapper {
             return {
                 type: 'llama-chat',
                 prompt: result,
-                has_image,
-                image_paths,
+                has_media,
+                media_paths,
             };
         }
         const jinjaResult = result;
         jinjaResult.type = 'jinja';
-        jinjaResult.has_image = has_image;
-        jinjaResult.image_paths = image_paths;
+        jinjaResult.has_media = has_media;
+        jinjaResult.media_paths = media_paths;
         return jinjaResult;
     }
     completion(options, callback) {
-        const { messages, image_paths = options.image_paths } = this._formatImageChat(options.messages);
-        return this.ctx.completion(Object.assign(Object.assign({}, options), { messages, image_paths: options.image_paths || image_paths }), callback || (() => { }));
+        const { messages, media_paths = options.media_paths } = this._formatMediaChat(options.messages);
+        return this.ctx.completion(Object.assign(Object.assign({}, options), { messages, media_paths: options.media_paths || media_paths }), callback || (() => { }));
     }
     stopCompletion() {
         return this.ctx.stopCompletion();
     }
-    tokenize(text, { image_paths } = {}) {
-        return this.ctx.tokenize(text, image_paths);
+    tokenize(text, { media_paths } = {}) {
+        return this.ctx.tokenize(text, media_paths);
     }
     detokenize(tokens) {
         return this.ctx.detokenize(tokens);
@@ -180,6 +201,9 @@ class LlamaContextWrapper {
     releaseMultimodal() {
         return this.ctx.releaseMultimodal();
     }
+    getMultimodalSupport() {
+        return this.ctx.getMultimodalSupport();
+    }
 }
 const loadModel = (options) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;

package/lib/index.ts

@@ -14,6 +14,8 @@ import type {
 
 export * from './binding'
 
+export const MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'
+
 export interface LlamaModelOptionsExtended extends LlamaModelOptions {
   lib_variant?: LibVariant
 }
@@ -63,8 +65,8 @@ const getJsonSchema = (responseFormat?: CompletionResponseFormat) => {
 export type FormattedChatResult = {
   type: 'jinja' | 'llama-chat'
   prompt: string
-  has_image: boolean
-  image_paths?: Array<string>
+  has_media: boolean
+  media_paths?: Array<string>
 }
 
 class LlamaContextWrapper {
@@ -91,17 +93,17 @@ class LlamaContextWrapper {
     return !!this.ctx.getModelInfo().chatTemplates.llamaChat
   }
 
-  _formatImageChat(messages: ChatMessage[] | undefined): {
+  _formatMediaChat(messages: ChatMessage[] | undefined): {
     messages: ChatMessage[] | undefined
-    has_image: boolean
-    image_paths?: string[]
+    has_media: boolean
+    media_paths?: string[]
   } {
     if (!messages)
       return {
         messages,
-        has_image: false,
+        has_media: false,
       }
-    const imagePaths: string[] = []
+    const mediaPaths: string[] = []
     return {
       messages: messages.map((msg) => {
         if (Array.isArray(msg.content)) {
@@ -109,10 +111,28 @@ class LlamaContextWrapper {
             // Handle multimodal content
             if (part.type === 'image_url') {
               let path = part.image_url?.url || ''
-              imagePaths.push(path)
+              mediaPaths.push(path)
               return {
                 type: 'text',
-                text: '<__image__>',
+                text: MTMD_DEFAULT_MEDIA_MARKER,
+              }
+            } else if (part.type === 'input_audio') {
+              const { input_audio: audio } = part
+              if (!audio) throw new Error('input_audio is required')
+
+              const { format } = audio
+              if (format != 'wav' && format != 'mp3') {
+                throw new Error(`Unsupported audio format: ${format}`)
+              }
+              if (audio.url) {
+                const path = audio.url.replace(/file:\/\//, '')
+                mediaPaths.push(path)
+              } else if (audio.data) {
+                mediaPaths.push(audio.data)
+              }
+              return {
+                type: 'text',
+                text: MTMD_DEFAULT_MEDIA_MARKER,
               }
             }
             return part
@@ -125,8 +145,8 @@ class LlamaContextWrapper {
         }
         return msg
       }),
-      has_image: imagePaths.length > 0,
-      image_paths: imagePaths,
+      has_media: mediaPaths.length > 0,
+      media_paths: mediaPaths,
     }
   }
 
@@ -143,9 +163,9 @@ class LlamaContextWrapper {
   ): FormattedChatResult {
     const {
       messages: chat,
-      has_image,
-      image_paths,
-    } = this._formatImageChat(messages)
+      has_media,
+      media_paths,
+    } = this._formatMediaChat(messages)
 
     const useJinja = this.isJinjaSupported() && params?.jinja
     let tmpl
@@ -164,14 +184,14 @@ class LlamaContextWrapper {
       return {
         type: 'llama-chat',
         prompt: result as string,
-        has_image,
-        image_paths,
+        has_media,
+        media_paths,
       }
     }
     const jinjaResult = result
     jinjaResult.type = 'jinja'
-    jinjaResult.has_image = has_image
-    jinjaResult.image_paths = image_paths
+    jinjaResult.has_media = has_media
+    jinjaResult.media_paths = media_paths
     return jinjaResult
   }
 
@@ -179,12 +199,12 @@ class LlamaContextWrapper {
     options: LlamaCompletionOptions,
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult> {
-    const { messages, image_paths = options.image_paths } =
-      this._formatImageChat(options.messages)
+    const { messages, media_paths = options.media_paths } =
+      this._formatMediaChat(options.messages)
     return this.ctx.completion({
       ...options,
       messages,
-      image_paths: options.image_paths || image_paths,
+      media_paths: options.media_paths || media_paths,
     }, callback || (() => {}))
   }
 
@@ -192,8 +212,8 @@ class LlamaContextWrapper {
     return this.ctx.stopCompletion()
   }
 
-  tokenize(text: string, { image_paths }: { image_paths?: string[] } = {}): Promise<TokenizeResult> {
-    return this.ctx.tokenize(text, image_paths)
+  tokenize(text: string, { media_paths }: { media_paths?: string[] } = {}): Promise<TokenizeResult> {
+    return this.ctx.tokenize(text, media_paths)
   }
 
   detokenize(tokens: number[]): Promise<string> {
@@ -242,6 +262,13 @@ class LlamaContextWrapper {
   releaseMultimodal(): Promise<void> {
     return this.ctx.releaseMultimodal()
   }
+
+  getMultimodalSupport(): Promise<{
+    vision: boolean
+    audio: boolean
+  }> {
+    return this.ctx.getMultimodalSupport()
+  }
 }
 
 export const loadModel = async (

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.4.7",
+  "version": "0.5.0",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {

package/src/LlamaCompletionWorker.cpp

@@ -29,10 +29,10 @@ LlamaCompletionWorker::LlamaCompletionWorker(
     Napi::Function callback, common_params params,
     std::vector<std::string> stop_words,
     int32_t chat_format,
-    std::vector<std::string> image_paths)
+    std::vector<std::string> media_paths)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
       _params(params), _stop_words(stop_words), _chat_format(chat_format),
-      _image_paths(image_paths) {
+      _media_paths(media_paths) {
   if (!callback.IsEmpty()) {
     _tsfn = Napi::ThreadSafeFunction::New(info.Env(), callback,
                                           "LlamaCompletionCallback", 0, 1);
@@ -64,19 +64,19 @@ void LlamaCompletionWorker::Execute() {
   LlamaCppSampling sampling{common_sampler_init(model, _params.sampling),
                             common_sampler_free};
 
-  // Process images if any are provided
-  if (!_image_paths.empty()) {
+  // Process media if any are provided
+  if (!_media_paths.empty()) {
     const auto* mtmd_ctx = _sess->get_mtmd_ctx();
     
     if (mtmd_ctx != nullptr) {
-      // Process the images and get the tokens
+      // Process the media and get the tokens
       try {
-        n_cur = process_image_prompt(
+        n_cur = processMediaPrompt(
           ctx,
           mtmd_ctx,
           _sess,
           _params,
-          _image_paths
+          _media_paths
         );
       } catch (const std::exception& e) {
         SetError(e.what());
@@ -85,12 +85,12 @@ void LlamaCompletionWorker::Execute() {
       }
       
       if (n_cur <= 0) {
-        SetError("Failed to process images");
+        SetError("Failed to process media");
         _sess->get_mutex().unlock();
         return;
       }
 
-      fprintf(stdout, "[DEBUG] Image processing successful, n_cur=%zu, tokens=%zu\n", 
+      fprintf(stdout, "[DEBUG] Media processing successful, n_cur=%zu, tokens=%zu\n", 
                        n_cur, _sess->tokens_ptr()->size());
 
       n_input = _sess->tokens_ptr()->size();

package/src/LlamaCompletionWorker.h

@@ -20,7 +20,7 @@ public:
                         Napi::Function callback, common_params params,
                         std::vector<std::string> stop_words,
                         int32_t chat_format,
-                        std::vector<std::string> image_paths = {});
+                        std::vector<std::string> media_paths = {});
 
   ~LlamaCompletionWorker();
 
@@ -44,7 +44,7 @@ private:
   common_params _params;
   std::vector<std::string> _stop_words;
   int32_t _chat_format;
-  std::vector<std::string> _image_paths;
+  std::vector<std::string> _media_paths;
   std::function<void()> _onComplete;
   bool _has_callback = false;
   bool _stop = false;

package/src/LlamaContext.cpp

@@ -135,6 +135,9 @@ void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
            static_cast<napi_property_attributes>(napi_enumerable)),
        StaticMethod<&LlamaContext::ToggleNativeLog>(
            "toggleNativeLog",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::GetMultimodalSupport>(
+           "getMultimodalSupport",
            static_cast<napi_property_attributes>(napi_enumerable))});
   Napi::FunctionReference *constructor = new Napi::FunctionReference();
   *constructor = Napi::Persistent(func);
@@ -607,22 +610,22 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
     }
   }
 
-  // Process image_paths parameter
-  std::vector<std::string> image_paths;
-  if (options.Has("image_paths")) {
-    if (options.Get("image_paths").IsArray()) {
-      auto image_paths_array = options.Get("image_paths").As<Napi::Array>();
-      for (size_t i = 0; i < image_paths_array.Length(); i++) {
-        image_paths.push_back(image_paths_array.Get(i).ToString().Utf8Value());
+  // Process media_paths parameter
+  std::vector<std::string> media_paths;
+  if (options.Has("media_paths")) {
+    if (options.Get("media_paths").IsArray()) {
+      auto media_paths_array = options.Get("media_paths").As<Napi::Array>();
+      for (size_t i = 0; i < media_paths_array.Length(); i++) {
+        media_paths.push_back(media_paths_array.Get(i).ToString().Utf8Value());
       }
-    } else if (options.Get("image_paths").IsString()) {
-      image_paths.push_back(options.Get("image_paths").ToString().Utf8Value());
+    } else if (options.Get("media_paths").IsString()) {
+      media_paths.push_back(options.Get("media_paths").ToString().Utf8Value());
     }
   }
 
-  // Check if multimodal is enabled when image_paths are provided
-  if (!image_paths.empty() && !(_has_multimodal && _mtmd_ctx != nullptr)) {
-    Napi::Error::New(env, "Multimodal support must be enabled via initMultimodal to use image_paths").ThrowAsJavaScriptException();
+  // Check if multimodal is enabled when media_paths are provided
+  if (!media_paths.empty() && !(_has_multimodal && _mtmd_ctx != nullptr)) {
+    Napi::Error::New(env, "Multimodal support must be enabled via initMultimodal to use media_paths").ThrowAsJavaScriptException();
     return env.Undefined();
   }
 
@@ -808,7 +811,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   }
 
   auto *worker =
-      new LlamaCompletionWorker(info, _sess, callback, params, stop_words, chat_format, image_paths);
+      new LlamaCompletionWorker(info, _sess, callback, params, stop_words, chat_format, media_paths);
   worker->Queue();
   _wip = worker;
   worker->OnComplete([this]() { _wip = nullptr; });
@@ -833,14 +836,14 @@ Napi::Value LlamaContext::Tokenize(const Napi::CallbackInfo &info) {
         .ThrowAsJavaScriptException();
   }
   auto text = info[0].ToString().Utf8Value();
-  std::vector<std::string> image_paths;
+  std::vector<std::string> media_paths;
   if (info.Length() >= 2 && info[1].IsArray()) {
-    auto image_paths_array = info[1].As<Napi::Array>();
-    for (size_t i = 0; i < image_paths_array.Length(); i++) {
-      image_paths.push_back(image_paths_array.Get(i).ToString().Utf8Value());
+    auto media_paths_array = info[1].As<Napi::Array>();
+    for (size_t i = 0; i < media_paths_array.Length(); i++) {
+      media_paths.push_back(media_paths_array.Get(i).ToString().Utf8Value());
     }
   }
-  auto *worker = new TokenizeWorker(info, _sess, text, image_paths);
+  auto *worker = new TokenizeWorker(info, _sess, text, media_paths);
   worker->Queue();
   return worker->Promise();
 }
@@ -1081,6 +1084,22 @@ Napi::Value LlamaContext::IsMultimodalEnabled(const Napi::CallbackInfo &info) {
   return Napi::Boolean::New(info.Env(), _has_multimodal && _mtmd_ctx != nullptr);
 }
 
+// getMultimodalSupport(): Promise<{ vision: boolean, audio: boolean }>
+Napi::Value LlamaContext::GetMultimodalSupport(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  auto result = Napi::Object::New(env);
+  
+  if (_has_multimodal && _mtmd_ctx != nullptr) {
+    result.Set("vision", Napi::Boolean::New(env, mtmd_support_vision(_mtmd_ctx)));
+    result.Set("audio", Napi::Boolean::New(env, mtmd_support_audio(_mtmd_ctx)));
+  } else {
+    result.Set("vision", Napi::Boolean::New(env, false));
+    result.Set("audio", Napi::Boolean::New(env, false));
+  }
+  
+  return result;
+}
+
 // releaseMultimodal(): void
 void LlamaContext::ReleaseMultimodal(const Napi::CallbackInfo &info) {
   if (_mtmd_ctx != nullptr) {

package/src/LlamaContext.h

@@ -31,6 +31,7 @@ private:
   // Multimodal methods
   Napi::Value InitMultimodal(const Napi::CallbackInfo &info);
   Napi::Value IsMultimodalEnabled(const Napi::CallbackInfo &info);
+  Napi::Value GetMultimodalSupport(const Napi::CallbackInfo &info);
   void ReleaseMultimodal(const Napi::CallbackInfo &info);
 
   std::string _info;

package/src/TokenizeWorker.cpp

@@ -2,17 +2,22 @@
 #include "LlamaContext.h"
 
 TokenizeWorker::TokenizeWorker(const Napi::CallbackInfo &info,
-                               LlamaSessionPtr &sess, std::string text, std::vector<std::string> image_paths)
-    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text), _image_paths(image_paths) {}
+                               LlamaSessionPtr &sess, std::string text, std::vector<std::string> media_paths)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text), _media_paths(media_paths) {}
 
 void TokenizeWorker::Execute() {
   auto mtmd_ctx = _sess->get_mtmd_ctx();
-  if (!_image_paths.empty()) {
-    _result = tokenizeWithImages(mtmd_ctx, _text, _image_paths);
+  if (!_media_paths.empty()) {
+    try {
+      _result = tokenizeWithMedia(mtmd_ctx, _text, _media_paths);
+      mtmd_input_chunks_free(_result.chunks);
+    } catch (const std::exception &e) {
+      SetError(e.what());
+    }
   } else {
     const auto tokens = common_tokenize(_sess->context(), _text, false);
     _result.tokens = tokens;
-    _result.has_image = false;
+    _result.has_media = false;
   }
 }
 
@@ -24,9 +29,8 @@ void TokenizeWorker::OnOK() {
   memcpy(tokens.Data(), _result.tokens.data(),
          _result.tokens.size() * sizeof(llama_token));
   result.Set("tokens", tokens);
-  if (_result.has_image) {
-    result.Set("has_image", _result.has_image);
-
+  result.Set("has_media", _result.has_media);
+  if (_result.has_media) {
     auto bitmap_hashes = Napi::Array::New(Napi::AsyncWorker::Env(), _result.bitmap_hashes.size());
     for (size_t i = 0; i < _result.bitmap_hashes.size(); i++) {
       bitmap_hashes.Set(i, _result.bitmap_hashes[i]);
@@ -37,11 +41,11 @@ void TokenizeWorker::OnOK() {
       chunk_pos.Set(i, _result.chunk_pos[i]);
     }
     result.Set("chunk_pos", chunk_pos);
-    auto chunk_pos_images = Napi::Array::New(Napi::AsyncWorker::Env(), _result.chunk_pos_images.size());
-    for (size_t i = 0; i < _result.chunk_pos_images.size(); i++) {
-      chunk_pos_images.Set(i, _result.chunk_pos_images[i]);
+    auto chunk_pos_media = Napi::Array::New(Napi::AsyncWorker::Env(), _result.chunk_pos_media.size());
+    for (size_t i = 0; i < _result.chunk_pos_media.size(); i++) {
+      chunk_pos_media.Set(i, _result.chunk_pos_media[i]);
     }
-    result.Set("chunk_pos_images", chunk_pos_images);
+    result.Set("chunk_pos_media", chunk_pos_media);
   }
   Napi::Promise::Deferred::Resolve(result);
 }

package/src/TokenizeWorker.h

@@ -5,7 +5,7 @@ class TokenizeWorker : public Napi::AsyncWorker,
                        public Napi::Promise::Deferred {
 public:
   TokenizeWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
-                 std::string text, std::vector<std::string> image_paths);
+                 std::string text, std::vector<std::string> media_paths);
 
 protected:
   void Execute();
@@ -15,6 +15,6 @@ protected:
 private:
   LlamaSessionPtr _sess;
   std::string _text;
-  std::vector<std::string> _image_paths;
+  std::vector<std::string> _media_paths;
   TokenizeResult _result;
 };