@fugood/llama.node 0.4.7 → 0.6.0

package/CMakeLists.txt

@@ -102,6 +102,10 @@ file(
     "src/LoadSessionWorker.h"
     "src/SaveSessionWorker.cpp"
     "src/SaveSessionWorker.h"
+    "src/DecodeAudioTokenWorker.cpp"
+    "src/DecodeAudioTokenWorker.h"
+    "src/tts_utils.cpp"
+    "src/tts_utils.h"
 )
 
 add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC})

package/lib/binding.ts

@@ -6,6 +6,11 @@ export type MessagePart = {
   text?: string,
   image_url?: {
     url?: string
+  },
+  input_audio?: {
+    format: string
+    data?: string
+    url?: string
   }
 }
 
@@ -103,12 +108,17 @@ export type LlamaCompletionOptions = {
   grammar_triggers?: { type: number; word: string; at_start: boolean }[]
   preserved_tokens?: string[]
   /**
-   * Path(s) to image file(s) to process before generating text.
-   * When provided, the image(s) will be processed and added to the context.
+   * Path(s) to media file(s) to process before generating text.
+   * When provided, the media will be processed and added to the context.
    * Requires multimodal support to be enabled via initMultimodal.
    * Supports both file paths and base64 data URLs.
    */
-  image_paths?: string | string[]
+  media_paths?: string | string[]
+  /**
+   * Guide tokens to use for audio completion.
+   * Help prevent hallucinations by forcing the TTS to use the correct words.
+   */
+  guide_tokens?: Int32Array
 }
 
 export type LlamaCompletionResult = {
@@ -137,10 +147,10 @@ export type LlamaCompletionToken = {
 
 export type TokenizeResult = {
   tokens: Int32Array
-  has_image: boolean
+  has_media: boolean
   bitmap_hashes: string[]
   chunk_pos: number[]
-  chunk_pos_images: number[]
+  chunk_pos_media: number[]
 }
 
 export type EmbeddingResult = {
@@ -167,7 +177,7 @@ export interface LlamaContext {
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult>
   stopCompletion(): void
-  tokenize(text: string, image_paths?: string[]): Promise<TokenizeResult>
+  tokenize(text: string, media_paths?: string[]): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
   embedding(text: string): Promise<EmbeddingResult>
   saveSession(path: string): Promise<void>
@@ -189,11 +199,61 @@ export interface LlamaContext {
    */
   isMultimodalEnabled(): Promise<boolean>
 
+  /**
+   * Get multimodal support capabilities
+   * @returns Promise resolving to an object with vision and audio support
+   */
+  getMultimodalSupport(): Promise<{
+    vision: boolean
+    audio: boolean
+  }>
+
   /**
    * Release multimodal support
    */
   releaseMultimodal(): Promise<void>
 
+  /**
+   * Load a vocoder model
+   * @param path Path to the vocoder model
+   * @returns Promise resolving to true if loading was successful
+   */
+  initVocoder(path: string): Promise<boolean>
+
+  /**
+   * Unload the vocoder model
+   * @returns Promise resolving to true if unloading was successful
+   */
+  releaseVocoder(): Promise<void>
+
+  /**
+   * Check if the vocoder model is enabled
+   * @returns Promise resolving to true if the vocoder model is enabled
+   */
+  isVocoderEnabled(): boolean
+
+  /**
+   * Get the formatted prompt for audio completion
+   * @param speaker Speaker name or null
+   * @param text Text to complete
+   * @returns Formatted audio completion
+   */
+  getFormattedAudioCompletion(speaker: string|null, text: string): string
+
+  /**
+   * Get guide tokens for audio completion
+   * @param text Text to complete
+   * @returns Guide tokens
+   */
+  getAudioCompletionGuideTokens(text: string): Int32Array
+
+  /**
+   * Decode audio tokens to audio data
+   * @param tokens Tokens to decode
+   * @returns Decoded audio tokens
+   */
+  decodeAudioTokens(tokens: Int32Array): Promise<Float32Array>
+
   // static
   loadModelInfo(path: string, skip: string[]): Promise<Object>
   toggleNativeLog(

package/lib/index.js

@@ -23,10 +23,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = void 0;
+exports.loadLlamaModelInfo = exports.initLlama = exports.loadModel = exports.toggleNativeLog = exports.MTMD_DEFAULT_MEDIA_MARKER = void 0;
 exports.addNativeLogListener = addNativeLogListener;
 const binding_1 = require("./binding");
 __exportStar(require("./binding"), exports);
+exports.MTMD_DEFAULT_MEDIA_MARKER = '<__media__>';
 const mods = {};
 const logListeners = [];
 const logCallback = (level, text) => {
@@ -78,13 +79,13 @@ class LlamaContextWrapper {
     isLlamaChatSupported() {
         return !!this.ctx.getModelInfo().chatTemplates.llamaChat;
     }
-    _formatImageChat(messages) {
+    _formatMediaChat(messages) {
         if (!messages)
             return {
                 messages,
-                has_image: false,
+                has_media: false,
             };
-        const imagePaths = [];
+        const mediaPaths = [];
         return {
             messages: messages.map((msg) => {
                 if (Array.isArray(msg.content)) {
@@ -93,10 +94,30 @@ class LlamaContextWrapper {
                         // Handle multimodal content
                         if (part.type === 'image_url') {
                             let path = ((_a = part.image_url) === null || _a === void 0 ? void 0 : _a.url) || '';
-                            imagePaths.push(path);
+                            mediaPaths.push(path);
                             return {
                                 type: 'text',
-                                text: '<__image__>',
+                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
+                            };
+                        }
+                        else if (part.type === 'input_audio') {
+                            const { input_audio: audio } = part;
+                            if (!audio)
+                                throw new Error('input_audio is required');
+                            const { format } = audio;
+                            if (format != 'wav' && format != 'mp3') {
+                                throw new Error(`Unsupported audio format: ${format}`);
+                            }
+                            if (audio.url) {
+                                const path = audio.url.replace(/file:\/\//, '');
+                                mediaPaths.push(path);
+                            }
+                            else if (audio.data) {
+                                mediaPaths.push(audio.data);
+                            }
+                            return {
+                                type: 'text',
+                                text: exports.MTMD_DEFAULT_MEDIA_MARKER,
                             };
                         }
                         return part;
@@ -105,12 +126,12 @@ class LlamaContextWrapper {
                 }
                 return msg;
             }),
-            has_image: imagePaths.length > 0,
-            image_paths: imagePaths,
+            has_media: mediaPaths.length > 0,
+            media_paths: mediaPaths,
         };
     }
     getFormattedChat(messages, template, params) {
-        const { messages: chat, has_image, image_paths, } = this._formatImageChat(messages);
+        const { messages: chat, has_media, media_paths, } = this._formatMediaChat(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
         if (template)
@@ -127,25 +148,25 @@ class LlamaContextWrapper {
             return {
                 type: 'llama-chat',
                 prompt: result,
-                has_image,
-                image_paths,
+                has_media,
+                media_paths,
             };
         }
         const jinjaResult = result;
         jinjaResult.type = 'jinja';
-        jinjaResult.has_image = has_image;
-        jinjaResult.image_paths = image_paths;
+        jinjaResult.has_media = has_media;
+        jinjaResult.media_paths = media_paths;
         return jinjaResult;
     }
     completion(options, callback) {
-        const { messages, image_paths = options.image_paths } = this._formatImageChat(options.messages);
-        return this.ctx.completion(Object.assign(Object.assign({}, options), { messages, image_paths: options.image_paths || image_paths }), callback || (() => { }));
+        const { messages, media_paths = options.media_paths } = this._formatMediaChat(options.messages);
+        return this.ctx.completion(Object.assign(Object.assign({}, options), { messages, media_paths: options.media_paths || media_paths }), callback || (() => { }));
     }
     stopCompletion() {
         return this.ctx.stopCompletion();
     }
-    tokenize(text, { image_paths } = {}) {
-        return this.ctx.tokenize(text, image_paths);
+    tokenize(text, { media_paths } = {}) {
+        return this.ctx.tokenize(text, media_paths);
     }
     detokenize(tokens) {
         return this.ctx.detokenize(tokens);
@@ -180,6 +201,27 @@ class LlamaContextWrapper {
     releaseMultimodal() {
         return this.ctx.releaseMultimodal();
     }
+    getMultimodalSupport() {
+        return this.ctx.getMultimodalSupport();
+    }
+    initVocoder(path) {
+        return this.ctx.initVocoder(path);
+    }
+    releaseVocoder() {
+        return this.ctx.releaseVocoder();
+    }
+    isVocoderEnabled() {
+        return this.ctx.isVocoderEnabled();
+    }
+    getFormattedAudioCompletion(speaker, text) {
+        return this.ctx.getFormattedAudioCompletion(speaker, text);
+    }
+    getAudioCompletionGuideTokens(text) {
+        return this.ctx.getAudioCompletionGuideTokens(text);
+    }
+    decodeAudioTokens(tokens) {
+        return this.ctx.decodeAudioTokens(tokens);
+    }
 }
 const loadModel = (options) => __awaiter(void 0, void 0, void 0, function* () {
     var _a, _b;

package/lib/index.ts

@@ -14,6 +14,8 @@ import type {
 
 export * from './binding'
 
+export const MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'
+
 export interface LlamaModelOptionsExtended extends LlamaModelOptions {
   lib_variant?: LibVariant
 }
@@ -63,8 +65,8 @@ const getJsonSchema = (responseFormat?: CompletionResponseFormat) => {
 export type FormattedChatResult = {
   type: 'jinja' | 'llama-chat'
   prompt: string
-  has_image: boolean
-  image_paths?: Array<string>
+  has_media: boolean
+  media_paths?: Array<string>
 }
 
 class LlamaContextWrapper {
@@ -91,17 +93,17 @@ class LlamaContextWrapper {
     return !!this.ctx.getModelInfo().chatTemplates.llamaChat
   }
 
-  _formatImageChat(messages: ChatMessage[] | undefined): {
+  _formatMediaChat(messages: ChatMessage[] | undefined): {
     messages: ChatMessage[] | undefined
-    has_image: boolean
-    image_paths?: string[]
+    has_media: boolean
+    media_paths?: string[]
   } {
     if (!messages)
       return {
         messages,
-        has_image: false,
+        has_media: false,
       }
-    const imagePaths: string[] = []
+    const mediaPaths: string[] = []
     return {
       messages: messages.map((msg) => {
         if (Array.isArray(msg.content)) {
@@ -109,10 +111,28 @@ class LlamaContextWrapper {
             // Handle multimodal content
             if (part.type === 'image_url') {
               let path = part.image_url?.url || ''
-              imagePaths.push(path)
+              mediaPaths.push(path)
               return {
                 type: 'text',
-                text: '<__image__>',
+                text: MTMD_DEFAULT_MEDIA_MARKER,
+              }
+            } else if (part.type === 'input_audio') {
+              const { input_audio: audio } = part
+              if (!audio) throw new Error('input_audio is required')
+
+              const { format } = audio
+              if (format != 'wav' && format != 'mp3') {
+                throw new Error(`Unsupported audio format: ${format}`)
+              }
+              if (audio.url) {
+                const path = audio.url.replace(/file:\/\//, '')
+                mediaPaths.push(path)
+              } else if (audio.data) {
+                mediaPaths.push(audio.data)
+              }
+              return {
+                type: 'text',
+                text: MTMD_DEFAULT_MEDIA_MARKER,
               }
             }
             return part
@@ -125,8 +145,8 @@ class LlamaContextWrapper {
         }
         return msg
       }),
-      has_image: imagePaths.length > 0,
-      image_paths: imagePaths,
+      has_media: mediaPaths.length > 0,
+      media_paths: mediaPaths,
     }
   }
 
@@ -143,9 +163,9 @@ class LlamaContextWrapper {
   ): FormattedChatResult {
     const {
       messages: chat,
-      has_image,
-      image_paths,
-    } = this._formatImageChat(messages)
+      has_media,
+      media_paths,
+    } = this._formatMediaChat(messages)
 
     const useJinja = this.isJinjaSupported() && params?.jinja
     let tmpl
@@ -164,14 +184,14 @@ class LlamaContextWrapper {
       return {
         type: 'llama-chat',
         prompt: result as string,
-        has_image,
-        image_paths,
+        has_media,
+        media_paths,
       }
     }
     const jinjaResult = result
     jinjaResult.type = 'jinja'
-    jinjaResult.has_image = has_image
-    jinjaResult.image_paths = image_paths
+    jinjaResult.has_media = has_media
+    jinjaResult.media_paths = media_paths
     return jinjaResult
   }
 
@@ -179,12 +199,12 @@ class LlamaContextWrapper {
     options: LlamaCompletionOptions,
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult> {
-    const { messages, image_paths = options.image_paths } =
-      this._formatImageChat(options.messages)
+    const { messages, media_paths = options.media_paths } =
+      this._formatMediaChat(options.messages)
     return this.ctx.completion({
       ...options,
       messages,
-      image_paths: options.image_paths || image_paths,
+      media_paths: options.media_paths || media_paths,
     }, callback || (() => {}))
   }
 
@@ -192,8 +212,8 @@ class LlamaContextWrapper {
     return this.ctx.stopCompletion()
   }
 
-  tokenize(text: string, { image_paths }: { image_paths?: string[] } = {}): Promise<TokenizeResult> {
-    return this.ctx.tokenize(text, image_paths)
+  tokenize(text: string, { media_paths }: { media_paths?: string[] } = {}): Promise<TokenizeResult> {
+    return this.ctx.tokenize(text, media_paths)
   }
 
   detokenize(tokens: number[]): Promise<string> {
@@ -242,6 +262,37 @@ class LlamaContextWrapper {
   releaseMultimodal(): Promise<void> {
     return this.ctx.releaseMultimodal()
   }
+
+  getMultimodalSupport(): Promise<{
+    vision: boolean
+    audio: boolean
+  }> {
+    return this.ctx.getMultimodalSupport()
+  }
+
+  initVocoder(path: string): Promise<boolean> {
+    return this.ctx.initVocoder(path)
+  }
+
+  releaseVocoder(): Promise<void> {
+    return this.ctx.releaseVocoder()
+  }
+
+  isVocoderEnabled(): boolean {
+    return this.ctx.isVocoderEnabled()
+  }
+
+  getFormattedAudioCompletion(speaker: string|null, text: string): string {
+    return this.ctx.getFormattedAudioCompletion(speaker, text)
+  }
+
+  getAudioCompletionGuideTokens(text: string): Int32Array {
+    return this.ctx.getAudioCompletionGuideTokens(text)
+  }
+
+  decodeAudioTokens(tokens: number[]|Int32Array): Promise<Float32Array> {
+    return this.ctx.decodeAudioTokens(tokens)
+  }
 }
 
 export const loadModel = async (

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.4.7",
+  "version": "0.6.0",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {

package/src/DecodeAudioTokenWorker.cpp

@@ -0,0 +1,40 @@
+#include "DecodeAudioTokenWorker.h"
+#include "tts_utils.h"
+#include <vector>
+
+DecodeAudioTokenWorker::DecodeAudioTokenWorker(
+    const Napi::CallbackInfo &info, llama_model *model, llama_context *ctx,
+    int n_threads, const std::vector<llama_token> &tokens)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _model(model), _ctx(ctx),
+      _n_threads(n_threads), _tokens(tokens) {}
+
+void DecodeAudioTokenWorker::Execute() {
+  const int n_codes = _tokens.size();
+  llama_batch batch = llama_batch_init(n_codes, 0, 1);
+  for (size_t i = 0; i < _tokens.size(); ++i) {
+    common_batch_add(batch, _tokens[i], i, {0}, true);
+  }
+  if (batch.n_tokens != n_codes) {
+    SetError("batch.n_tokens != n_codes");
+    return;
+  }
+  if (llama_encode(_ctx, batch) != 0) {
+    SetError("llama_encode() failed");
+    return;
+  }
+  llama_synchronize(_ctx);
+  const int n_embd = llama_model_n_embd(_model);
+  const float *embd = llama_get_embeddings(_ctx);
+  _result = embd_to_audio(embd, n_codes, n_embd, _n_threads);
+}
+
+void DecodeAudioTokenWorker::OnOK() {
+  auto result =
+      Napi::Float32Array::New(Napi::AsyncWorker::Env(), _result.size());
+  memcpy(result.Data(), _result.data(), _result.size() * sizeof(float));
+  Napi::Promise::Deferred::Resolve(result);
+}
+
+void DecodeAudioTokenWorker::OnError(const Napi::Error &err) {
+  Napi::Promise::Deferred::Reject(err.Value());
+}

package/src/DecodeAudioTokenWorker.h

@@ -0,0 +1,22 @@
+#include "common.hpp"
+#include <vector>
+
+class DecodeAudioTokenWorker : public Napi::AsyncWorker,
+                               public Napi::Promise::Deferred {
+public:
+  DecodeAudioTokenWorker(const Napi::CallbackInfo &info, llama_model *model,
+                         llama_context *ctx, int n_threads,
+                         const std::vector<llama_token> &tokens);
+
+protected:
+  void Execute();
+  void OnOK();
+  void OnError(const Napi::Error &err);
+
+private:
+  llama_model *_model;
+  llama_context *_ctx;
+  int _n_threads;
+  std::vector<llama_token> _tokens;
+  std::vector<float> _result;
+};

package/src/EmbeddingWorker.cpp

@@ -2,8 +2,10 @@
 #include "LlamaContext.h"
 
 EmbeddingWorker::EmbeddingWorker(const Napi::CallbackInfo &info,
-                                 LlamaSessionPtr &sess, std::string text, common_params &params)
-    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text), _params(params) {}
+                                 LlamaSessionPtr &sess, std::string text,
+                                 common_params &params)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text),
+      _params(params) {}
 
 void EmbeddingWorker::Execute() {
   llama_kv_self_clear(_sess->context());
@@ -17,8 +19,7 @@ void EmbeddingWorker::Execute() {
   do {
     auto ctx = _sess->context();
     int ret =
-        llama_decode(ctx,
-                     llama_batch_get_one(tokens.data(), tokens.size()));
+        llama_decode(ctx, llama_batch_get_one(tokens.data(), tokens.size()));
     if (ret < 0) {
       SetError("Failed to inference, code: " + std::to_string(ret));
       break;
@@ -37,7 +38,8 @@ void EmbeddingWorker::Execute() {
     }
     _result.embedding.resize(n_embd);
     std::vector<float> embedding(embd, embd + n_embd), out(embd, embd + n_embd);
-        common_embd_normalize(embedding.data(), out.data(), n_embd, _params.embd_normalize);
+    common_embd_normalize(embedding.data(), out.data(), n_embd,
+                          _params.embd_normalize);
     memcpy(_result.embedding.data(), out.data(), n_embd * sizeof(float));
   } while (false);
 }