@fugood/llama.node 1.0.6 → 1.1.1

package/CMakeLists.txt

@@ -73,9 +73,9 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
       set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O1 /Ob1 /bigobj")
     endif()
   else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -funroll-loops -flto")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -funroll-loops -flto")
-    set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -flto")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -funroll-loops -flto=auto")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -funroll-loops -flto=auto")
+    set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -flto=auto")
   endif()
 endif()
 

package/lib/binding.ts

@@ -1,6 +1,3 @@
-import * as path from 'path'
-
-
 export type MessagePart = {
   type: string,
   text?: string,
@@ -53,6 +50,11 @@ export type LlamaModelOptions = {
    * Enable context shifting to handle prompts larger than context size
    */
   ctx_shift?: boolean
+  /**
+   * Use a unified buffer across the input sequences when computing the attention.
+   * Try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix.
+   */
+  kv_unified?: boolean
   use_mlock?: boolean
   use_mmap?: boolean
   vocab_only?: boolean
@@ -65,9 +67,9 @@ export type CompletionResponseFormat = {
   type: 'text' | 'json_object' | 'json_schema'
   json_schema?: {
     strict?: boolean
-    schema: object
+    schema: Record<string, any>
   }
-  schema?: object // for json_object type
+  schema?: Record<string, any> // for json_object type
 }
 
 export type LlamaCompletionOptions = {
@@ -76,7 +78,7 @@ export type LlamaCompletionOptions = {
   reasoning_format?: string
   chat_template?: string
   response_format?: CompletionResponseFormat
-  tools?: object
+  tools?: Tool[]
   parallel_tool_calls?: boolean
   tool_choice?: string
   enable_thinking?: boolean
@@ -107,7 +109,7 @@ export type LlamaCompletionOptions = {
   stop?: string[]
   grammar?: string
   grammar_lazy?: boolean
-  grammar_triggers?: { type: number; word: string; at_start: boolean }[]
+  grammar_triggers?: { type: number; value: string; token?: number }[]
   preserved_tokens?: string[]
   /**
    * Path(s) to media file(s) to process before generating text.
@@ -120,13 +122,14 @@ export type LlamaCompletionOptions = {
    * Guide tokens to use for audio completion.
    * Help prevent hallucinations by forcing the TTS to use the correct words.
    */
-  guide_tokens?: Int32Array
+  guide_tokens?: number[] | Int32Array
 }
 
 export type LlamaCompletionResult = {
   text: string
   reasoning_content?: string
   content?: string
+  chat_format: number
   tokens_predicted: number
   tokens_evaluated: number
   truncated: boolean
@@ -169,21 +172,101 @@ export type RerankResult = {
   index: number
 }
 
+export type ModelInfo = {
+  desc: string
+  nEmbd: number
+  nParams: number
+  size: number
+  chatTemplates: {
+    llamaChat: boolean
+    minja: {
+      default: boolean
+      defaultCaps: {
+        tools: boolean
+        toolCalls: boolean
+        toolResponses: boolean
+        systemRole: boolean
+        parallelToolCalls: boolean
+        toolCallId: boolean
+      }
+      toolUse: boolean
+      toolUseCaps?: {
+        tools: boolean
+        toolCalls: boolean
+        toolResponses: boolean
+        systemRole: boolean
+        parallelToolCalls: boolean
+        toolCallId: boolean
+      }
+    }
+  }
+  metadata: Record<string, string>
+  isChatTemplateSupported: boolean
+}
+
+export type GGUFModelInfo = {
+  version?: number
+  alignment?: number
+  data_offset?: number
+  [key: string]: string | number | undefined
+}
+
+export type FormattedChatResult = {
+  type: 'jinja' | 'llama-chat'
+  prompt: string
+  has_media: boolean
+  media_paths?: Array<string>
+}
+
+export type JinjaFormattedChatResult = {
+  prompt: string
+  chat_format: number
+  grammar: string
+  grammea_lazy: boolean
+  grammar_triggers: Array<{
+    type: number
+    value: string
+    token: number
+  }>
+  thinking_forced_open: boolean
+  preserved_tokens: string[]
+  additional_stops: string[]
+}
+
+export type Tool = {
+  type: 'function'
+  function: {
+    name: string
+    description: string
+    parameters: Record<string, any>
+  }
+}
+
+export type ToolCall = {
+  type: 'function'
+  function: {
+    name: string
+    arguments: string
+  }
+  id?: string
+}
+
 export interface LlamaContext {
   new (options: LlamaModelOptions): LlamaContext
   getSystemInfo(): string
-  getModelInfo(): object
+  getModelInfo(): ModelInfo
   getFormattedChat(
     messages: ChatMessage[],
     chat_template?: string,
     params?: {
       jinja?: boolean
       response_format?: CompletionResponseFormat
-      tools?: object
-      parallel_tool_calls?: object
+      tools?: Tool[]
+      parallel_tool_calls?: boolean
       tool_choice?: string
+      enable_thinking?: boolean
     },
-  ): object | string
+  ): JinjaFormattedChatResult | string
   completion(
     options: LlamaCompletionOptions,
     callback?: (token: LlamaCompletionToken) => void,
@@ -197,51 +280,50 @@ export interface LlamaContext {
   loadSession(path: string): Promise<void>
   release(): Promise<void>
   applyLoraAdapters(adapters: { path: string; scaled: number }[]): void
-  removeLoraAdapters(adapters: { path: string }[]): void
+  removeLoraAdapters(): void
   getLoadedLoraAdapters(): { path: string; scaled: number }[]
   /**
    * Initialize multimodal support with a mmproj file
-   * @param mmproj_path Path to the multimodal projector file
-   * @returns Promise resolving to true if initialization was successful
+   * @param options Object containing path and optional use_gpu flag
+   * @returns boolean indicating if initialization was successful
    */
-  initMultimodal(options: { path: string; use_gpu?: boolean }): Promise<boolean>
+  initMultimodal(options: { path: string; use_gpu?: boolean }): boolean
 
   /**
    * Check if multimodal support is enabled
-   * @returns Promise resolving to true if multimodal is enabled
+   * @returns boolean indicating if multimodal is enabled
    */
-  isMultimodalEnabled(): Promise<boolean>
+  isMultimodalEnabled(): boolean
 
   /**
    * Get multimodal support capabilities
-   * @returns Promise resolving to an object with vision and audio support
+   * @returns Object with vision and audio support
    */
-  getMultimodalSupport(): Promise<{
+  getMultimodalSupport(): {
     vision: boolean
     audio: boolean
-  }>
+  }
 
   /**
    * Release multimodal support
    */
-  releaseMultimodal(): Promise<void>
+  releaseMultimodal(): void
 
   /**
    * Load a vocoder model
-   * @param path Path to the vocoder model
-   * @returns Promise resolving to true if loading was successful
+   * @param options Object containing path and optional n_batch
+   * @returns boolean indicating if loading was successful
    */
-  initVocoder(options: { path: string, n_batch?: number }): Promise<boolean>
+  initVocoder(options: { path: string, n_batch?: number }): boolean
 
   /**
    * Unload the vocoder model
-   * @returns Promise resolving to true if unloading was successful
    */
-  releaseVocoder(): Promise<void>
+  releaseVocoder(): void
 
   /**
    * Check if the vocoder model is enabled
-   * @returns Promise resolving to true if the vocoder model is enabled
+   * @returns boolean indicating if the vocoder model is enabled
    */
   isVocoderEnabled(): boolean
 
@@ -251,7 +333,10 @@ export interface LlamaContext {
    * @param text Text to complete
    * @returns Formatted audio completion
    */
-  getFormattedAudioCompletion(speaker: string|null, text: string): string
+  getFormattedAudioCompletion(speaker: string|null, text: string): {
+    prompt: string
+    grammar?: string
+  }
 
   /**
    * Get guide tokens for audio completion
@@ -263,12 +348,12 @@ export interface LlamaContext {
   /**
    * Decode audio tokens to audio data
    * @param tokens Tokens to decode
-   * @returns Decoded audio tokens
+   * @returns Promise resolving to decoded audio tokens
    */
-  decodeAudioTokens(tokens: Int32Array): Promise<Float32Array>
+  decodeAudioTokens(tokens: number[]|Int32Array): Promise<Float32Array>
 
   // static
-  loadModelInfo(path: string, skip: string[]): Promise<Object>
+  loadModelInfo(path: string, skip: string[]): Promise<GGUFModelInfo>
   toggleNativeLog(
     enable: boolean,
     callback: (level: string, text: string) => void,

package/lib/index.js

@@ -140,7 +140,7 @@ class LlamaContextWrapper {
         const jsonSchema = getJsonSchema(params === null || params === void 0 ? void 0 : params.response_format);
         const result = this.ctx.getFormattedChat(chat, tmpl, {
             jinja: useJinja,
-            json_schema: jsonSchema,
+            response_format: params === null || params === void 0 ? void 0 : params.response_format,
             tools: params === null || params === void 0 ? void 0 : params.tools,
             parallel_tool_calls: params === null || params === void 0 ? void 0 : params.parallel_tool_calls,
             tool_choice: params === null || params === void 0 ? void 0 : params.tool_choice,
@@ -155,10 +155,8 @@ class LlamaContextWrapper {
             };
         }
         const jinjaResult = result;
-        jinjaResult.type = 'jinja';
-        jinjaResult.has_media = has_media;
-        jinjaResult.media_paths = media_paths;
-        return jinjaResult;
+        return Object.assign({ type: 'jinja', has_media,
+            media_paths }, jinjaResult);
     }
     completion(options, callback) {
         const { messages, media_paths = options.media_paths } = this._formatMediaChat(options.messages);
@@ -196,8 +194,8 @@ class LlamaContextWrapper {
     applyLoraAdapters(adapters) {
         return this.ctx.applyLoraAdapters(adapters);
     }
-    removeLoraAdapters(adapters) {
-        return this.ctx.removeLoraAdapters(adapters);
+    removeLoraAdapters() {
+        this.ctx.removeLoraAdapters();
     }
     getLoadedLoraAdapters() {
         return this.ctx.getLoadedLoraAdapters();
@@ -209,7 +207,7 @@ class LlamaContextWrapper {
         return this.ctx.isMultimodalEnabled();
     }
     releaseMultimodal() {
-        return this.ctx.releaseMultimodal();
+        this.ctx.releaseMultimodal();
     }
     getMultimodalSupport() {
         return this.ctx.getMultimodalSupport();
@@ -218,7 +216,7 @@ class LlamaContextWrapper {
         return this.ctx.initVocoder(options);
     }
     releaseVocoder() {
-        return this.ctx.releaseVocoder();
+        this.ctx.releaseVocoder();
     }
     isVocoderEnabled() {
         return this.ctx.isVocoderEnabled();

package/lib/index.ts

@@ -12,6 +12,10 @@ import type {
   RerankParams,
   RerankResult,
   CompletionResponseFormat,
+  ModelInfo,
+  JinjaFormattedChatResult,
+  Tool,
+  GGUFModelInfo,
 } from './binding'
 
 export * from './binding'
@@ -72,9 +76,9 @@ export type FormattedChatResult = {
 }
 
 class LlamaContextWrapper {
-  ctx: any
+  ctx: LlamaContext
 
-  constructor(nativeCtx: any) {
+  constructor(nativeCtx: LlamaContext) {
     this.ctx = nativeCtx
   }
 
@@ -82,7 +86,7 @@ class LlamaContextWrapper {
     return this.ctx.getSystemInfo()
   }
 
-  getModelInfo(): object {
+  getModelInfo(): ModelInfo {
     return this.ctx.getModelInfo()
   }
 
@@ -158,8 +162,8 @@ class LlamaContextWrapper {
     params?: {
       jinja?: boolean
       response_format?: CompletionResponseFormat
-      tools?: object
-      parallel_tool_calls?: object
+      tools?: Tool[]
+      parallel_tool_calls?: boolean
       tool_choice?: string,
       enable_thinking?: boolean,
     },
@@ -175,9 +179,9 @@ class LlamaContextWrapper {
     if (template) tmpl = template // Force replace if provided
     const jsonSchema = getJsonSchema(params?.response_format)
 
-    const result = this.ctx.getFormattedChat(chat, tmpl, {
+    const result = this.ctx.getFormattedChat(chat!, tmpl, {
       jinja: useJinja,
-      json_schema: jsonSchema,
+      response_format: params?.response_format,
       tools: params?.tools,
       parallel_tool_calls: params?.parallel_tool_calls,
       tool_choice: params?.tool_choice,
@@ -192,11 +196,13 @@ class LlamaContextWrapper {
         media_paths,
       }
     }
-    const jinjaResult = result
-    jinjaResult.type = 'jinja'
-    jinjaResult.has_media = has_media
-    jinjaResult.media_paths = media_paths
-    return jinjaResult
+    const jinjaResult = result as JinjaFormattedChatResult
+    return {
+      type: 'jinja',
+      has_media,
+      media_paths,
+      ...jinjaResult,
+    }
   }
 
   completion(
@@ -256,8 +262,8 @@ class LlamaContextWrapper {
     return this.ctx.applyLoraAdapters(adapters)
   }
 
-  removeLoraAdapters(adapters: { path: string }[]): void {
-    return this.ctx.removeLoraAdapters(adapters)
+  removeLoraAdapters(): void {
+    this.ctx.removeLoraAdapters()
   }
 
   getLoadedLoraAdapters(): { path: string; scaled: number }[] {
@@ -267,38 +273,41 @@ class LlamaContextWrapper {
   initMultimodal(options: {
     path: string
     use_gpu?: boolean
-  }): Promise<boolean> {
+  }): boolean {
     return this.ctx.initMultimodal(options)
   }
 
-  isMultimodalEnabled(): Promise<boolean> {
+  isMultimodalEnabled(): boolean {
     return this.ctx.isMultimodalEnabled()
   }
 
-  releaseMultimodal(): Promise<void> {
-    return this.ctx.releaseMultimodal()
+  releaseMultimodal(): void {
+    this.ctx.releaseMultimodal()
   }
 
-  getMultimodalSupport(): Promise<{
+  getMultimodalSupport(): {
     vision: boolean
     audio: boolean
-  }> {
+  } {
     return this.ctx.getMultimodalSupport()
   }
 
-  initVocoder(options: { path: string, n_batch?: number }): Promise<boolean> {
+  initVocoder(options: { path: string, n_batch?: number }): boolean {
     return this.ctx.initVocoder(options)
   }
 
-  releaseVocoder(): Promise<void> {
-    return this.ctx.releaseVocoder()
+  releaseVocoder(): void {
+    this.ctx.releaseVocoder()
   }
 
   isVocoderEnabled(): boolean {
     return this.ctx.isVocoderEnabled()
   }
 
-  getFormattedAudioCompletion(speaker: string|null, text: string): string {
+  getFormattedAudioCompletion(speaker: string|null, text: string): {
+    prompt: string
+    grammar?: string
+  } {
     return this.ctx.getFormattedAudioCompletion(speaker, text)
   }
 
@@ -332,7 +341,7 @@ const modelInfoSkip = [
   'tokenizer.ggml.scores',
 ]
 
-export const loadLlamaModelInfo = async (path: string): Promise<Object> => {
+export const loadLlamaModelInfo = async (path: string): Promise<GGUFModelInfo> => {
   const variant = 'default'
   mods[variant] ??= await loadModule(variant)
   refreshNativeLogSetup()

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.0.6",
+  "version": "1.1.1",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -9,6 +9,7 @@
     "postinstall": "node scripts/check.js",
     "pretest": "node scripts/download-test-models.js",
     "test": "jest",
+    "typecheck": "tsc --noEmit",
     "build": "npx cmake-js build",
     "build-js": "tsc",
     "prepack": "npm run build-js",
@@ -70,19 +71,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.0.6",
-    "@fugood/node-llama-linux-x64-vulkan": "1.0.6",
-    "@fugood/node-llama-linux-x64-cuda": "1.0.6",
-    "@fugood/node-llama-linux-arm64": "1.0.6",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.0.6",
-    "@fugood/node-llama-linux-arm64-cuda": "1.0.6",
-    "@fugood/node-llama-win32-x64": "1.0.6",
-    "@fugood/node-llama-win32-x64-vulkan": "1.0.6",
-    "@fugood/node-llama-win32-x64-cuda": "1.0.6",
-    "@fugood/node-llama-win32-arm64": "1.0.6",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.0.6",
-    "@fugood/node-llama-darwin-x64": "1.0.6",
-    "@fugood/node-llama-darwin-arm64": "1.0.6"
+    "@fugood/node-llama-linux-x64": "1.1.1",
+    "@fugood/node-llama-linux-x64-vulkan": "1.1.1",
+    "@fugood/node-llama-linux-x64-cuda": "1.1.1",
+    "@fugood/node-llama-linux-arm64": "1.1.1",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.1.1",
+    "@fugood/node-llama-linux-arm64-cuda": "1.1.1",
+    "@fugood/node-llama-win32-x64": "1.1.1",
+    "@fugood/node-llama-win32-x64-vulkan": "1.1.1",
+    "@fugood/node-llama-win32-x64-cuda": "1.1.1",
+    "@fugood/node-llama-win32-arm64": "1.1.1",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.1.1",
+    "@fugood/node-llama-darwin-x64": "1.1.1",
+    "@fugood/node-llama-darwin-arm64": "1.1.1"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",
@@ -91,10 +92,12 @@
     "@commitlint/config-conventional": "^19.2.2",
     "@types/jest": "^29.5.12",
     "@types/node": "^22.0.0",
+    "@types/node-wav": "^0.0.4",
     "cmake-js": "^7.3.0",
     "husky": "^9.0.11",
     "jest": "^29.7.0",
     "node-addon-api": "^8.0.0",
+    "node-wav": "^0.0.2",
     "release-it": "^17.7.0",
     "rimraf": "^6.0.1",
     "typescript": "^5.4.5",

package/src/LlamaCompletionWorker.cpp

@@ -110,7 +110,7 @@ void LlamaCompletionWorker::Execute() {
   } else {
     // Text-only path
     std::vector<llama_token> prompt_tokens =
-        ::common_tokenize(ctx, _params.prompt, add_bos);
+        ::common_tokenize(ctx, _params.prompt, add_bos, true);
     n_input = prompt_tokens.size();
 
     if (_sess->tokens_ptr()->size() > 0) {
@@ -157,10 +157,26 @@ void LlamaCompletionWorker::Execute() {
     // For multimodal input, n_past might already be set
     // Only decode text tokens if we have any input left
     if (n_input > 0) {
-      int ret = llama_decode(ctx, llama_batch_get_one(embd->data() + n_cur, n_input));
-      if (ret < 0) {
-        SetError("Failed to decode token, code: " + std::to_string(ret));
-        break;
+      // Decode tokens in batches using n_batch as chunk size
+      int n_past_batch = n_cur;
+      int n_remaining = n_input;
+      
+      while (n_remaining > 0) {
+        int n_eval = n_remaining;
+        if (n_eval > _params.n_batch) {
+          n_eval = _params.n_batch;
+        }
+        
+        int ret = llama_decode(ctx, llama_batch_get_one(embd->data() + n_past_batch, n_eval));
+        if (ret < 0) {
+          SetError("Failed to decode token batch, code: " + std::to_string(ret) + 
+                   ", n_eval: " + std::to_string(n_eval) + 
+                   ", n_past_batch: " + std::to_string(n_past_batch));
+          break;
+        }
+        
+        n_past_batch += n_eval;
+        n_remaining -= n_eval;
       }
     }
 
@@ -177,7 +193,7 @@ void LlamaCompletionWorker::Execute() {
     
     // Collect audio tokens for TTS if vocoder is enabled
     if (_has_vocoder) {
-      if ((_tts_type == OUTETTS_V0_2 || _tts_type == OUTETTS_V0_3) && 
+      if ((_tts_type == OUTETTS_V0_1 || _tts_type == OUTETTS_V0_2 || _tts_type == OUTETTS_V0_3) && 
           (new_token_id >= 151672 && new_token_id <= 155772)) {
         _result.audio_tokens.push_back(new_token_id);
       }
@@ -255,6 +271,8 @@ void LlamaCompletionWorker::OnOK() {
     try {
       common_chat_syntax chat_syntax;
       chat_syntax.format = static_cast<common_chat_format>(_chat_format);
+      result.Set("chat_format", Napi::Number::New(env, _chat_format));
+
       chat_syntax.thinking_forced_open = _thinking_forced_open;
 
       if (_reasoning_format == "deepseek") {

package/src/LlamaContext.cpp

@@ -247,6 +247,7 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   params.cache_type_v = kv_cache_type_from_str(
       get_option<std::string>(options, "cache_type_v", "f16").c_str());
   params.ctx_shift = get_option<bool>(options, "ctx_shift", true);
+  params.kv_unified = get_option<bool>(options, "kv_unified", false);
 
   params.use_mlock = get_option<bool>(options, "use_mlock", false);
   params.use_mmap = get_option<bool>(options, "use_mmap", true);
@@ -904,9 +905,27 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   // guide_tokens
   std::vector<llama_token> guide_tokens;
   if (options.Has("guide_tokens")) {
-    auto guide_tokens_array = options.Get("guide_tokens").As<Napi::Array>();
-    for (size_t i = 0; i < guide_tokens_array.Length(); i++) {
-      guide_tokens.push_back(guide_tokens_array.Get(i).ToNumber().Int32Value());
+    auto guide_tokens_value = options.Get("guide_tokens");
+    if (guide_tokens_value.IsArray()) {
+      auto guide_tokens_array = guide_tokens_value.As<Napi::Array>();
+      for (size_t i = 0; i < guide_tokens_array.Length(); i++) {
+        guide_tokens.push_back(guide_tokens_array.Get(i).ToNumber().Int32Value());
+      }
+    } else if (guide_tokens_value.IsTypedArray()) {
+      auto guide_tokens_typed_array = guide_tokens_value.As<Napi::TypedArray>();
+      if (guide_tokens_typed_array.TypedArrayType() == napi_int32_array) {
+        auto guide_tokens_int32_array = guide_tokens_value.As<Napi::Int32Array>();
+        size_t length = guide_tokens_int32_array.ElementLength();
+        const int32_t* data = guide_tokens_int32_array.Data();
+        guide_tokens.resize(length);
+        memcpy(guide_tokens.data(), data, length * sizeof(int32_t));
+      } else {
+        Napi::TypeError::New(env, "guide_tokens must be Array<number> or Int32Array").ThrowAsJavaScriptException();
+        return env.Undefined();
+      }
+    } else {
+      Napi::TypeError::New(env, "guide_tokens must be Array<number> or Int32Array").ThrowAsJavaScriptException();
+      return env.Undefined();
     }
   }
 
@@ -1345,7 +1364,7 @@ Napi::Value LlamaContext::IsVocoderEnabled(const Napi::CallbackInfo &info) {
   return Napi::Boolean::New(env, _has_vocoder);
 }
 
-// getFormattedAudioCompletion(speaker: string|null, text: string): string
+// getFormattedAudioCompletion(speaker: string|null, text: string): object
 Napi::Value
 LlamaContext::GetFormattedAudioCompletion(const Napi::CallbackInfo &info) {
   Napi::Env env = info.Env();
@@ -1372,9 +1391,16 @@ LlamaContext::GetFormattedAudioCompletion(const Napi::CallbackInfo &info) {
     audio_text = audio_text_from_speaker(speaker, type);
     audio_data = audio_data_from_speaker(speaker, type);
   }
-  return Napi::String::New(env, "<|im_start|>\n" + audio_text +
-                                    process_text(text, type) +
-                                    "<|text_end|>\n" + audio_data + "\n");
+  std::string prompt = "<|im_start|>\n" + audio_text +
+                       process_text(text, type) +
+                       "<|text_end|>\n" + audio_data + "\n";
+  Napi::Object result = Napi::Object::New(env);
+  result.Set("prompt", prompt);
+  const char *grammar = get_tts_grammar(type);
+  if (grammar != nullptr) {
+    result.Set("grammar", grammar);
+  }
+  return result;
 }
 
 // getAudioCompletionGuideTokens(text: string): Int32Array
@@ -1415,6 +1441,10 @@ LlamaContext::GetAudioCompletionGuideTokens(const Napi::CallbackInfo &info) {
   if (tmp.size() > 0) {
     result.push_back(tmp[0]);
   }
+
+  // Add Audio End, forcing stop generation
+  result.push_back(common_tokenize(vocab, "<|audio_end|>", false, true)[0]);
+
   auto tokens = Napi::Int32Array::New(env, result.size());
   memcpy(tokens.Data(), result.data(), result.size() * sizeof(int32_t));
   return tokens;
@@ -1449,7 +1479,7 @@ Napi::Value LlamaContext::DecodeAudioTokens(const Napi::CallbackInfo &info) {
         .ThrowAsJavaScriptException();
     return env.Undefined();
   }
-  if (type == OUTETTS_V0_3 || type == OUTETTS_V0_2) {
+  if (type == OUTETTS_V0_1 || type == OUTETTS_V0_2 || type == OUTETTS_V0_3) {
     tokens.erase(
         std::remove_if(tokens.begin(), tokens.end(),
                        [](llama_token t) { return t < 151672 || t > 155772; }),