@fugood/llama.node 1.1.7 → 1.1.9

package/lib/binding.ts

@@ -59,6 +59,10 @@ export type LlamaModelOptions = {
    * Use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
    */
   swa_full?: boolean
+  /**
+   * Number of layers to keep MoE weights on CPU
+   */
+  n_cpu_moe?: number
   use_mlock?: boolean
   use_mmap?: boolean
   vocab_only?: boolean

package/lib/index.js

@@ -148,7 +148,12 @@ class LlamaContextWrapper {
             enable_thinking: (_a = params === null || params === void 0 ? void 0 : params.enable_thinking) !== null && _a !== void 0 ? _a : true,
             add_generation_prompt: params === null || params === void 0 ? void 0 : params.add_generation_prompt,
             now: params === null || params === void 0 ? void 0 : params.now,
-            chat_template_kwargs: params === null || params === void 0 ? void 0 : params.chat_template_kwargs,
+            chat_template_kwargs: (params === null || params === void 0 ? void 0 : params.chat_template_kwargs)
+                ? Object.entries(params.chat_template_kwargs).reduce((acc, [key, value]) => {
+                    acc[key] = JSON.stringify(value); // Each value is a stringified JSON object
+                    return acc;
+                }, {})
+                : undefined,
         });
         if (!useJinja) {
             return {
@@ -179,7 +184,9 @@ class LlamaContextWrapper {
         return this.ctx.embedding(text);
     }
     rerank(query, documents, params) {
-        return this.ctx.rerank(query, documents, params).then((results) => {
+        return this.ctx
+            .rerank(query, documents, params)
+            .then((results) => {
             // Sort by score descending and add document text for convenience
             return results
                 .map((result) => (Object.assign(Object.assign({}, result), { document: documents[result.index] })))

package/lib/index.ts

@@ -165,11 +165,11 @@ class LlamaContextWrapper {
       response_format?: CompletionResponseFormat
       tools?: Tool[]
       parallel_tool_calls?: boolean
-      tool_choice?: string,
-      enable_thinking?: boolean,
-      add_generation_prompt?: boolean,
-      now?: string | number,
-      chat_template_kwargs?: Record<string, string>,
+      tool_choice?: string
+      enable_thinking?: boolean
+      add_generation_prompt?: boolean
+      now?: string | number
+      chat_template_kwargs?: Record<string, string>
     },
   ): FormattedChatResult {
     const {
@@ -192,7 +192,15 @@ class LlamaContextWrapper {
       enable_thinking: params?.enable_thinking ?? true,
       add_generation_prompt: params?.add_generation_prompt,
       now: params?.now,
-      chat_template_kwargs: params?.chat_template_kwargs,
+      chat_template_kwargs: params?.chat_template_kwargs
+        ? Object.entries(params.chat_template_kwargs).reduce(
+            (acc, [key, value]) => {
+              acc[key] = JSON.stringify(value) // Each value is a stringified JSON object
+              return acc
+            },
+            {} as Record<string, any>,
+          )
+        : undefined,
     })
 
     if (!useJinja) {
@@ -218,18 +226,24 @@ class LlamaContextWrapper {
   ): Promise<LlamaCompletionResult> {
     const { messages, media_paths = options.media_paths } =
       this._formatMediaChat(options.messages)
-    return this.ctx.completion({
-      ...options,
-      messages,
-      media_paths: options.media_paths || media_paths,
-    }, callback || (() => {}))
+    return this.ctx.completion(
+      {
+        ...options,
+        messages,
+        media_paths: options.media_paths || media_paths,
+      },
+      callback || (() => {}),
+    )
   }
 
   stopCompletion(): void {
     return this.ctx.stopCompletion()
   }
 
-  tokenize(text: string, { media_paths }: { media_paths?: string[] } = {}): Promise<TokenizeResult> {
+  tokenize(
+    text: string,
+    { media_paths }: { media_paths?: string[] } = {},
+  ): Promise<TokenizeResult> {
     return this.ctx.tokenize(text, media_paths)
   }
 
@@ -241,16 +255,27 @@ class LlamaContextWrapper {
     return this.ctx.embedding(text)
   }
 
-  rerank(query: string, documents: string[], params?: RerankParams): Promise<Array<RerankResult & { document: string }>> {
-    return this.ctx.rerank(query, documents, params).then((results: RerankResult[]) => {
-      // Sort by score descending and add document text for convenience
-      return results
-        .map((result: RerankResult) => ({
-          ...result,
-          document: documents[result.index],
-        }))
-        .sort((a: RerankResult & { document: string }, b: RerankResult & { document: string }) => b.score - a.score)
-    })
+  rerank(
+    query: string,
+    documents: string[],
+    params?: RerankParams,
+  ): Promise<Array<RerankResult & { document: string }>> {
+    return this.ctx
+      .rerank(query, documents, params)
+      .then((results: RerankResult[]) => {
+        // Sort by score descending and add document text for convenience
+        return results
+          .map((result: RerankResult) => ({
+            ...result,
+            document: documents[result.index],
+          }))
+          .sort(
+            (
+              a: RerankResult & { document: string },
+              b: RerankResult & { document: string },
+            ) => b.score - a.score,
+          )
+      })
   }
 
   saveSession(path: string): Promise<void> {
@@ -277,10 +302,7 @@ class LlamaContextWrapper {
     return this.ctx.getLoadedLoraAdapters()
   }
 
-  initMultimodal(options: {
-    path: string
-    use_gpu?: boolean
-  }): boolean {
+  initMultimodal(options: { path: string; use_gpu?: boolean }): boolean {
     return this.ctx.initMultimodal(options)
   }
 
@@ -299,7 +321,7 @@ class LlamaContextWrapper {
     return this.ctx.getMultimodalSupport()
   }
 
-  initVocoder(options: { path: string, n_batch?: number }): boolean {
+  initVocoder(options: { path: string; n_batch?: number }): boolean {
     return this.ctx.initVocoder(options)
   }
 
@@ -311,7 +333,10 @@ class LlamaContextWrapper {
     return this.ctx.isVocoderEnabled()
   }
 
-  getFormattedAudioCompletion(speaker: string|null, text: string): {
+  getFormattedAudioCompletion(
+    speaker: string | null,
+    text: string,
+  ): {
     prompt: string
     grammar?: string
   } {
@@ -322,7 +347,7 @@ class LlamaContextWrapper {
     return this.ctx.getAudioCompletionGuideTokens(text)
   }
 
-  decodeAudioTokens(tokens: number[]|Int32Array): Promise<Float32Array> {
+  decodeAudioTokens(tokens: number[] | Int32Array): Promise<Float32Array> {
     return this.ctx.decodeAudioTokens(tokens)
   }
 }
@@ -348,7 +373,9 @@ const modelInfoSkip = [
   'tokenizer.ggml.scores',
 ]
 
-export const loadLlamaModelInfo = async (path: string): Promise<GGUFModelInfo> => {
+export const loadLlamaModelInfo = async (
+  path: string,
+): Promise<GGUFModelInfo> => {
   const variant = 'default'
   mods[variant] ??= await loadModule(variant)
   refreshNativeLogSetup()

package/lib/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BUILD_COMMIT = exports.BUILD_NUMBER = void 0;
-exports.BUILD_NUMBER = '6096';
-exports.BUILD_COMMIT = 'fd1234cb';
+exports.BUILD_NUMBER = '6250';
+exports.BUILD_COMMIT = 'e92734d51';

package/lib/version.ts

@@ -1,2 +1,2 @@
-export const BUILD_NUMBER = '6096';
-export const BUILD_COMMIT = 'fd1234cb';
+export const BUILD_NUMBER = '6250';
+export const BUILD_COMMIT = 'e92734d51';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.1.7",
+  "version": "1.1.9",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -71,19 +71,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.1.7",
-    "@fugood/node-llama-linux-x64-vulkan": "1.1.7",
-    "@fugood/node-llama-linux-x64-cuda": "1.1.7",
-    "@fugood/node-llama-linux-arm64": "1.1.7",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.1.7",
-    "@fugood/node-llama-linux-arm64-cuda": "1.1.7",
-    "@fugood/node-llama-win32-x64": "1.1.7",
-    "@fugood/node-llama-win32-x64-vulkan": "1.1.7",
-    "@fugood/node-llama-win32-x64-cuda": "1.1.7",
-    "@fugood/node-llama-win32-arm64": "1.1.7",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.1.7",
-    "@fugood/node-llama-darwin-x64": "1.1.7",
-    "@fugood/node-llama-darwin-arm64": "1.1.7"
+    "@fugood/node-llama-linux-x64": "1.1.9",
+    "@fugood/node-llama-linux-x64-vulkan": "1.1.9",
+    "@fugood/node-llama-linux-x64-cuda": "1.1.9",
+    "@fugood/node-llama-linux-arm64": "1.1.9",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.1.9",
+    "@fugood/node-llama-linux-arm64-cuda": "1.1.9",
+    "@fugood/node-llama-win32-x64": "1.1.9",
+    "@fugood/node-llama-win32-x64-vulkan": "1.1.9",
+    "@fugood/node-llama-win32-x64-cuda": "1.1.9",
+    "@fugood/node-llama-win32-arm64": "1.1.9",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.1.9",
+    "@fugood/node-llama-darwin-x64": "1.1.9",
+    "@fugood/node-llama-darwin-arm64": "1.1.9"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/src/LlamaContext.cpp

@@ -15,6 +15,7 @@
 #include "llama-impl.h"
 
 #include <atomic>
+#include <list>
 #include <mutex>
 #include <queue>
 
@@ -258,6 +259,16 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   params.numa =
       static_cast<ggml_numa_strategy>(get_option<uint32_t>(options, "numa", 0));
 
+  int n_cpu_moe = get_option<int32_t>(options, "n_cpu_moe", 0);
+  if (n_cpu_moe > 0) {
+    static std::list<std::string> buft_overrides;
+    for (int i = 0; i < n_cpu_moe; ++i) {
+      buft_overrides.push_back(string_format("blk\\.%d\\.ffn_(up|down|gate)_exps", i));
+      params.tensor_buft_overrides.push_back({buft_overrides.back().c_str(), ggml_backend_cpu_buffer_type()});
+    }
+    params.tensor_buft_overrides.push_back({nullptr, nullptr});
+  }
+
   llama_backend_init();
   llama_numa_init(params.numa);
 
@@ -636,6 +647,15 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
           _sess, _templates, messages, chat_template, json_schema_str, tools_str,
           parallel_tool_calls, tool_choice, enable_thinking,
           add_generation_prompt, now_str, chat_template_kwargs);
+    } catch (const nlohmann::json_abi_v3_12_0::detail::parse_error& e) {
+      Napi::Error::New(env, e.what()).ThrowAsJavaScriptException();
+      return env.Undefined();
+    } catch (const std::invalid_argument& e) {
+      Napi::Error::New(env, e.what()).ThrowAsJavaScriptException();
+      return env.Undefined();
+    } catch (const std::runtime_error& e) {
+      Napi::Error::New(env, e.what()).ThrowAsJavaScriptException();
+      return env.Undefined();
     } catch (const std::exception &e) {
       Napi::Error::New(env, e.what()).ThrowAsJavaScriptException();
       return env.Undefined();

package/src/common.hpp

@@ -461,7 +461,14 @@ processMediaPrompt(llama_context *ctx, mtmd_context *mtmd_ctx,
   }
 
   // Clear all KV cache entries after position n_past
-  llama_memory_seq_rm(llama_get_memory(ctx), 0, n_past, -1);
+  auto * kv = llama_get_memory(ctx);
+  bool clear_result = llama_memory_seq_rm(kv, 0, n_past, -1);
+  if (!clear_result) {
+    fprintf(stdout, "[DEBUG] llama_memory_seq_rm failed (likely using a non-Transformer model)! Trying full clear...");
+    llama_memory_clear(kv, false);
+    n_past = 0;
+    new_n_past = n_past;
+  }
 
   size_t num_chunks = mtmd_input_chunks_size(chunks);
 

package/src/llama.cpp/common/arg.cpp

@@ -1530,6 +1530,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.ctx_shift = false;
         }
     ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_PERPLEXITY}).set_env("LLAMA_ARG_NO_CONTEXT_SHIFT"));
+    add_opt(common_arg(
+        {"--context-shift"},
+        string_format("enables context shift on infinite text generation (default: %s)", params.ctx_shift ? "enabled" : "disabled"),
+        [](common_params & params) {
+            params.ctx_shift = true;
+        }
+    ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_IMATRIX, LLAMA_EXAMPLE_PERPLEXITY}).set_env("LLAMA_ARG_CONTEXT_SHIFT"));
     add_opt(common_arg(
         {"--chunks"}, "N",
         string_format("max number of chunks to process (default: %d, -1 = all)", params.n_chunks),
@@ -1748,7 +1755,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params) {
             params.warmup = false;
         }
-    ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_RETRIEVAL}));
+    ).set_examples({LLAMA_EXAMPLE_MAIN, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_EMBEDDING, LLAMA_EXAMPLE_RETRIEVAL, LLAMA_EXAMPLE_PERPLEXITY}));
     add_opt(common_arg(
         {"--spm-infill"},
         string_format(
@@ -1823,7 +1830,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params, const std::string & value) {
             params.sampling.top_n_sigma = std::stof(value);
         }
-    ).set_examples({LLAMA_EXAMPLE_MAIN}).set_sparam());
+    ).set_sparam());
     add_opt(common_arg(
         {"--xtc-probability"}, "N",
         string_format("xtc probability (default: %.1f, 0.0 = disabled)", (double)params.sampling.xtc_probability),
@@ -2247,9 +2254,11 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_examples({LLAMA_EXAMPLE_PERPLEXITY}));
     add_opt(common_arg(
         {"-dt", "--defrag-thold"}, "N",
-        string_format("KV cache defragmentation threshold (default: %.1f, < 0 - disabled)", (double)params.defrag_thold),
+        string_format("KV cache defragmentation threshold (DEPRECATED)"),
         [](common_params & params, const std::string & value) {
-            params.defrag_thold = std::stof(value);
+            GGML_UNUSED(params);
+            GGML_UNUSED(value);
+            LOG_WRN("DEPRECATED: --defrag-thold is deprecated and no longer necessary to specify\n");
         }
     ).set_env("LLAMA_ARG_DEFRAG_THOLD"));
     add_opt(common_arg(

package/src/llama.cpp/common/chat.cpp

@@ -134,6 +134,7 @@ struct templates_params {
     json extra_context;
     bool add_bos;
     bool add_eos;
+    bool is_inference = true;
 };
 
 common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
@@ -619,7 +620,6 @@ const char * common_reasoning_format_name(common_reasoning_format format) {
         case COMMON_REASONING_FORMAT_AUTO:     return "auto";
         case COMMON_REASONING_FORMAT_DEEPSEEK: return "deepseek";
         case COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY: return "deepseek-legacy";
-        case COMMON_REASONING_FORMAT_GRANITE: return "granite";
         default:
             throw std::runtime_error("Unknown reasoning format");
     }
@@ -1324,6 +1324,17 @@ static common_chat_params common_chat_params_init_gpt_oss(const common_chat_temp
     common_chat_params data;
     auto prompt = apply(tmpl, inputs);
 
+    // Check if we need to replace the return token with end token during
+    // inference and without generation prompt. For more details see:
+    // https://github.com/ggml-org/llama.cpp/issues/15417
+    if (inputs.is_inference && !inputs.add_generation_prompt) {
+        static constexpr std::string_view return_token = "<|return|>";
+        static constexpr std::string_view end_token    = "<|end|>";
+        if (size_t pos = prompt.rfind(return_token); pos != std::string::npos) {
+            prompt.replace(pos, return_token.length(), end_token);
+        }
+    }
+
     data.prompt = prompt;
     data.format = COMMON_CHAT_FORMAT_GPT_OSS;
 
@@ -1337,6 +1348,26 @@ static common_chat_params common_chat_params_init_gpt_oss(const common_chat_temp
         "<|end|>",
     };
 
+    if (!inputs.json_schema.is_null()) {
+        data.grammar_lazy = false;
+        data.grammar = build_grammar([&](const common_grammar_builder & builder) {
+            auto schema = inputs.json_schema;
+            builder.resolve_refs(schema);
+
+            auto not_end = builder.add_rule("not-end",
+                "[^<] | \"<\" [^|] | \"<|\" [^e] | \"<|e\" [^n] | \"<|en\" [^d] | \"<|end\" [^|] | \"<|end|\" [^>]");
+            auto analysis = builder.add_rule("analysis",
+                "\"<|channel|>analysis<|message|>\" ( " + not_end + " )* \"<|end|>\"");
+            auto constraint = builder.add_rule("constraint", "\"<|constrain|>\"? [a-zA-Z0-9_-]+");
+            auto final = builder.add_rule("final",
+                "\"<|channel|>final\" ( \" \" " + constraint + " )? \"<|message|>\" " +
+                builder.add_schema("response", schema)
+            );
+
+            builder.add_rule("root", "( " + analysis + " \"<|start|>assistant\" )? " + final);
+        });
+    }
+
     if (inputs.tools.is_array() && !inputs.tools.empty()) {
         data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
         data.grammar = build_grammar([&](const common_grammar_builder & builder) {
@@ -2097,7 +2128,7 @@ static common_chat_params common_chat_templates_apply_jinja(
     }
 
     // GPT-OSS
-    if (src.find("<|channel|>") != std::string::npos && params.json_schema.is_null()) {
+    if (src.find("<|channel|>") != std::string::npos) {
         return common_chat_params_init_gpt_oss(tmpl, params);
     }
 

package/src/llama.cpp/common/common.cpp

@@ -558,13 +558,6 @@ std::string string_from(const struct llama_context * ctx, const std::vector<llam
 
         auto detokenized = common_token_to_piece(ctx, token);
 
-        detokenized.erase(
-            std::remove_if(
-                detokenized.begin(),
-                detokenized.end(),
-                [](const unsigned char c) { return !std::isprint(c); }),
-            detokenized.end());
-
         buf << "'" << detokenized << "'"
             << ":" << std::to_string(token);
     }
@@ -589,13 +582,6 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat
 
         auto detokenized = common_token_to_piece(ctx, batch.token[i]);
 
-        detokenized.erase(
-                std::remove_if(
-                    detokenized.begin(),
-                    detokenized.end(),
-                    [](const unsigned char c) { return !std::isprint(c); }),
-                detokenized.end());
-
         buf << "\n"          << std::to_string(i)
             << ", token '"   << detokenized << "'"
             << ", pos "      << std::to_string(batch.pos[i])
@@ -1167,7 +1153,6 @@ struct llama_context_params common_context_params_to_llama(const common_params &
     cparams.yarn_orig_ctx     = params.yarn_orig_ctx;
     cparams.pooling_type      = params.pooling_type;
     cparams.attention_type    = params.attention_type;
-    cparams.defrag_thold      = params.defrag_thold;
     cparams.cb_eval           = params.cb_eval;
     cparams.cb_eval_user_data = params.cb_eval_user_data;
     cparams.offload_kqv       = !params.no_kv_offload;

package/src/llama.cpp/common/common.h

@@ -239,12 +239,15 @@ struct common_params_diffusion {
     bool    add_gumbel_noise = false; // add gumbel noise to the logits if temp > 0.0
 };
 
+// reasoning API response format (not to be confused as chat template's reasoning format)
 enum common_reasoning_format {
     COMMON_REASONING_FORMAT_NONE,
-    COMMON_REASONING_FORMAT_AUTO,
+    COMMON_REASONING_FORMAT_AUTO,            // Same as deepseek, using `message.reasoning_content`
     COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY, // Extract thinking tag contents and return as `message.reasoning_content`, or leave inline in <think> tags in stream mode
     COMMON_REASONING_FORMAT_DEEPSEEK,        // Extract thinking tag contents and return as `message.reasoning_content`, including in streaming deltas.
-    COMMON_REASONING_FORMAT_GRANITE,         // Extract thinking tag contents and return as `message.reasoning_content`, including in streaming deltas.
+    // do not extend this enum unless you absolutely have to
+    // in most cases, use COMMON_REASONING_FORMAT_AUTO
+    // see: https://github.com/ggml-org/llama.cpp/pull/15408
 };
 
 
@@ -286,7 +289,6 @@ struct common_params {
     float   yarn_beta_fast        = 32.0f; // YaRN low correction dim
     float   yarn_beta_slow        =  1.0f; // YaRN high correction dim
     int32_t yarn_orig_ctx         =     0; // YaRN original context length
-    float   defrag_thold          =  0.1f; // KV cache defragmentation threshold
 
     // offload params
     std::vector<ggml_backend_dev_t> devices; // devices to use for offloading
@@ -373,7 +375,7 @@ struct common_params {
     bool cont_batching     = true;  // insert new sequences for decoding on-the-fly
     bool flash_attn        = false; // flash attention
     bool no_perf           = false; // disable performance metrics
-    bool ctx_shift         = true;  // context shift on inifinite text generation
+    bool ctx_shift         = false;  // context shift on infinite text generation
     bool swa_full          = false; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
     bool kv_unified        = false; // enable unified KV cache
 

package/src/llama.cpp/ggml/CMakeLists.txt

@@ -158,7 +158,6 @@ option(GGML_CUDA                            "ggml: use CUDA"
 option(GGML_MUSA                            "ggml: use MUSA"                                  OFF)
 option(GGML_CUDA_FORCE_MMQ                  "ggml: use mmq kernels instead of cuBLAS"         OFF)
 option(GGML_CUDA_FORCE_CUBLAS               "ggml: always use cuBLAS instead of mmq kernels"  OFF)
-option(GGML_CUDA_F16                        "ggml: use 16 bit floats for some calculations"   OFF)
 set   (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
                                             "ggml: max. batch size for using peer access")
 option(GGML_CUDA_NO_PEER_COPY               "ggml: do not use peer to peer copies"            OFF)

package/src/llama.cpp/ggml/include/ggml.h

@@ -244,6 +244,13 @@
 #define GGML_MROPE_SECTIONS   4
 
 #define GGML_UNUSED(x) (void)(x)
+#ifdef __CUDACC__
+template<typename... Args>
+__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {}
+#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__)
+#else
+#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0)
+#endif // __CUDACC__
 
 #define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))
 
@@ -505,6 +512,7 @@ extern "C" {
         GGML_OP_IM2COL,
         GGML_OP_IM2COL_BACK,
         GGML_OP_CONV_2D,
+        GGML_OP_CONV_3D,
         GGML_OP_CONV_2D_DW,
         GGML_OP_CONV_TRANSPOSE_2D,
         GGML_OP_POOL_1D,
@@ -1933,6 +1941,23 @@ extern "C" {
             int                   d0,  // dilation dimension 0
             int                   d1); // dilation dimension 1
 
+    GGML_API struct ggml_tensor * ggml_conv_3d(
+            struct ggml_context * ctx,
+            struct ggml_tensor  * a,   // kernel [KW, KH, KD, IC * OC]
+            struct ggml_tensor  * b,   // input  [W, H, D, C * N]
+            int                   s0,  // stride
+            int                   s1,
+            int                   s2,
+            int                   p0,  // padding
+            int                   p1,
+            int                   p2,
+            int                   d0,  // dilation
+            int                   d1,
+            int                   d2,
+            int                   n_channels,
+            int                   n_batch,
+            int                   n_channels_out);
+
     enum ggml_op_pool {
         GGML_OP_POOL_MAX,
         GGML_OP_POOL_AVG,

package/src/llama.cpp/ggml/src/ggml-cpu/arch/powerpc/quants.c

@@ -278,6 +278,72 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
 #endif
 }
 
+void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
+    assert(nrc == 1);
+    UNUSED(nrc);
+    UNUSED(bx);
+    UNUSED(by);
+    UNUSED(bs);
+    assert(n % QK_MXFP4 == 0);
+    static_assert(QK_MXFP4 == QK8_0, "QK_MXFP4 and QK8_0 must be the same");
+
+    const block_mxfp4 * GGML_RESTRICT x = vx;
+    const block_q8_0 * GGML_RESTRICT y = vy;
+
+    const int nb = n / QK_MXFP4;
+
+    int ib = 0;
+    float sumf = 0;
+
+#if defined(__POWER9_VECTOR__)
+    const vector signed char lowMask = vec_splats((signed char)0xF);
+    const vector unsigned char vshift4 = vec_splats((unsigned char)4);
+    vector float vsumf0 = vec_splats(0.0f);
+
+    vector signed char kv = vec_xl(0, (const signed char *)kvalues_mxfp4);
+
+#pragma GCC unroll 8
+    for (; ib < nb; ++ib) {
+        __builtin_prefetch(x[ib].qs, 0, 1);
+        __builtin_prefetch(y[ib].qs, 0, 1);
+
+        vector float vyd = vec_splats(GGML_CPU_FP16_TO_FP32(y[ib].d) *
+                                      GGML_E8M0_TO_FP32_HALF(x[ib].e));
+
+        vector signed char q8y0 = vec_xl( 0, y[ib].qs);
+        vector signed char q8y1 = vec_xl(16, y[ib].qs);
+
+        vector signed char qxs = (vector signed char)vec_xl(0, x[ib].qs);
+
+        vector unsigned char lo_nibbles = (vector unsigned char)vec_and(qxs, lowMask);
+        vector unsigned char hi_nibbles = (vector unsigned char)vec_sr(qxs, vshift4);
+
+        vector signed char q4x0 = vec_perm(kv, kv, lo_nibbles);
+        vector signed char q4x1 = vec_perm(kv, kv, hi_nibbles);
+
+        vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0));
+        vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1));
+
+        vector signed int vsumi0 = vec_splats((int32_t)0);
+        vsumi0 = vec_sum4s(qv0, vsumi0);
+        vsumi0 = vec_sum4s(qv1, vsumi0);
+
+        vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vyd, vsumf0);
+    }
+
+    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
+    vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
+    sumf = vec_extract(vsumf0, 0);
+    *s = sumf;
+#else
+    UNUSED(x);
+    UNUSED(y);
+    UNUSED(ib);
+    UNUSED(sumf);
+    ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
+#endif
+}
+
 void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
     const int qk = QK8_0;
     const int nb = n / qk;