@novastera-oss/llamarn 0.2.4 → 0.2.6

package/cpp/rn-completion.cpp

@@ -147,30 +147,23 @@ CompletionResult run_completion(
         json data = options.to_json();
         // Prepare the sampling parameters
         const auto& params = rn_ctx->params;
-
-        // Set the prompt
-        if (data.contains("prompt")) {
-            // Tokenize the prompt
-            const auto& tokenized_prompts = tokenize_input_prompts(rn_ctx->vocab, data["prompt"], true, true);
-            if (tokenized_prompts.empty() || tokenized_prompts[0].empty()) {
-                result.success = false;
-                result.error_msg = "Empty prompt";
-                result.error_type = RN_ERROR_INVALID_PARAM;
-                return result;
+        
+        // Create a copy of sampling parameters and apply grammar if provided
+        common_params_sampling sampling_params = params.sampling;
+        if (!options.grammar.empty()) {
+            sampling_params.grammar = options.grammar;
+            // Force grammar_lazy to false whenever tools are present to ensure strict JSON format enforcement
+            if (!options.tools.empty()) {
+                sampling_params.grammar_lazy = false;
+            } else {
+                sampling_params.grammar_lazy = options.grammar_lazy;
+            }
+            // Pass grammar_triggers if any were provided by chat_params and passed via options
+            if (!options.grammar_triggers.empty()) {
+                sampling_params.grammar_triggers = options.grammar_triggers;
             }
-            state.prompt_tokens = std::move(tokenized_prompts[0]);
-        } else {
-            result.success = false;
-            result.error_msg = "No prompt provided";
-            result.error_type = RN_ERROR_INVALID_PARAM;
-            return result;
         }
 
-        // Configure state
-        state.n_ctx = llama_n_ctx(rn_ctx->ctx);
-        state.n_predict = options.n_predict > 0 ? options.n_predict : params.n_predict;
-        state.n_remaining = state.n_predict;
-
         // Parse tool_choice
         if (options.tool_choice == "auto") {
             state.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO;
@@ -179,8 +172,8 @@ CompletionResult run_completion(
         } else if (options.tool_choice == "required") {
             state.tool_choice = COMMON_CHAT_TOOL_CHOICE_REQUIRED;
         }
-        // Initialize the sampler
-        state.sampler = common_sampler_init(rn_ctx->model, params.sampling);
+        // Initialize the sampler with the updated sampling parameters
+        state.sampler = common_sampler_init(rn_ctx->model, sampling_params);
         if (!state.sampler) {
             result.success = false;
             result.error_msg = "Failed to initialize sampler";
@@ -201,6 +194,29 @@ CompletionResult run_completion(
             }
         }
 
+        // Set the prompt
+        if (data.contains("prompt")) {
+            // Tokenize the prompt
+            const auto& tokenized_prompts = tokenize_input_prompts(rn_ctx->vocab, data["prompt"], true, true);
+            if (tokenized_prompts.empty() || tokenized_prompts[0].empty()) {
+                result.success = false;
+                result.error_msg = "Empty prompt";
+                result.error_type = RN_ERROR_INVALID_PARAM;
+                return result;
+            }
+            state.prompt_tokens = std::move(tokenized_prompts[0]);
+        } else {
+            result.success = false;
+            result.error_msg = "No prompt provided";
+            result.error_type = RN_ERROR_INVALID_PARAM;
+            return result;
+        }
+
+        // Configure state
+        state.n_ctx = llama_n_ctx(rn_ctx->ctx);
+        state.n_predict = options.n_predict > 0 ? options.n_predict : params.n_predict;
+        state.n_remaining = state.n_predict;
+
         // Process the prompt
         for (int i = 0; i < (int)state.prompt_tokens.size(); ++i) {
             llama_token token = state.prompt_tokens[i];
@@ -222,12 +238,22 @@ CompletionResult run_completion(
                 return result;
             }
 
-            common_sampler_accept(state.sampler, token, true);
+            // For lazy grammars, we need to accept prompt tokens to properly set up the grammar state
+            // For non-lazy grammars, we only accept if no grammar is present (grammar needs clean state)
+            if (sampling_params.grammar.empty() || sampling_params.grammar_lazy) {
+                common_sampler_accept(state.sampler, token, true);
+            }
             state.n_past++;
         }
 
         result.n_prompt_tokens = state.prompt_tokens.size();
 
+        // If using a non-lazy grammar, ensure the sampler is in a clean state for the grammar
+        if (!sampling_params.grammar.empty() && !sampling_params.grammar_lazy) {
+            common_sampler_free(state.sampler);
+            state.sampler = common_sampler_init(rn_ctx->model, sampling_params);
+        }
+
         // Start generating tokens
         const int64_t t_start_generation = ggml_time_us();
 
@@ -328,6 +354,15 @@ CompletionResult run_chat_completion(
     std::function<bool(const std::string&, bool)> callback) {
 
     CompletionResult result;
+    // Log incoming tools via callback
+    /*
+    if (callback) {
+        std::string tools_json_str = options.tools.dump(2);
+        std::string debug_msg = "[DEBUG RN_COMPLETION_OPTIONS_TOOLS] options.tools JSON: " + tools_json_str;
+        callback(debug_msg, false); // false for is_done
+    }
+    */
+    completion_state state;
 
     if (!rn_ctx || !rn_ctx->model || !rn_ctx->ctx) {
         result.success = false;
@@ -362,9 +397,9 @@ CompletionResult run_chat_completion(
         // Parse tools if present
         if (data.contains("tools") && !data["tools"].empty()) {
             template_inputs.tools = common_chat_tools_parse_oaicompat(data["tools"]);
-            // Check if parallel tool calls are allowed (advanced feature)
-            template_inputs.parallel_tool_calls = data.contains("parallel_tool_calls") ?
-                json_value(data, "parallel_tool_calls", false) : false;
+            // Force parallel_tool_calls to true if tools are present, as this generally
+            // aligns with grammars expecting a list of tool calls.
+            template_inputs.parallel_tool_calls = true;
         }
 
         // Parse tool_choice if present
@@ -378,13 +413,42 @@ CompletionResult run_chat_completion(
         // Apply template
         const auto& chat_params = common_chat_templates_apply(rn_ctx->chat_templates.get(), template_inputs);
 
-        // Set up completion options
         CompletionOptions cmpl_options = options;
         cmpl_options.prompt = chat_params.prompt;
 
-        // Apply grammar if needed
         if (!chat_params.grammar.empty()) {
             cmpl_options.grammar = chat_params.grammar;
+            // Always force grammar_lazy to false when tools are present
+            if (!template_inputs.tools.empty()) {
+                cmpl_options.grammar_lazy = false;
+            } else {
+                // Only use chat_params.grammar_lazy if no tools are present
+                cmpl_options.grammar_lazy = chat_params.grammar_lazy;
+            }
+            // Default to grammar_triggers provided by chat_params
+            cmpl_options.grammar_triggers = chat_params.grammar_triggers;
+
+            bool original_grammar_lazy = chat_params.grammar_lazy; // Store original for logging
+
+            // Add a debug log to observe final grammar_lazy and grammar_triggers
+            /*
+            if (callback) {
+                std::string tool_choice_str;
+                switch (template_inputs.tool_choice) {
+                    case COMMON_CHAT_TOOL_CHOICE_AUTO: tool_choice_str = "auto"; break;
+                    case COMMON_CHAT_TOOL_CHOICE_NONE: tool_choice_str = "none"; break;
+                    case COMMON_CHAT_TOOL_CHOICE_REQUIRED: tool_choice_str = "required"; break;
+                    default: tool_choice_str = "unknown"; break;
+                }
+                std::string debug_msg = "[DEBUG CHAT_PARAMS] grammar_lazy: " +
+                                      std::string(cmpl_options.grammar_lazy ? "true" : "false") +
+                                      " | grammar_triggers_count: " + std::to_string(cmpl_options.grammar_triggers.size()) + // Log triggers from cmpl_options
+                                      " | For Tool Choice: " + tool_choice_str +
+                                      " | Parallel Tool Calls: " + std::string(template_inputs.parallel_tool_calls ? "true" : "false") +
+                                      " | Original chat_params.grammar_lazy: " + std::string(original_grammar_lazy ? "true" : "false"); // Log original lazy
+                callback(debug_msg, false);
+            }
+            */
         }
 
         // Run standard completion with the processed prompt
@@ -435,31 +499,15 @@ CompletionResult run_chat_completion(
             
             // Add parsed content and tool calls if available
             if (has_parsed_content && !parsed_msg.tool_calls.empty()) {
-                // Set content to the parsed content (may be null for tool-only responses)
-                if (!parsed_msg.content.empty()) {
-                    choice["message"]["content"] = parsed_msg.content;
-                } else {
-                    choice["message"]["content"] = nullptr;
-                }
-                
-                // Add tool calls to the message
-                json tool_calls = json::array();
-                for (const auto& tool_call : parsed_msg.tool_calls) {
-                    json tc = {
-                        {"id", tool_call.id.empty() ? ("call_" + std::to_string(std::rand())) : tool_call.id},
-                        {"type", "function"},
-                        {"function", {
-                            {"name", tool_call.name},
-                            {"arguments", tool_call.arguments}
-                        }}
-                    };
-                    tool_calls.push_back(tc);
-                }
-                choice["message"]["tool_calls"] = tool_calls;
+                // Use the server.cpp approach: let the common_chat_msg handle the JSON conversion
+                choice["message"] = parsed_msg.to_json_oaicompat<json>();
                 choice["finish_reason"] = "tool_calls";
+            } else if (has_parsed_content && !parsed_msg.content.empty()) {
+                // Regular text response with parsed content
+                choice["message"]["content"] = parsed_msg.content;
             } else {
-                // Regular text response
-                choice["message"]["content"] = has_parsed_content ? parsed_msg.content : result.content;
+                // Fallback to raw content if parsing failed or no tools
+                choice["message"]["content"] = result.content;
             }
 
             choices.push_back(choice);
@@ -487,3 +535,4 @@ CompletionResult run_chat_completion(
 
 } // namespace facebook::react
 
+

package/cpp/rn-utils.hpp

@@ -10,7 +10,7 @@
 
 // Change JSON_ASSERT from assert() to GGML_ASSERT:
 #define JSON_ASSERT GGML_ASSERT
-#include "json.hpp"
+#include "nlohmann/json.hpp"
 #include "base64.hpp"
 #include "chat.h"
 
@@ -66,6 +66,7 @@ struct CompletionOptions {
     int seed = -1;
     json tools;         // tools for function calling
     std::string tool_choice = "auto"; // tool choice mode: "auto", "none", or "required"
+    std::vector<common_grammar_trigger> grammar_triggers; // For lazy grammar
 
     // Convert to JSON for the completion API
     json to_json() const {
@@ -98,6 +99,12 @@ struct CompletionOptions {
             j["tools"] = tools;
             j["tool_choice"] = tool_choice;
         }
+        // Add grammar_triggers if available (mainly for internal use, not direct API option)
+        if (!grammar_triggers.empty()) {
+            // This part is tricky as json can't directly hold common_grammar_trigger easily.
+            // For now, we'll skip adding it to the generic to_json() as it's passed internally.
+            // If it were needed for an API, we'd need a proper serialization for grammar_triggers.
+        }
         return j;
     }
 

package/ios/include/common/minja/chat-template.hpp

@@ -22,7 +22,7 @@
 #include <string>
 #include <vector>
 
-#include <json.hpp>
+#include <nlohmann/json.hpp>
 
 using json = nlohmann::ordered_json;
 

package/ios/include/common/minja/minja.hpp

@@ -29,7 +29,7 @@
 #include <utility>
 #include <vector>
 
-#include <json.hpp>
+#include <nlohmann/json.hpp>
 
 using json = nlohmann::ordered_json;
 

package/ios/include/json-schema-to-grammar.h

@@ -1,9 +1,9 @@
 #pragma once
 
-#include "ggml.h"
-// Change JSON_ASSERT from assert() to GGML_ASSERT:
-#define JSON_ASSERT GGML_ASSERT
-#include "json.hpp"
+#include <nlohmann/json_fwd.hpp>
+
+#include <functional>
+#include <string>
 
 std::string json_schema_to_grammar(const nlohmann::ordered_json & schema,
                                    bool force_gbnf = false);

package/ios/include/llama.h

@@ -259,9 +259,9 @@ extern "C" {
         llama_token  *  token;
         float        *  embd;
         llama_pos    *  pos;
-        int32_t      *  n_seq_id;
-        llama_seq_id ** seq_id;
-        int8_t       *  logits; // TODO: rename this to "output"
+        int32_t      *  n_seq_id; // TODO: remove, should belong to only 1 sequence
+        llama_seq_id ** seq_id;   // TODO: become llama_seq_id * seq_id;
+        int8_t       *  logits;   // TODO: rename this to "output"
     } llama_batch;
 
     enum llama_model_kv_override_type {
@@ -366,6 +366,8 @@ extern "C" {
         bool no_perf;     // measure performance timings
         bool op_offload;  // offload host tensor operations to device
         bool swa_full;    // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
+                          // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases
+                          //       ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573
     };
 
     // model quantization parameters
@@ -502,6 +504,7 @@ extern "C" {
     LLAMA_API int32_t llama_model_n_layer    (const struct llama_model * model);
     LLAMA_API int32_t llama_model_n_head     (const struct llama_model * model);
     LLAMA_API int32_t llama_model_n_head_kv  (const struct llama_model * model);
+    LLAMA_API int32_t llama_model_n_swa      (const struct llama_model * model);
 
     // Get the model's RoPE frequency scaling factor
     LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model);
@@ -652,7 +655,6 @@ extern "C" {
     // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
     // If the KV cache is RoPEd, the KV data is updated accordingly:
     //   - lazily on next llama_decode()
-    //   - explicitly with llama_kv_self_update()
     // p0 < 0 : [0,  p1]
     // p1 < 0 : [p0, inf)
     LLAMA_API void llama_kv_self_seq_add(
@@ -665,7 +667,6 @@ extern "C" {
     // Integer division of the positions by factor of `d > 1`
     // If the KV cache is RoPEd, the KV data is updated accordingly:
     //   - lazily on next llama_decode()
-    //   - explicitly with llama_kv_self_update()
     // p0 < 0 : [0,  p1]
     // p1 < 0 : [p0, inf)
     LLAMA_API void llama_kv_self_seq_div(
@@ -677,12 +678,14 @@ extern "C" {
 
     // Returns the smallest position present in the KV cache for the specified sequence
     // This is typically non-zero only for SWA caches
+    // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the KV cache
     // Return -1 if the sequence is empty
     LLAMA_API llama_pos llama_kv_self_seq_pos_min(
             struct llama_context * ctx,
                     llama_seq_id   seq_id);
 
     // Returns the largest position present in the KV cache for the specified sequence
+    // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the KV cache
     // Return -1 if the sequence is empty
     LLAMA_API llama_pos llama_kv_self_seq_pos_max(
             struct llama_context * ctx,
@@ -691,14 +694,15 @@ extern "C" {
     // Defragment the KV cache
     // This will be applied:
     //   - lazily on next llama_decode()
-    //   - explicitly with llama_kv_self_update()
-    LLAMA_API void llama_kv_self_defrag(struct llama_context * ctx);
+    LLAMA_API DEPRECATED(void llama_kv_self_defrag(struct llama_context * ctx),
+            "simply remove this call, the context will automatically decide when to do a defragmentation based on 'defrag_thold'");
 
     // Check if the context supports KV cache shifting
     LLAMA_API bool llama_kv_self_can_shift(const struct llama_context * ctx);
 
     // Apply the KV cache updates (such as K-shifts, defragmentation, etc.)
-    LLAMA_API void llama_kv_self_update(struct llama_context * ctx);
+    LLAMA_API DEPRECATED(void llama_kv_self_update(struct llama_context * ctx),
+            "simply remove this call, updates are applied lazily on the next llama_decode()");
 
     //
     // State / sessions