cohere-transcribe 0.1.0

data/vendor/crispasr/src/core/gguf_loader.h

@@ -0,0 +1,216 @@
+// src/core/gguf_loader.h — shared GGUF weight loading scaffolding.
+//
+// Every model implementation in src/ has its own copy of the "open a
+// GGUF file, read its hyperparameters, allocate a backend buffer, mmap
+// the weight data, and build a name -> tensor lookup map" dance. The
+// code is ~40-60 lines per model and is essentially identical across
+// them, with only the model-specific prefix and tensor naming scheme
+// changing.
+//
+// This helper extracts the shared scaffolding. What stays model-specific:
+//
+//   * Hyperparameter reading (each model has its own hparams struct
+//     and GGUF key prefix, e.g. "parakeet.n_layers" vs "voxtral.n_layers").
+//   * Vocabulary / tokenizer loading (varies by tokenizer type).
+//   * The actual per-field assignment loop that pulls tensors out of
+//     the map and stores them in per-layer struct fields.
+//
+// What this helper does for the model:
+//
+//   * Opens the GGUF file in two passes (metadata, then tensor alloc).
+//   * Provides scalar / string / array reader helpers with defaults.
+//   * Allocates the backend buffer and mmap-copies the weight data.
+//   * Builds the std::map<std::string, ggml_tensor *> tensor
+//     lookup map and returns it in a WeightLoad struct.
+//   * Provides require() / try_get() tensor lookup helpers that log a
+//     sensible error message when a required tensor is missing.
+//
+// Usage pattern (each model's *_model_load function):
+//
+//   // 1. Metadata pass — read hyperparameters.
+//   gguf_context * meta = core_gguf::open_metadata(path);
+//   if (!meta) return false;
+//   hp.n_layers = core_gguf::kv_u32(meta, "mymodel.n_layers", hp.n_layers);
+//   // ... other hparams
+//   core_gguf::load_vocab_array(meta, "tokenizer.ggml.tokens", vocab);
+//   core_gguf::free_metadata(meta);
+//
+//   // 2. Weight pass — allocate backend buffer, mmap, build tensor map.
+//   core_gguf::WeightLoad wl;
+//   if (!core_gguf::load_weights(path, backend, wl)) return false;
+//   model.ctx     = wl.ctx;
+//   model.buf     = wl.buf;
+//   model.tensors = std::move(wl.tensors);
+//
+//   // 3. Bind named tensors into struct fields.
+//   model.attn.q_w = core_gguf::require(model.tensors, "encoder.attn.q.weight", "mymodel");
+//   ... etc.
+
+#pragma once
+
+#include "ggml.h"
+#include "ggml-backend.h"
+#include "gguf.h"
+
+#include <cstdint>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace core_gguf {
+
+// ---------------------------------------------------------------------------
+// Pass 1: metadata (hyperparameters + vocab).
+// ---------------------------------------------------------------------------
+
+// Open the GGUF for metadata-only reading. Returns a gguf_context owned
+// by the caller — free with free_metadata() when done reading keys.
+// Returns nullptr and prints an error to stderr on failure.
+gguf_context* open_metadata(const char* path);
+
+// Free a gguf_context obtained from open_metadata().
+void free_metadata(gguf_context* gctx);
+
+// Scalar key readers with defaults. All return the default value when
+// the key is absent or the type doesn't match.
+uint32_t kv_u32(gguf_context* gctx, const char* key, uint32_t default_val);
+int32_t kv_i32(gguf_context* gctx, const char* key, int32_t default_val);
+float kv_f32(gguf_context* gctx, const char* key, float default_val);
+bool kv_bool(gguf_context* gctx, const char* key, bool default_val);
+std::string kv_str(gguf_context* gctx, const char* key, const char* default_val);
+
+// Read a string array (e.g. tokenizer.ggml.tokens). Returns an empty
+// vector when the key is missing or has the wrong type.
+std::vector<std::string> kv_str_array(gguf_context* gctx, const char* key);
+
+// Read a float32 array (e.g. tokenizer.ggml.scores). Returns an empty vector
+// when the key is missing or has the wrong type.
+std::vector<float> kv_f32_array(gguf_context* gctx, const char* key);
+
+// ---------------------------------------------------------------------------
+// Pass 2: tensor allocation + weight data copy.
+// ---------------------------------------------------------------------------
+
+// CROSS-REPO TENSOR-MAP CONTRACT (read before changing the map type!)
+// ------------------------------------------------------------------
+// `core/gguf_loader.{h,cpp}` exists in BOTH CrispASR and CrispEmbed. When
+// CrispEmbed builds, it compiles CrispASR's `crisp_audio`/`crisp_lid` sources
+// against CrispEmbed's copy of this header (they link `crispembed-core`).
+// CrispASR standalone prefers `std::map`; CrispEmbed prefers
+// `std::unordered_map` (faster). A consumer doing
+// `ctx.tensors = std::move(wl.tensors)` needs its field to be the SAME type as
+// `WeightLoad::tensors` — but that type differs per repo. Hard-coding either
+// type in the consumer broke the other build, causing the repeated
+// std::map<->unordered_map flip-flop (commits e6693b23/ad869798/d1bd3b91/
+// 1e4f1184/844f89d3, …).
+//
+// FIX: expose the type as a single alias `core_gguf::tensor_map`. Each repo's
+// header defines it as its own choice; consumers (crisp_audio/audio_tower,
+// crisp_lid/lid_cld3) declare `core_gguf::tensor_map tensors;` so it tracks
+// whichever gguf_loader.h is compiled. Do NOT hard-code the map type in those
+// consumer structs again — change this alias instead.
+using tensor_map = std::map<std::string, ggml_tensor*>;
+
+struct WeightLoad {
+    ggml_context* ctx = nullptr;
+    ggml_backend_buffer_t buf = nullptr;
+    // PLAN #69a layer offload: optional second backend buffer for tensors
+    // routed off-GPU. Non-null only when load_weights_split() was used.
+    ggml_backend_buffer_t buf_cpu = nullptr;
+    tensor_map tensors;
+};
+
+// Load all tensor metadata + weights into a new ggml_context backed by
+// a newly-allocated backend buffer. On success the WeightLoad struct is
+// populated and the caller takes ownership of ctx/buf (typically moving
+// them into the model struct).
+//
+// model_tag is used only in error messages ("parakeet: ...").
+bool load_weights(const char* path, ggml_backend_t backend, const char* model_tag, WeightLoad& out);
+
+// PLAN #69a: layer-residency-aware weight loader. Tensors for which
+// `is_gpu(tensor_name, user) == true` go on the GPU backend; the rest
+// go on the CPU backend. ggml_backend_sched then auto-routes ops to
+// follow weight residency, giving llama.cpp's `--n-gpu-layers` behaviour.
+//
+// Caller takes ownership of `out.ctx`, `out.buf` (gpu partition), and
+// `out.buf_cpu` (cpu partition). All three must be freed by the caller
+// or via free_weights() / free_weights_split() at shutdown.
+//
+// Falls back to the legacy alloc+copy path internally — the mmap
+// optimisations in load_weights() require contiguous tensor regions
+// that the split partition can't satisfy. Acceptable trade-off: users
+// who set N_GPU_LAYERS are accepting the extra RAM hit to fit the
+// model at all.
+//
+// Returns false on any allocation / load failure with a logged
+// stderr message; partial state is freed before returning.
+using IsGpuTensor = bool (*)(const char* tensor_name, void* user);
+bool load_weights_split(const char* path, ggml_backend_t gpu_backend, ggml_backend_t cpu_backend, IsGpuTensor is_gpu,
+                        void* user, const char* model_tag, WeightLoad& out);
+
+// PLAN #69a — generic predicate for the "<prefix><N>." tensor naming
+// used by every LLM-decode backend in src/. Each backend has its own
+// prefix:
+//   "blk."          (voxtral, voxtral4b, qwen3_asr, granite_speech, gemma4_e2b, mimo_asr)
+//   "llm.blk."      (glm_asr)
+//   "talker.blk."   (orpheus)
+//   "dec."          (omniasr)
+// Returns -1 if the tensor name doesn't match `<prefix><integer>.`.
+int blk_layer_of_with_prefix(const char* tensor_name, const char* prefix);
+
+// Convenience for backends using bare "blk.<N>." (the most common
+// scheme). Equivalent to blk_layer_of_with_prefix(name, "blk.").
+int blk_layer_of(const char* tensor_name);
+
+// Configurable predicate: tensors named `<cfg.prefix><N>.<rest>` go to
+// CPU iff N >= cfg.threshold; anything else (different prefix, no
+// integer, threshold-violating layer) stays on GPU. Pass a pointer
+// to a LayerSplitConfig as `user`.
+struct LayerSplitConfig {
+    const char* prefix; // e.g. "blk.", "llm.blk.", "talker.blk.", "dec."
+    int threshold;      // N — first CPU-resident layer
+};
+bool is_gpu_tensor_with_prefix(const char* tensor_name, void* user);
+
+// Bare "blk." convenience: pass a pointer to an int (the threshold)
+// as `user`. Equivalent to is_gpu_tensor_with_prefix() with
+// LayerSplitConfig{ "blk.", *N }.
+bool is_gpu_tensor_blk(const char* tensor_name, void* user);
+
+// Free a WeightLoad's resources. Call when the model is being destroyed
+// and the buffer/context are not held elsewhere.
+void free_weights(WeightLoad& wl);
+
+// PLAN #60g: hint the kernel that the mmap'd weight region is now being
+// accessed in random order (e.g., the per-layer KV revisit pattern of
+// decode steps), and that readahead is therefore wasted IO. No-op if
+// the buffer wasn't allocated through one of our mmap paths (e.g.,
+// when `CRISPASR_GGUF_MMAP=0` opts out of the default mmap loader).
+// Safe to call multiple times.
+//
+// Recommended use: after prefill completes, before entering the decode
+// loop. See PLAN #60g for the rationale.
+void mmap_advise_random(ggml_backend_buffer_t buf);
+
+// ---------------------------------------------------------------------------
+// Tensor lookup helpers
+// ---------------------------------------------------------------------------
+
+// Look up a tensor by name. Returns nullptr (silently) if missing.
+// Uses `tensor_map` (see the cross-repo contract note above) so the signature
+// tracks the per-repo map choice automatically.
+ggml_tensor* try_get(const tensor_map& tensors, const char* name);
+
+// Look up a tensor by name. Prints an error to stderr if missing but
+// still returns nullptr — the caller decides whether a missing tensor
+// is fatal.
+ggml_tensor* require(const tensor_map& tensors, const char* name, const char* model_tag);
+
+// Build a shell command that produces the formatted tensor name for a
+// per-layer lookup. Avoids the snprintf(buf, sizeof(buf), "...", i) line
+// that every loader repeats.
+std::string format_layer_name(const char* fmt, int i);
+std::string format_layer_name(const char* fmt, int i, int j);
+
+} // namespace core_gguf

data/vendor/crispasr/src/core/gpu_backend_pref.h

@@ -0,0 +1,119 @@
+#pragma once
+// gpu_backend_pref.h — process-global GPU backend preference
+//
+// Issue #214: `--gpu-backend vulkan` was ignored because every backend
+// called `ggml_backend_init_best()` which unconditionally picks CUDA
+// over Vulkan when both are compiled in. This header provides:
+//
+//   crispasr_set_gpu_backend_pref("vulkan")  — set once at startup
+//   crispasr_init_gpu_backend()              — drop-in replacement for
+//                                              ggml_backend_init_best()
+//
+// The preference is matched against ggml backend registry names
+// (case-insensitive). Common values: "cuda", "vulkan", "metal", "cpu".
+// Empty or null = auto (same as ggml_backend_init_best).
+
+#include "ggml-backend.h"
+
+#include <cstdio>
+#include <cstring>
+#include <mutex>
+#include <string>
+
+namespace crispasr_gpu_pref {
+
+// The preference is a simple global — set once at process start,
+// read many times. Thread-safe via the string copy in get().
+inline std::string& pref_storage() {
+    static std::string s;
+    return s;
+}
+
+inline std::mutex& pref_mutex() {
+    static std::mutex m;
+    return m;
+}
+
+} // namespace crispasr_gpu_pref
+
+// Set the GPU backend preference. Call once at startup, before any
+// backend init_from_file. Empty string = auto.
+inline void crispasr_set_gpu_backend_pref(const char* name) {
+    std::lock_guard<std::mutex> lock(crispasr_gpu_pref::pref_mutex());
+    crispasr_gpu_pref::pref_storage() = name ? name : "";
+}
+
+inline std::string crispasr_get_gpu_backend_pref() {
+    std::lock_guard<std::mutex> lock(crispasr_gpu_pref::pref_mutex());
+    return crispasr_gpu_pref::pref_storage();
+}
+
+// Case-insensitive prefix check: does `haystack` start with `needle`?
+inline bool ci_starts_with(const char* haystack, const char* needle) {
+    for (; *needle; ++haystack, ++needle) {
+        if (!*haystack)
+            return false;
+        if (tolower((unsigned char)*haystack) != tolower((unsigned char)*needle))
+            return false;
+    }
+    return true;
+}
+
+// Drop-in replacement for ggml_backend_init_best().
+// If a gpu_backend preference is set, iterate registered devices and
+// pick the first GPU/iGPU device whose name starts with the preference
+// (e.g. "vulkan" matches "Vulkan0", "Vulkan1", …).
+// Falls back to ggml_backend_init_best() when no preference is set or
+// the preferred backend isn't found.
+inline ggml_backend_t crispasr_init_gpu_backend() {
+    std::string pref = crispasr_get_gpu_backend_pref();
+
+    if (!pref.empty()) {
+        // Iterate all registered devices and find the first GPU whose
+        // name starts with the preference string.
+        for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
+            ggml_backend_dev_t dev = ggml_backend_dev_get(i);
+            enum ggml_backend_dev_type dt = ggml_backend_dev_type(dev);
+            if (dt != GGML_BACKEND_DEVICE_TYPE_GPU && dt != GGML_BACKEND_DEVICE_TYPE_IGPU)
+                continue;
+            const char* dev_name = ggml_backend_dev_name(dev);
+            if (ci_starts_with(dev_name, pref.c_str())) {
+                ggml_backend_t result = ggml_backend_dev_init(dev, nullptr);
+                if (result) {
+                    fprintf(stderr, "%s: using preferred GPU backend: %s\n", __func__, dev_name);
+                    return result;
+                }
+                fprintf(stderr, "%s: preferred GPU device '%s' failed to init, trying fallback\n", __func__, dev_name);
+            }
+        }
+
+        // Also try matching against the registry (backend library) name,
+        // e.g. the user writes "vulkan" and the registry name is "Vulkan".
+        for (size_t i = 0; i < ggml_backend_reg_count(); ++i) {
+            ggml_backend_reg_t reg = ggml_backend_reg_get(i);
+            const char* reg_name = ggml_backend_reg_name(reg);
+            if (!ci_starts_with(reg_name, pref.c_str()))
+                continue;
+            // Found the registry — pick the first GPU device from it.
+            for (size_t j = 0; j < ggml_backend_reg_dev_count(reg); ++j) {
+                ggml_backend_dev_t dev = ggml_backend_reg_dev_get(reg, j);
+                enum ggml_backend_dev_type dt = ggml_backend_dev_type(dev);
+                if (dt != GGML_BACKEND_DEVICE_TYPE_GPU && dt != GGML_BACKEND_DEVICE_TYPE_IGPU)
+                    continue;
+                ggml_backend_t result = ggml_backend_dev_init(dev, nullptr);
+                if (result) {
+                    fprintf(stderr, "%s: using preferred GPU backend: %s (via registry '%s')\n", __func__,
+                            ggml_backend_dev_name(dev), reg_name);
+                    return result;
+                }
+            }
+        }
+
+        fprintf(stderr,
+                "%s: WARNING: --gpu-backend '%s' requested but no matching "
+                "GPU device found, falling back to auto\n",
+                __func__, pref.c_str());
+    }
+
+    return ggml_backend_init_best();
+}