whispercpp 1.3.5 → 1.3.7

data/ext/sources/examples/talk-llama/models/plamo2.cpp

@@ -1,316 +0,0 @@
-#include "models.h"
-
-llm_build_plamo2::llm_build_plamo2(const llama_model & model, const llm_graph_params & params) :
-    llm_graph_context_mamba(params) {
-    ggml_tensor * cur;
-    ggml_tensor * inpL;
-
-    // {n_embd, n_tokens}
-    inpL = build_inp_embd(model.tok_embd);
-    cb(inpL, "embedding_output", -1);
-
-    ggml_tensor * inp_pos = build_inp_pos();
-
-    auto * inp_hybrid = build_inp_mem_hybrid();
-
-    ggml_tensor * inp_out_ids = build_inp_out_ids();
-
-    for (int il = 0; il < n_layer; ++il) {
-        ggml_tensor * residual = inpL;
-
-        // ggml_graph_add_node(gf, model.layers[il].attn_norm);
-        // cb(model.layers[il].attn_norm, "attn_norm", il);
-
-        // pre_mixer_norm
-        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
-
-        // check if this layer is Mamba or Attention
-        bool is_mamba_layer = hparams.is_recurrent(il);
-
-        if (is_mamba_layer) {
-            // PLaMo-2 Mamba layer
-            cur = build_plamo2_mamba_layer(inp_hybrid->get_recr(), cur, model, ubatch, il);
-        } else {
-            // PLaMo-2 Attention layer
-            cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il);
-        }
-
-        // post_mixer_norm
-        cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);
-        cb(cur, "attn_post_norm", il);
-
-        // residual connection
-        cur = ggml_add(ctx0, cur, residual);
-        cb(cur, "attn_residual", il);
-        residual = cur;
-
-        // pre-ffn norm
-        cur = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
-        cb(cur, "ffn_pre_norm", il);
-
-        // feed-forward network
-        cur = build_ffn(cur,
-                model.layers[il].ffn_up, NULL, NULL,
-                NULL, NULL, NULL,
-                model.layers[il].ffn_down, NULL, NULL,
-                NULL, LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);
-        cb(cur, "ffn_out", il);
-
-        // post ffn norm
-        cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);
-        cb(cur, "ffn_post_norm", il);
-
-        if (il == n_layer - 1 && inp_out_ids) {
-            cur      = ggml_get_rows(ctx0, cur, inp_out_ids);
-            residual = ggml_get_rows(ctx0, residual, inp_out_ids);
-        }
-
-        // residual connection
-        cur = ggml_add(ctx0, cur, residual);
-        cb(cur, "ffn_residual", il);
-
-        inpL = cur;
-    }
-
-    cur = inpL;
-
-    // final norm
-    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
-    cb(cur, "result_norm", -1);
-
-    res->t_embd = cur;
-
-    // lm_head
-    cur = build_lora_mm(model.output, cur);
-    cb(cur, "result_output", -1);
-
-    // Explicitly mark as output tensor to ensure proper backend assignment
-    ggml_set_output(cur);
-
-    res->t_logits = cur;
-
-    ggml_build_forward_expand(gf, cur);
-}
-
-ggml_tensor * llm_build_plamo2::build_plamo2_attn_layer(llm_graph_input_attn_kv * inp,
-                                                        ggml_tensor *             inp_pos,
-                                                        ggml_tensor *             cur,
-                                                        const llama_model &       model,
-                                                        int                       il) {
-    // self-attention
-    {
-        // PLaMo-2 uses combined QKV tensor
-        ggml_tensor * qkv = build_lora_mm(model.layers[il].wqkv, cur);
-        cb(qkv, "wqkv", il);
-
-        // split QKV tensor into Q, K, V
-        const int64_t n_embd_head_q = hparams.n_embd_head_k;
-        const int64_t n_embd_head_k = hparams.n_embd_head_k;
-        const int64_t n_embd_head_v = hparams.n_embd_head_v;
-        int32_t       n_head        = hparams.n_head(il);
-        int32_t       n_head_kv     = hparams.n_head_kv(il);
-
-        const int64_t q_offset = 0;
-        const int64_t k_offset = n_embd_head_q * n_head;
-        const int64_t v_offset = k_offset + n_embd_head_k * n_head_kv;
-
-        ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_q, n_head, n_tokens, n_embd_head_q * sizeof(float),
-                                          qkv->nb[1], q_offset * ggml_element_size(qkv));
-        ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv, n_tokens, n_embd_head_k * sizeof(float),
-                                          qkv->nb[1], k_offset * ggml_element_size(qkv));
-        ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv, n_tokens, n_embd_head_v * sizeof(float),
-                                          qkv->nb[1], v_offset * ggml_element_size(qkv));
-
-        cb(Qcur, "Qcur", il);
-        cb(Kcur, "Kcur", il);
-        cb(Vcur, "Vcur", il);
-
-        Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
-        cb(Qcur, "Qcur_normed", il);
-
-        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-                             ext_factor, attn_factor, beta_fast, beta_slow);
-
-        Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
-        cb(Kcur, "Kcur_normed", il);
-
-        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-                             ext_factor, attn_factor, beta_fast, beta_slow);
-
-        cur = build_attn(inp,
-            model.layers[il].wo, NULL,
-            Qcur, Kcur, Vcur, NULL, NULL, NULL, 1.0f / sqrtf(float(n_embd_head_v)), il);
-    }
-
-    cb(cur, "attn_out", il);
-
-    return cur;
-}
-
-ggml_tensor * llm_build_plamo2::build_plamo2_mamba_layer(llm_graph_input_rs * inp,
-                                                         ggml_tensor *        cur,
-                                                         const llama_model &  model,
-                                                         const llama_ubatch & ubatch,
-                                                         int                  il) {
-    const auto * mctx_cur = inp->mctx;
-
-    const auto kv_head = mctx_cur->get_head();
-
-    const int64_t d_conv   = hparams.ssm_d_conv;
-    const int64_t d_inner  = hparams.ssm_d_inner;
-    const int64_t d_state  = hparams.ssm_d_state;
-    const int64_t n_heads  = hparams.ssm_dt_rank;
-    const int64_t head_dim = d_inner / n_heads;
-    const int64_t n_group  = hparams.ssm_n_group;
-    const int64_t n_seqs   = ubatch.n_seqs;
-
-    const int64_t n_seq_tokens = ubatch.n_seq_tokens;
-
-    GGML_ASSERT(n_seqs != 0);
-    GGML_ASSERT(ubatch.equal_seqs());
-    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
-
-    ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
-    ggml_tensor * ssm_states_all  = mctx_cur->get_s_l(il);
-
-    ggml_tensor * conv = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);
-    conv               = ggml_reshape_3d(ctx0, conv, d_conv - 1, d_inner + 2 * n_group * d_state, n_seqs);
-
-    // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs}
-    cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);
-
-    // in_proj: {n_embd, 2*d_inner} @ {n_embd, n_seq_tokens, n_seqs} => {2*d_inner, n_seq_tokens, n_seqs}
-    ggml_tensor * zx = build_lora_mm(model.layers[il].ssm_in, cur);
-    cb(zx, "mamba_in_proj", il);
-    // {8192, 5, 1, 1} -> {8192, 1, 5, 1}
-    zx = ggml_permute(ctx0, zx, 0, 2, 1, 3);
-    zx = ggml_cont_4d(ctx0, zx, head_dim * 2, n_heads, n_seq_tokens, n_seqs);
-    cb(zx, "mamba_in_proj_out", il);
-
-    // split into z and x
-    // => {head_dim * n_heads, n_seq_tokens, n_seqs}
-    ggml_tensor * x = ggml_view_4d(ctx0, zx, head_dim, n_heads, n_seq_tokens, n_seqs, zx->nb[1], zx->nb[2], zx->nb[3],
-                                   head_dim * ggml_element_size(zx));
-    x               = ggml_cont_3d(ctx0, x, head_dim * n_heads, n_seq_tokens, n_seqs);
-    // x = ggml_permute(ctx0, x, 0, 2, 1, 3);
-    cb(x, "mamba_x_split", il);
-
-    ggml_tensor * z =
-        ggml_view_4d(ctx0, zx, head_dim, n_heads, n_seq_tokens, n_seqs, zx->nb[1], zx->nb[2], zx->nb[3], 0);
-    cb(z, "mamba_z_split", il);
-
-    // conv1d
-    {
-        // => {d_conv - 1 + n_seq_tokens, d_inner, n_seqs}
-        ggml_tensor * conv_x = ggml_concat(ctx0, conv, ggml_transpose(ctx0, x), 0);
-        cb(conv_x, "mamba_conv1d_input", il);
-
-        // copy last (d_conv - 1) columns back into the state cache
-        ggml_tensor * last_conv = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, conv_x->nb[1], conv_x->nb[2],
-                                               n_seq_tokens * (conv_x->nb[0]));
-
-        ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv,
-                                               ggml_view_1d(ctx0, conv_states_all,
-                                                            (d_conv - 1) * (d_inner + 2 * n_group * d_state) * (n_seqs),
-                                                            kv_head * (d_conv - 1) * (d_inner + 2 * n_group * d_state) *
-                                                                ggml_element_size(conv_states_all))));
-        cb(conv_states_all, "mamba_conv1d_state", il);
-
-        // 1D convolution
-        x = ggml_ssm_conv(ctx0, conv_x, model.layers[il].ssm_conv1d);
-        cb(x, "mamba_conv1d", il);
-
-        x = ggml_silu(ctx0, x);
-        cb(x, "mamba_conv1d_silu", il);
-    }
-
-    // SSM
-    {
-        // bcdt_proj: {d_inner, dt_rank + 2*d_state} @ {d_inner, n_seq_tokens, n_seqs} => {dt_rank + 2*d_state, n_seq_tokens, n_seqs}
-        ggml_tensor * x_bcdt = build_lora_mm(model.layers[il].ssm_x, x);
-        cb(x_bcdt, "mamba_bcdt_proj", il);
-
-        // split into dt, B, C
-        const int64_t dt_dim = std::max(64, int(hparams.n_embd / 16));
-        ggml_tensor * B  = ggml_view_3d(ctx0, x_bcdt, d_state, n_seq_tokens, n_seqs, x_bcdt->nb[1], x_bcdt->nb[2], 0);
-        ggml_tensor * C  = ggml_view_3d(ctx0, x_bcdt, d_state, n_seq_tokens, n_seqs, x_bcdt->nb[1], x_bcdt->nb[2],
-                                        ggml_element_size(x_bcdt) * d_state);
-        ggml_tensor * dt = ggml_view_3d(ctx0, x_bcdt, dt_dim, n_seq_tokens, n_seqs, x_bcdt->nb[1], x_bcdt->nb[2],
-                                        ggml_element_size(x_bcdt) * (2 * d_state));
-        cb(B, "mamba_B_raw", il);
-        cb(C, "mamba_C_raw", il);
-        cb(dt, "mamba_dt_raw", il);
-
-        // Apply RMS norm to dt, B, C (PLaMo-2 specific)
-        B  = build_norm(B, model.layers[il].ssm_b_norm, NULL, LLM_NORM_RMS, il);
-        C  = build_norm(C, model.layers[il].ssm_c_norm, NULL, LLM_NORM_RMS, il);
-        dt = build_norm(dt, model.layers[il].ssm_dt_norm, NULL, LLM_NORM_RMS, il);
-        cb(B, "mamba_B_normed", il);
-        cb(C, "mamba_C_normed", il);
-        cb(dt, "mamba_dt_normed", il);
-
-        // dt_proj: {dt_rank, d_inner} @ {dt_rank, n_seq_tokens, n_seqs} => {d_inner, n_seq_tokens, n_seqs}
-        dt = build_lora_mm(model.layers[il].ssm_dt, dt);
-        dt = ggml_add(ctx0, dt, model.layers[il].ssm_dt_b);
-        cb(dt, "mamba_dt_proj", il);
-
-        ggml_tensor * A = ggml_reshape_2d(ctx0, model.layers[il].ssm_a, 1, n_heads);
-        cb(A, "mamba_A", il);
-
-        x = ggml_view_4d(ctx0, x, head_dim, n_heads, n_seq_tokens, n_seqs, head_dim * ggml_element_size(x),
-                         head_dim * n_heads * ggml_element_size(x),
-                         head_dim * n_heads * n_seq_tokens * ggml_element_size(x), 0);
-        B = ggml_view_4d(ctx0, B, d_state, 1, n_seq_tokens, n_seqs, d_state * B->nb[0], B->nb[1], B->nb[2], 0);
-        C = ggml_view_4d(ctx0, C, d_state, 1, n_seq_tokens, n_seqs, d_state * C->nb[0], C->nb[1], C->nb[2], 0);
-
-        // use the states and the indices provided by build_recurrent_state
-        // (this is necessary in order to properly use the states before they are overwritten,
-        //  while avoiding to make unnecessary copies of the states)
-        auto get_ssm_rows = [&](ggml_context * ctx, ggml_tensor * states, ggml_tensor * ids) {
-            ggml_tensor * ssm = ggml_reshape_4d(ctx, states, d_state, head_dim, n_heads, mctx_cur->get_size());
-
-            // Custom operator to optimize the parallel associative scan
-            // as described in the Annex D of the Mamba paper.
-            // => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs}
-            return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids);
-        };
-
-        ggml_tensor * y_ssm = build_rs(inp, ssm_states_all, hparams.n_embd_s(), ubatch.n_seqs, get_ssm_rows);
-        cb(y_ssm, "mamba_ssm_scan", il);
-
-        // store last states
-        ggml_build_forward_expand(
-            gf, ggml_cpy(
-                    ctx0,
-                    ggml_view_1d(ctx0, y_ssm, n_heads * head_dim * d_state * n_seqs,
-                                 n_heads * head_dim * n_seq_tokens * n_seqs * ggml_element_size(y_ssm)),
-                    ggml_view_1d(ctx0, ssm_states_all, n_heads * head_dim * d_state * n_seqs,
-                                 kv_head * n_seqs * n_heads * head_dim * d_state * ggml_element_size(ssm_states_all))));
-        cb(ssm_states_all, "mamba_ssm_states", il);
-
-        ggml_tensor * y = ggml_view_4d(ctx0, y_ssm, head_dim, n_heads, n_seq_tokens, n_seqs,
-                                       head_dim * ggml_element_size(x), head_dim * n_heads * ggml_element_size(x),
-                                       head_dim * n_heads * n_seq_tokens * ggml_element_size(x), 0);
-        cb(y, "mamba_y_view", il);
-
-        // Add D parameter and apply gating with z
-        // {d_inner, n_seq_tokens, n_seqs} * {d_inner} => {d_inner, n_seq_tokens, n_seqs}
-        ggml_tensor * D = ggml_reshape_2d(ctx0, model.layers[il].ssm_d, 1, n_heads);
-        y               = ggml_add(ctx0, y, ggml_mul(ctx0, x, D));
-        cb(y, "mamba_y_add_d", il);
-
-        y = ggml_swiglu_split(ctx0, ggml_cont(ctx0, z), y);
-        cb(y, "mamba_y_swiglu_z", il);
-
-        // out_proj: {d_inner, n_embd} @ {d_inner, n_seq_tokens, n_seqs} => {n_embd, n_seq_tokens, n_seqs}
-        y   = ggml_view_3d(ctx0, y, head_dim * n_heads, n_seq_tokens, n_seqs, y->nb[2], y->nb[3], 0);
-        cur = build_lora_mm(model.layers[il].ssm_out, y);
-        cb(cur, "mamba_out_proj", il);
-    }
-
-    // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}
-    cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs);
-    cb(cur, "mamba_out", il);
-
-    return cur;
-}

data/ext/sources/examples/talk-llama/models/plamo3.cpp

@@ -1,128 +0,0 @@
-#include "models.h"
-
-template <bool iswa>
-llm_build_plamo3<iswa>::llm_build_plamo3(const llama_model & model, const llm_graph_params & params) :
-    llm_graph_context(params) {
-    const int64_t head_dim_q = hparams.n_embd_head_k;
-    const int64_t head_dim_v = hparams.n_embd_head_v;
-
-    ggml_tensor * cur;
-    ggml_tensor * inpL = build_inp_embd(model.tok_embd);
-    ggml_tensor * inp_pos = build_inp_pos();
-
-    using inp_attn_type = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;
-    inp_attn_type * inp_attn = nullptr;
-
-    if constexpr (iswa) {
-        inp_attn = build_attn_inp_kv_iswa();
-    } else {
-        inp_attn = build_attn_inp_kv();
-    }
-
-    ggml_tensor * inp_out_ids = build_inp_out_ids();
-
-    for (int il = 0; il < n_layer; ++il) {
-        ggml_tensor * residual = inpL;
-
-        float freq_base_l  = 0.0f;
-        float freq_scale_l = 0.0f;
-        if constexpr (iswa) {
-            freq_base_l  = model.get_rope_freq_base (cparams, il);
-            freq_scale_l = model.get_rope_freq_scale(cparams, il);
-        } else {
-            freq_base_l  = freq_base;
-            freq_scale_l = freq_scale;
-        }
-
-        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
-        cb(cur, "attn_norm", il);
-
-        ggml_tensor * qkv = build_lora_mm(model.layers[il].wqkv, cur);
-        cb(cur, "wqkv", il);
-
-        const int32_t n_head    = hparams.n_head(il);
-        const int32_t n_head_kv = hparams.n_head_kv(il);
-
-        const int64_t q_offset = 0;
-        const int64_t k_offset = head_dim_q * n_head;
-        const int64_t v_offset = k_offset + head_dim_q * n_head_kv;
-
-        ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, head_dim_q, n_head, n_tokens,
-                head_dim_q * sizeof(float), qkv->nb[1], q_offset * ggml_element_size(qkv));
-        ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, head_dim_q, n_head_kv, n_tokens,
-                head_dim_q * sizeof(float), qkv->nb[1], k_offset * ggml_element_size(qkv));
-        ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, head_dim_v, n_head_kv, n_tokens,
-                head_dim_v * sizeof(float), qkv->nb[1], v_offset * ggml_element_size(qkv));
-
-        cb(Qcur, "Qcur", il);
-        cb(Kcur, "Kcur", il);
-        cb(Vcur, "Vcur", il);
-
-        Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
-        cb(Qcur, "attn_q_norm", il);
-        Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
-        cb(Kcur, "attn_k_norm", il);
-
-        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
-                n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
-                ext_factor, attn_factor, beta_fast, beta_slow);
-        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
-                n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
-                ext_factor, attn_factor, beta_fast, beta_slow);
-
-        const float attn_scale = 1.0f / sqrtf(float(head_dim_q));
-
-        cur = build_attn(inp_attn,
-                model.layers[il].wo, NULL,
-                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, attn_scale, il);
-        cb(cur, "attn_out", il);
-
-        if (il == n_layer - 1 && inp_out_ids) {
-            cur      = ggml_get_rows(ctx0, cur, inp_out_ids);
-            residual = ggml_get_rows(ctx0, residual, inp_out_ids);
-        }
-
-        cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);
-        cb(cur, "attn_post_norm", il);
-
-        cur = ggml_add(ctx0, cur, residual);
-        cb(cur, "attn_residual", il);
-
-        residual = cur;
-
-        cur = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
-        cb(cur, "ffn_norm", il);
-
-        cur = build_ffn(cur,
-                model.layers[il].ffn_up,   NULL, NULL,
-                NULL,                      NULL, NULL,
-                model.layers[il].ffn_down, NULL, NULL,
-                NULL,
-                LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);
-        cb(cur, "ffn_out", il);
-
-        cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);
-        cb(cur, "ffn_post_norm", il);
-
-        cur = ggml_add(ctx0, cur, residual);
-        cb(cur, "ffn_residual", il);
-
-        cur = build_cvec(cur, il);
-        cb(cur, "l_out", il);
-        inpL = cur;
-    }
-
-    cur = inpL;
-
-    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
-    res->t_embd = cur;
-
-    cur = build_lora_mm(model.output, cur);
-    res->t_logits = cur;
-
-    ggml_build_forward_expand(gf, cur);
-}
-
-// Explicit template instantiations
-template struct llm_build_plamo3<false>;
-template struct llm_build_plamo3<true>;

data/ext/sources/examples/talk-llama/models/plm.cpp

@@ -1,168 +0,0 @@
-#include "models.h"
-
-llm_build_plm::llm_build_plm(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
-    const float kq_scale = 1.0f/sqrtf(float(hparams.n_embd_head_k));
-
-    const uint32_t n_embd_head_qk_rope = hparams.n_rot;
-    const uint32_t n_embd_head_qk_nope = hparams.n_embd_head_k - hparams.n_rot;
-    const uint32_t kv_lora_rank = hparams.n_lora_kv;
-
-    ggml_tensor * cur;
-    ggml_tensor * inpL;
-
-    // {n_embd, n_tokens}
-    inpL = build_inp_embd(model.tok_embd);
-
-    // inp_pos - contains the positions
-    ggml_tensor * inp_pos = build_inp_pos();
-
-    auto * inp_attn = build_attn_inp_kv();
-
-    ggml_tensor * inp_out_ids = build_inp_out_ids();
-
-    for (int il = 0; il < n_layer; ++il) {
-        ggml_tensor * inpSA = inpL;
-
-        // norm
-        cur = build_norm(inpL,
-                model.layers[il].attn_norm, NULL,
-                LLM_NORM_RMS, il);
-        cb(cur, "attn_norm", il);
-
-        // self_attention
-        {
-            ggml_tensor * q = NULL;
-            q = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
-            cb(q, "q", il);
-
-            // split into {n_head * n_embd_head_qk_nope, n_tokens}
-            ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
-                    ggml_row_size(q->type, hparams.n_embd_head_k),
-                    ggml_row_size(q->type, hparams.n_embd_head_k * n_head),
-                    0);
-            cb(q_nope, "q_nope", il);
-
-            // and {n_head * n_embd_head_qk_rope, n_tokens}
-            ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
-                    ggml_row_size(q->type, hparams.n_embd_head_k),
-                    ggml_row_size(q->type, hparams.n_embd_head_k * n_head),
-                    ggml_row_size(q->type, n_embd_head_qk_nope));
-            cb(q_pe, "q_pe", il);
-
-            // {n_embd, kv_lora_rank + n_embd_head_qk_rope} * {n_embd, n_tokens} -> {kv_lora_rank + n_embd_head_qk_rope, n_tokens}
-            ggml_tensor * kv_pe_compresseed = ggml_mul_mat(ctx0, model.layers[il].wkv_a_mqa, cur);
-            cb(kv_pe_compresseed, "kv_pe_compresseed", il);
-
-            // split into {kv_lora_rank, n_tokens}
-            ggml_tensor * kv_compressed = ggml_view_2d(ctx0, kv_pe_compresseed, kv_lora_rank, n_tokens,
-                    kv_pe_compresseed->nb[1],
-                    0);
-            cb(kv_compressed, "kv_compressed", il);
-
-            // and {n_embd_head_qk_rope, n_tokens}
-            ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_pe_compresseed, n_embd_head_qk_rope, 1, n_tokens,
-                    kv_pe_compresseed->nb[1],
-                    kv_pe_compresseed->nb[1],
-                    ggml_row_size(kv_pe_compresseed->type, kv_lora_rank));
-            cb(k_pe, "k_pe", il);
-
-            kv_compressed = build_norm(kv_compressed,
-                    model.layers[il].attn_kv_a_norm, NULL,
-                    LLM_NORM_RMS, il);
-            cb(kv_compressed, "kv_compressed", il);
-
-            // {kv_lora_rank, n_head * (n_embd_head_qk_nope + n_embd_head_v)} * {kv_lora_rank, n_tokens} -> {n_head * (n_embd_head_qk_nope + n_embd_head_v), n_tokens}
-            ggml_tensor * kv = ggml_mul_mat(ctx0, model.layers[il].wkv_b, kv_compressed);
-            cb(kv, "kv", il);
-
-            // split into {n_head * n_embd_head_qk_nope, n_tokens}
-            ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens,
-                    ggml_row_size(kv->type, n_embd_head_qk_nope + hparams.n_embd_head_v),
-                    ggml_row_size(kv->type, n_head * (n_embd_head_qk_nope + hparams.n_embd_head_v)),
-                    0);
-            cb(k_nope, "k_nope", il);
-
-            // and {n_head * n_embd_head_v, n_tokens}
-            ggml_tensor * v_states = ggml_view_3d(ctx0, kv, hparams.n_embd_head_v, n_head, n_tokens,
-                    ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)),
-                    ggml_row_size(kv->type, (n_embd_head_qk_nope + hparams.n_embd_head_v)*n_head),
-                    ggml_row_size(kv->type, (n_embd_head_qk_nope)));
-            cb(v_states, "v_states", il);
-
-            v_states = ggml_cont(ctx0, v_states);
-            cb(v_states, "v_states", il);
-
-            v_states = ggml_view_2d(ctx0, v_states, hparams.n_embd_head_v * n_head, n_tokens,
-                    ggml_row_size(kv->type, hparams.n_embd_head_v * n_head),
-                    0);
-            cb(v_states, "v_states", il);
-
-            q_pe = ggml_rope_ext(
-                    ctx0, q_pe, inp_pos, nullptr,
-                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-                    ext_factor, attn_factor, beta_fast, beta_slow
-                    );
-            cb(q_pe, "q_pe", il);
-
-            // shared RoPE key
-            k_pe = ggml_rope_ext(
-                    ctx0, k_pe, inp_pos, nullptr,
-                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-                    ext_factor, attn_factor, beta_fast, beta_slow
-                    );
-            cb(k_pe, "k_pe", il);
-
-            ggml_tensor * q_states = ggml_concat(ctx0, q_nope, q_pe, 0);
-            cb(q_states, "q_states", il);
-
-            ggml_tensor * k_states = ggml_concat(ctx0, k_nope, ggml_repeat(ctx0, k_pe, q_pe), 0);
-            cb(k_states, "k_states", il);
-
-            cur = build_attn(inp_attn,
-                    model.layers[il].wo, NULL,
-                    q_states, k_states, v_states, nullptr, nullptr, nullptr, kq_scale, il);
-        }
-        if (il == n_layer - 1 && inp_out_ids) {
-            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);
-            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
-        }
-        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
-        cb(ffn_inp, "ffn_inp", il);
-
-        cur = build_norm(ffn_inp,
-                model.layers[il].ffn_norm, NULL,
-                LLM_NORM_RMS, il);
-        cb(cur, "ffn_norm", il);
-
-        cur = build_ffn(cur,
-                model.layers[il].ffn_up,   NULL, NULL,
-                NULL, NULL, NULL,
-                model.layers[il].ffn_down, NULL, NULL,
-                NULL,
-                LLM_FFN_RELU_SQR, LLM_FFN_SEQ, il);
-        cb(cur, "ffn_out", il);
-
-        cur = ggml_add(ctx0, cur, ffn_inp);
-
-        cur = build_cvec(cur, il);
-        cb(cur, "l_out", il);
-
-        // input for next layer
-        inpL = cur;
-    }
-    cur = inpL;
-
-    cur = build_norm(cur,
-            model.output_norm, NULL,
-            LLM_NORM_RMS, -1);
-
-    cb(cur, "result_norm", -1);
-    res->t_embd = cur;
-
-    cur = build_lora_mm(model.output, cur);
-
-    cb(cur, "result_output", -1);
-    res->t_logits = cur;
-
-    ggml_build_forward_expand(gf, cur);
-}