toy 0.8.0

data/vendor-patches/0004-cuda-cpy-strided.patch

@@ -0,0 +1,61 @@
+From 6ff957b629d08a2239b316f31ac0c9ee58fa13a7 Mon Sep 17 00:00:00 2001
+From: Ori Pekelman <ori@pekelman.com>
+Date: Mon, 18 May 2026 20:48:40 +0000
+Subject: [PATCH] ggml-cuda: fix cpy into strided view_2d destination
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+ggml_cuda_cpy enabled cpy_scalar_transpose whenever src looked like a
+"compact transpose" (nb01 == elt_size, nb02 == ne00*ne01*elt_size),
+without checking the destination's layout. cpy_scalar_transpose
+indexes dst as `imat*n + (ty+j)*ne00 + tx` — a contiguous write,
+ignoring nb1 on dst.
+
+For the common KV-cache pattern
+  V_slot = view_2d(t_V, ne0=1, ne1=d_head, nb1=max_T*4, offset=pos*4)
+  cpy(t_v_new, V_slot)
+the transpose kernel was triggered because src (the matmul result)
+satisfied the trivial transpose conditions. The dst is a strided
+view (nb1 != ne0*elt_size), so the kernel packed the source's
+d_head bytes contiguously at offset pos*4 instead of distributing
+them one element per max_T-spaced row.
+
+Symptom on Qwen2.5-1.5B inference: V-cache writes silently corrupted
+every decode step; logits diverged from CPU on layer 1 onward.
+
+Fix: also require ggml_is_contiguous(src1) before enabling the
+transpose path. Falls back to the general cpy_scalar kernel which
+correctly uses dst strides nb10/nb11/nb12/nb13.
+
+Repro: tinynn/ab_smoke_cpy_strided_{cpu,cuda}.rb in
+oripekelman/toy_ruby_neural_network@46a61c0 — pre-patch CPU and
+CUDA diverge; post-patch they agree.
+
+Filed upstream as ggml-org/ggml#1497.
+---
+ src/ggml-cuda/cpy.cu | 7 ++++++-
+ 1 file changed, 6 insertions(+), 1 deletion(-)
+
+diff --git a/src/ggml-cuda/cpy.cu b/src/ggml-cuda/cpy.cu
+index d208acf..f1235f4 100644
+--- a/src/ggml-cuda/cpy.cu
++++ b/src/ggml-cuda/cpy.cu
+@@ -404,8 +404,13 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
+     char * src1_ddc = (char *) src1->data;
+ 
+     const bool contiguous_srcs = ggml_is_contiguous(src0) && ggml_is_contiguous(src1);
++    // cpy_scalar_transpose writes dst contiguously (no nb1[i] strides) — it
++    // is only correct when dst itself is contiguous. Without this guard, a
++    // strided dst (e.g. view_2d into a KV-cache slot) gets its source bytes
++    // packed at offset 0 instead of distributed across rows.
+     const bool can_be_transposed = nb01 == (int64_t)ggml_element_size(src0) &&
+-        src0->ne[3] == 1 && nb02 == ne00 * ne01 * (int64_t)ggml_element_size(src0);
++        src0->ne[3] == 1 && nb02 == ne00 * ne01 * (int64_t)ggml_element_size(src0) &&
++        ggml_is_contiguous(src1);
+ 
+     if (src0->type == src1->type && contiguous_srcs) {
+         GGML_ASSERT(ggml_nbytes(src0) == ggml_nbytes(src1));
+-- 
+2.43.0
+

data/vendor-patches/0005-concat-backward.patch

@@ -0,0 +1,36 @@
+diff --git a/src/ggml.c b/src/ggml.c
+index 476c307..df5de39 100644
+--- a/src/ggml.c
++++ b/src/ggml.c
+@@ -6511,6 +6511,31 @@ static void ggml_compute_backward(
+                 ggml_add_or_set(ctx, cgraph, isrc0, ggml_repeat(ctx, grad, src0));
+             }
+         } break;
++        case GGML_OP_CONCAT: {
++            // Backward of concat(a, b, dim) is the two slices of the
++            // upstream grad along that dim. Vendored locally for
++            // SmolLM2 LoRA training (F1.2) — per-head Q outputs are
++            // concat'd along dim=0 before the O projection, so we need
++            // to back-propagate through this op to reach the LoRA
++            // params.  See docs/design/concat-back-patch-2026-05-21.md.
++            const int dim = ((const int32_t *) tensor->op_params)[0];
++            GGML_ASSERT(dim >= 0 && dim < GGML_MAX_DIMS);
++            if (src0_needs_grads) {
++                ggml_add_or_set(ctx, cgraph, isrc0,
++                    ggml_view_4d(ctx, grad,
++                                  src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
++                                  grad->nb[1], grad->nb[2], grad->nb[3],
++                                  /*offset=*/ 0));
++            }
++            if (src1_needs_grads) {
++                size_t offset = (size_t) src0->ne[dim] * grad->nb[dim];
++                ggml_add_or_set(ctx, cgraph, isrc1,
++                    ggml_view_4d(ctx, grad,
++                                  src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],
++                                  grad->nb[1], grad->nb[2], grad->nb[3],
++                                  offset));
++            }
++        } break;
+         case GGML_OP_RMS_NORM: {
+             if (src0_needs_grads) {
+                 float eps;

data/vendor-patches/0006-getrows-back-large-vocab.patch

@@ -0,0 +1,69 @@
+From a0000060000000000000000000000000000000a0 Mon Sep 17 00:00:00 2001
+From: Ori Pekelman <ori@pekelman.com>
+Date: Thu, 21 May 2026 00:00:00 +0000
+Subject: [PATCH] ggml-cuda: chunk get_rows_back launch for vocab > 65535
+
+`ggml_cuda_op_get_rows_back` launches the per-row reduction with
+gridDim.y = nrows_dst (the embedding vocab size in the autograd
+case). CUDA caps gridDim.y at 65535 on every shipping arch, so for
+Qwen2.5 (V=151936) the launch fails with cudaErrorInvalidConfiguration
+("invalid argument"), aborting training the moment a model with
+V > 65535 hits the embedding gradient on the backward pass.
+
+The fix: add a row_offset to the kernel and split the launch into
+chunks of at most 65535 rows each. Each chunk's gridDim.y stays
+inside the limit; row_offset positions the chunk in the destination
+vocab axis.
+
+Verified on toy_ruby_neural_network's F3 path:
+- SmolLM2-135M (V=49152): single chunk, no behavior change.
+- Qwen2.5-1.5B (V=151936): three chunks (65535 + 65535 + 20866),
+  embed gradient training now converges.
+---
+ src/ggml-cuda/getrows.cu | 22 ++++++++++++++++------
+ 1 file changed, 16 insertions(+), 6 deletions(-)
+
+diff --git a/src/ggml-cuda/getrows.cu b/src/ggml-cuda/getrows.cu
+index 3fe3738..74bfd5f 100644
+--- a/src/ggml-cuda/getrows.cu
++++ b/src/ggml-cuda/getrows.cu
+@@ -72,14 +72,15 @@ static __global__ void k_get_rows_float(
+
+ template<typename grad_t, typename dst_t>
+ static __global__ void k_get_rows_back_float(
+-        const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst, const int64_t ncols, const int64_t nrows_grad) {
++        const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst,
++        const int64_t ncols, const int64_t nrows_grad, const int row_offset) {
+     const int col = blockIdx.x*blockDim.x + threadIdx.x;
+
+     if (col >= ncols) {
+         return;
+     }
+
+-    const int dst_row = blockIdx.y*blockDim.y + threadIdx.y;
++    const int dst_row = blockIdx.y*blockDim.y + threadIdx.y + row_offset;
+
+     float sum = 0.0f;
+
+@@ -294,7 +295,16 @@ void ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * d
+
+     const dim3 block_dims(CUDA_GET_ROWS_BACK_BLOCK_SIZE, 1, 1);
+     const int block_num_x = (ne00 + CUDA_GET_ROWS_BACK_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BACK_BLOCK_SIZE;
+-    const dim3 block_nums(block_num_x, ne1, 1);
+-
+-    k_get_rows_back_float<<<block_nums, block_dims, 0, stream>>>(src0_d, src1_d, dst_d, ne00, ne10);
++
++    // CUDA caps gridDim.y at 65535 on every shipping arch. ne1 = vocab
++    // for the embedding-backward use; we chunk the launch when it
++    // exceeds the cap.
++    const int Y_MAX = 65535;
++    int row_offset = 0;
++    while (row_offset < ne1) {
++        const int chunk = (ne1 - row_offset) < Y_MAX ? (ne1 - row_offset) : Y_MAX;
++        const dim3 block_nums(block_num_x, chunk, 1);
++        k_get_rows_back_float<<<block_nums, block_dims, 0, stream>>>(src0_d, src1_d, dst_d, ne00, ne10, row_offset);
++        row_offset += chunk;
++    }
+ }
+--
+2.43.0

data/vendor-patches/0007-gpt2-backward-kernels.patch

@@ -0,0 +1,438 @@
+diff --git a/include/ggml.h b/include/ggml.h
+index f672526..88ecfc1 100644
+--- a/include/ggml.h
++++ b/include/ggml.h
+@@ -583,6 +583,10 @@ extern "C" {
+ 
+         GGML_OP_GLU,
+ 
++        // toy: vendored fused backwards for GPT-2 training (CPU reference)
++        GGML_OP_GELU_BACK,
++        GGML_OP_NORM_BACK,
++
+         GGML_OP_COUNT,
+     };
+ 
+@@ -1196,6 +1200,14 @@ extern "C" {
+             struct ggml_tensor  * a,
+             struct ggml_tensor  * b);
+ 
++    // toy: backward of the tanh-approx gelu (GPT-2 training)
++    // a - dy (gradient w.r.t. gelu output)
++    // b - x  (gelu input from the forward pass)
++    GGML_API struct ggml_tensor * ggml_gelu_back(
++            struct ggml_context * ctx,
++            struct ggml_tensor  * a,
++            struct ggml_tensor  * b);
++
+     // hardswish(x) = x * relu6(x + 3) / 6
+     GGML_API struct ggml_tensor * ggml_hardswish(
+             struct ggml_context * ctx,
+@@ -1411,6 +1423,15 @@ extern "C" {
+             struct ggml_tensor  * b,
+             float                 eps);
+ 
++    // toy: backward of the pure normalize ggml_norm (no affine) for GPT-2 training
++    // a - x  (norm input from the forward pass)
++    // b - dy (gradient w.r.t. the normalize output)
++    GGML_API struct ggml_tensor * ggml_norm_back(
++            struct ggml_context * ctx,
++            struct ggml_tensor  * a,
++            struct ggml_tensor  * b,
++            float                 eps);
++
+     // A: k columns, n rows => [ne03, ne02, n, k]
+     // B: k columns, m rows  (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
+     // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
+diff --git a/src/ggml-alloc.c b/src/ggml-alloc.c
+index 3bda9ab..f989c29 100644
+--- a/src/ggml-alloc.c
++++ b/src/ggml-alloc.c
+@@ -38,6 +38,8 @@ bool ggml_op_can_inplace(enum ggml_op op) {
+         case GGML_OP_ROPE:
+         case GGML_OP_ROPE_BACK:
+         case GGML_OP_SILU_BACK:
++        case GGML_OP_GELU_BACK:
++        case GGML_OP_NORM_BACK:
+         case GGML_OP_RMS_NORM:
+         case GGML_OP_RMS_NORM_BACK:
+         case GGML_OP_SOFT_MAX:
+diff --git a/src/ggml-backend-meta.cpp b/src/ggml-backend-meta.cpp
+index df0f405..f445d60 100644
+--- a/src/ggml-backend-meta.cpp
++++ b/src/ggml-backend-meta.cpp
+@@ -834,10 +834,12 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co
+             case GGML_OP_CONCAT: {
+                 split_state = handle_concat(src_ss);
+             } break;
+-            case GGML_OP_SILU_BACK: {
++            case GGML_OP_SILU_BACK:
++            case GGML_OP_GELU_BACK: {
+                 split_state = handle_generic(src_ss, /*scalar_only =*/ false);
+             } break;
+             case GGML_OP_NORM:
++            case GGML_OP_NORM_BACK:
+             case GGML_OP_RMS_NORM:
+             case GGML_OP_RMS_NORM_BACK:
+             case GGML_OP_GROUP_NORM:
+diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c
+index cd5c61a..f2284f3 100644
+--- a/src/ggml-cpu/ggml-cpu.c
++++ b/src/ggml-cpu/ggml-cpu.c
+@@ -1804,10 +1804,18 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
+             {
+                 ggml_compute_forward_silu_back(params, tensor);
+             } break;
++        case GGML_OP_GELU_BACK:
++            {
++                ggml_compute_forward_gelu_back(params, tensor);
++            } break;
+         case GGML_OP_NORM:
+             {
+                 ggml_compute_forward_norm(params, tensor);
+             } break;
++        case GGML_OP_NORM_BACK:
++            {
++                ggml_compute_forward_norm_back(params, tensor);
++            } break;
+         case GGML_OP_RMS_NORM:
+             {
+                 ggml_compute_forward_rms_norm(params, tensor);
+@@ -2287,9 +2295,11 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
+             }
+             break;
+         case GGML_OP_SILU_BACK:
++        case GGML_OP_GELU_BACK:
+         case GGML_OP_MUL:
+         case GGML_OP_DIV:
+         case GGML_OP_NORM:
++        case GGML_OP_NORM_BACK:
+         case GGML_OP_RMS_NORM:
+         case GGML_OP_RMS_NORM_BACK:
+         case GGML_OP_L2_NORM:
+diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp
+index 7485ba4..0a36422 100644
+--- a/src/ggml-cpu/ops.cpp
++++ b/src/ggml-cpu/ops.cpp
+@@ -2842,6 +2842,70 @@ void ggml_compute_forward_silu_back(
+     }
+ }
+ 
++// ggml_compute_forward_gelu_back
++// toy: backward of the tanh-approx gelu (GPT-2 training). f32-only reference.
++
++static void ggml_compute_forward_gelu_back_f32(
++        const ggml_compute_params * params,
++        ggml_tensor * dst) {
++
++    const ggml_tensor * grad = dst->src[0]; // dy
++    const ggml_tensor * src1 = dst->src[1]; // x
++
++    assert(ggml_is_contiguous_1(grad));
++    assert(ggml_is_contiguous_1(src1));
++    assert(ggml_is_contiguous_1(dst));
++    assert(ggml_are_same_shape(src1, dst));
++    assert(ggml_are_same_shape(src1, grad));
++
++    const int ith = params->ith;
++    const int nth = params->nth;
++
++    const int nc = src1->ne[0];
++    const int nr = ggml_nrows(src1);
++
++    // rows per thread
++    const int dr = (nr + nth - 1)/nth;
++
++    // row range for this thread
++    const int ir0 = dr*ith;
++    const int ir1 = MIN(ir0 + dr, nr);
++
++    for (int i1 = ir0; i1 < ir1; i1++) {
++        ggml_vec_gelu_backward_f32(nc,
++                (float *) ((char *) dst->data  + i1*( dst->nb[1])),
++                (float *) ((char *) src1->data + i1*(src1->nb[1])),
++                (float *) ((char *) grad->data + i1*(grad->nb[1])));
++
++#ifndef NDEBUG
++        for (int k = 0; k < nc; k++) {
++            const float x = ((float *) ((char *) dst->data + i1*( dst->nb[1])))[k];
++            GGML_UNUSED(x);
++            assert(!isnan(x));
++            assert(!isinf(x));
++        }
++#endif // NDEBUG
++    }
++}
++
++void ggml_compute_forward_gelu_back(
++        const ggml_compute_params * params,
++        ggml_tensor * dst) {
++
++    const ggml_tensor * src1 = dst->src[1];
++
++    switch (src1->type) {
++        case GGML_TYPE_F32:
++            {
++                ggml_compute_forward_gelu_back_f32(params, dst);
++            } break;
++        default:
++            {
++                GGML_ABORT("fatal error");
++            }
++    }
++}
++
+ // ggml_compute_forward_reglu
+ 
+ static void ggml_compute_forward_reglu_f32(
+@@ -4003,6 +4067,100 @@ void ggml_compute_forward_rms_norm_back(
+     }
+ }
+ 
++// ggml_compute_forward_norm_back
++// toy: backward of the pure normalize ggml_norm (no affine), GPT-2 training.
++//   y    = (x - mean)/sqrt(var + eps),  xhat = (x - mean)*rstd
++//   dx_i = rstd * ( dy_i - mean(dy) - xhat_i * mean(dy .* xhat) )
++// mean/var convention matches ggml_compute_forward_norm_f32 (mean over ne0,
++// population variance). src[0] = x, src[1] = dy.
++
++static void ggml_compute_forward_norm_back_f32(
++        const ggml_compute_params * params,
++        ggml_tensor * dst) {
++
++    const ggml_tensor * src0 = dst->src[0]; // x  from the forward pass
++    const ggml_tensor * src1 = dst->src[1]; // dy (gradient of the normalize output)
++
++    GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_are_same_shape(src0, src1));
++
++    GGML_ASSERT(src0->nb[0] == sizeof(float));
++    GGML_ASSERT(src1->nb[0] == sizeof(float));
++
++    const int ith = params->ith;
++    const int nth = params->nth;
++
++    GGML_TENSOR_BINARY_OP_LOCALS
++
++    float eps;
++    memcpy(&eps, dst->op_params, sizeof(float));
++    GGML_ASSERT(eps >= 0.0f);
++
++    for (int64_t i03 = 0; i03 < ne03; i03++) {
++        for (int64_t i02 = 0; i02 < ne02; i02++) {
++            for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
++                // src1 (dy) is same shape as src0 (x) => same indices
++                const int64_t i11 = i01;
++                const int64_t i12 = i02;
++                const int64_t i13 = i03;
++
++                const float * x  = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
++                const float * dy = (float *) ((char *) src1->data + i11*nb11 + i12*nb12 + i13*nb13);
++                float       * dx = (float *) ((char *) dst->data  + i01*nb1  + i02*nb2  + i03*nb3);
++
++                // mean(x)
++                ggml_float sum_x = 0.0;
++                for (int64_t i00 = 0; i00 < ne00; i00++) {
++                    sum_x += (ggml_float)x[i00];
++                }
++                const float mean = (float)(sum_x/ne00);
++
++                // var(x) (population), then rstd
++                ggml_float sum_v = 0.0;
++                for (int64_t i00 = 0; i00 < ne00; i00++) {
++                    const float xc = x[i00] - mean;
++                    sum_v += (ggml_float)(xc*xc);
++                }
++                const float var  = (float)(sum_v/ne00);
++                const float rstd = 1.0f/sqrtf(var + eps);
++
++                // mean(dy) and mean(dy .* xhat)
++                ggml_float sum_dy   = 0.0;
++                ggml_float sum_dyxh = 0.0;
++                for (int64_t i00 = 0; i00 < ne00; i00++) {
++                    const float xhat = (x[i00] - mean)*rstd;
++                    sum_dy   += (ggml_float)dy[i00];
++                    sum_dyxh += (ggml_float)(dy[i00]*xhat);
++                }
++                const float mean_dy   = (float)(sum_dy/ne00);
++                const float mean_dyxh = (float)(sum_dyxh/ne00);
++
++                for (int64_t i00 = 0; i00 < ne00; i00++) {
++                    const float xhat = (x[i00] - mean)*rstd;
++                    dx[i00] = rstd*(dy[i00] - mean_dy - xhat*mean_dyxh);
++                }
++            }
++        }
++    }
++}
++
++void ggml_compute_forward_norm_back(
++        const ggml_compute_params * params,
++        ggml_tensor * dst) {
++
++    const ggml_tensor * src0 = dst->src[0];
++
++    switch (src0->type) {
++        case GGML_TYPE_F32:
++            {
++                ggml_compute_forward_norm_back_f32(params, dst);
++            } break;
++        default:
++            {
++                GGML_ABORT("fatal error");
++            }
++    }
++}
++
+ // ggml_compute_forward_group_norm
+ 
+ static void ggml_compute_forward_group_norm_f32(
+diff --git a/src/ggml-cpu/ops.h b/src/ggml-cpu/ops.h
+index 7398e56..552280c 100644
+--- a/src/ggml-cpu/ops.h
++++ b/src/ggml-cpu/ops.h
+@@ -42,7 +42,9 @@ void ggml_compute_forward_repeat(const struct ggml_compute_params * params, stru
+ void ggml_compute_forward_repeat_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+ void ggml_compute_forward_concat(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+ void ggml_compute_forward_silu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
++void ggml_compute_forward_gelu_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+ void ggml_compute_forward_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);
++void ggml_compute_forward_norm_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+ void ggml_compute_forward_rms_norm(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+ void ggml_compute_forward_rms_norm_mul_fused(const struct ggml_compute_params * params, struct ggml_tensor * dst_rms_norm, struct ggml_tensor * dst_mul);
+ void ggml_compute_forward_rms_norm_back(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+diff --git a/src/ggml-cpu/vec.h b/src/ggml-cpu/vec.h
+index bcd68da..772dfe3 100644
+--- a/src/ggml-cpu/vec.h
++++ b/src/ggml-cpu/vec.h
+@@ -1416,6 +1416,24 @@ inline static void ggml_vec_silu_backward_f16(const int n, ggml_fp16_t * dx, con
+     }
+ }
+ 
++// derivative of the tanh-approximation gelu used by ggml_gelu_f32:
++//   gelu(x) = 0.5 x (1 + tanh(g)),  g  = SQRT_2_OVER_PI (x + GELU_COEF_A x^3)
++//   gelu'(x) = 0.5 (1 + t) + 0.5 x (1 - t^2) g',  g' = SQRT_2_OVER_PI (1 + 3 GELU_COEF_A x^2)
++inline static float ggml_gelu_backward_f32(float x, float dy) {
++    const float x2 = x*x;
++    const float g  = SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x2);
++    const float t  = tanhf(g);
++    const float dg = SQRT_2_OVER_PI*(1.0f + 3.0f*GELU_COEF_A*x2);
++    const float dgelu = 0.5f*(1.0f + t) + 0.5f*x*(1.0f - t*t)*dg;
++    return dy*dgelu;
++}
++
++inline static void ggml_vec_gelu_backward_f32(const int n, float * dx, const float * x, const float * dy) {
++    for (int i = 0; i < n; ++i) {
++        dx[i] = ggml_gelu_backward_f32(x[i], dy[i]);
++    }
++}
++
+ inline static void ggml_vec_reglu_f32 (const int n, float * y, const float * x, const float * g) {
+     for (int i = 0; i < n; ++i) {
+         y[i] = (x[i] > 0.f) ? x[i] * g[i] : 0.f;
+diff --git a/src/ggml.c b/src/ggml.c
+index df5de39..24714eb 100644
+--- a/src/ggml.c
++++ b/src/ggml.c
+@@ -1078,9 +1078,12 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
+     "OPT_STEP_SGD",
+ 
+     "GLU",
++
++    "GELU_BACK",
++    "NORM_BACK",
+ };
+ 
+-static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT != 96");
++static_assert(GGML_OP_COUNT == 98, "GGML_OP_COUNT != 98");
+ 
+ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
+     "none",
+@@ -1188,9 +1191,12 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
+     "sgd(x)",
+ 
+     "glu(x)",
++
++    "gelu_back(x)",
++    "norm_back(x)",
+ };
+ 
+-static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT != 96");
++static_assert(GGML_OP_COUNT == 98, "GGML_OP_COUNT != 98");
+ 
+ static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
+ 
+@@ -2832,6 +2838,21 @@ struct ggml_tensor * ggml_silu_back(
+     return result;
+ }
+ 
++// ggml_gelu_back
++
++struct ggml_tensor * ggml_gelu_back(
++        struct ggml_context * ctx,
++        struct ggml_tensor  * a,
++        struct ggml_tensor  * b) {
++    struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
++
++    result->op     = GGML_OP_GELU_BACK;
++    result->src[0] = a; // dy
++    result->src[1] = b; // x
++
++    return result;
++}
++
+ // ggml hardswish
+ 
+ struct ggml_tensor * ggml_hardswish(
+@@ -3161,6 +3182,24 @@ struct ggml_tensor * ggml_rms_norm_back(
+     return result;
+ }
+ 
++// ggml_norm_back
++
++struct ggml_tensor * ggml_norm_back(
++        struct ggml_context * ctx,
++        struct ggml_tensor  * a,
++        struct ggml_tensor  * b,
++        float                 eps) {
++    struct ggml_tensor * result = ggml_dup_tensor(ctx, a);
++
++    ggml_set_op_params(result, &eps, sizeof(eps));
++
++    result->op     = GGML_OP_NORM_BACK;
++    result->src[0] = a; // x
++    result->src[1] = b; // dy
++
++    return result;
++}
++
+ // ggml_group_norm
+ 
+ static struct ggml_tensor * ggml_group_norm_impl(
+@@ -6543,6 +6582,14 @@ static void ggml_compute_backward(
+                 ggml_add_or_set(ctx, cgraph, isrc0, ggml_rms_norm_back(ctx, grad, src0, eps));
+             }
+         } break;
++        case GGML_OP_NORM: {
++            // toy: pure-normalize backward (no affine) for GPT-2 training
++            if (src0_needs_grads) {
++                float eps;
++                memcpy(&eps, tensor->op_params, sizeof(float));
++                ggml_add_or_set(ctx, cgraph, isrc0, ggml_norm_back(ctx, src0, grad, eps));
++            }
++        } break;
+         case GGML_OP_MUL_MAT: {
+             // https://cs231n.github.io/optimization-2/#staged
+             // # forward pass
+@@ -6822,6 +6869,12 @@ static void ggml_compute_backward(
+                         ggml_add_or_set(ctx, cgraph, isrc0, ggml_silu_back(ctx, grad, src0));
+                     }
+                 } break;
++                case GGML_UNARY_OP_GELU: {
++                    // toy: tanh-approx gelu backward (GPT-2 training)
++                    if (src0_needs_grads) {
++                        ggml_add_or_set(ctx, cgraph, isrc0, ggml_gelu_back(ctx, grad, src0));
++                    }
++                } break;
+                 case GGML_UNARY_OP_EXP: {
+                     if (src0_needs_grads) {
+                         ggml_add_or_set(ctx, cgraph, isrc0, ggml_mul(ctx, tensor, grad));

data/vendor-patches/0008-mul-mat-backward-mixed-precision.patch

@@ -0,0 +1,50 @@
+diff --git a/src/ggml.c b/src/ggml.c
+index 476c307..1fc8886 100644
+--- a/src/ggml.c
++++ b/src/ggml.c
+@@ -6556,19 +6628,32 @@ static void ggml_compute_backward(
+                 ggml_add_or_set(ctx, cgraph, isrc0, tmp);
+             }
+             if (src1_needs_grads) {
+-                ggml_add_or_set(ctx, cgraph, isrc1,
+-                        // ggml_mul_mat(ctx,                   // [n,p,qq,rr]
+-                        //     ggml_cont(ctx,                  // [m,n,q1,r1]
+-                        //         ggml_transpose(ctx, src0)), // [m,n,q1,r1]
+-                        //     grad),                          // [m,p,qq,rr]
+-
+-                        // when src0 is bigger than tensor->grad (this is mostly the case in llama),
+-                        // avoid transpose of src0, rather transpose smaller tensor->grad
+-                        // and then use ggml_out_prod
+-                        ggml_out_prod(ctx,      // [n,p,qq,rr]
+-                            src0,               // [n,m,q1,r1]
+-                            ggml_transpose(ctx, // [p,m,qq,rr]
+-                                grad)));        // [m,p,qq,rr]
++                // toy (GH#9, mixed-precision training): the fast path below uses
++                // ggml_out_prod(src0, ...) for the src1 (activation) gradient, but
++                // OUT_PROD only implements an f32 src0 (CPU: ggml-cpu.cpp supports_op;
++                // CUDA: out_prod is f32-only). With f16/bf16 weight storage src0 is
++                // low-precision, so fall back to the algebraically-equivalent
++                // mul_mat(cont(transpose(src0)), grad) form, which MUL_MAT supports
++                // for f16/bf16 src0 + f32 src1 on every backend (and runs on tensor
++                // cores on CUDA). f32 weights keep the original out_prod fast path.
++                if (src0->type != GGML_TYPE_F32) {
++                    // MUL_MAT requires a contiguous src1 (out_prod tolerated a
++                    // transposed/strided grad; mul_mat does not), so cont() grad.
++                    ggml_add_or_set(ctx, cgraph, isrc1,
++                            ggml_mul_mat(ctx,                       // [n,p,qq,rr]
++                                ggml_cont(ctx,                      // [m,n,q1,r1]
++                                    ggml_transpose(ctx, src0)),     // [m,n,q1,r1]
++                                ggml_cont(ctx, grad)));             // [m,p,qq,rr]
++                } else {
++                    ggml_add_or_set(ctx, cgraph, isrc1,
++                            // when src0 is bigger than tensor->grad (this is mostly the case in llama),
++                            // avoid transpose of src0, rather transpose smaller tensor->grad
++                            // and then use ggml_out_prod
++                            ggml_out_prod(ctx,      // [n,p,qq,rr]
++                                src0,               // [n,m,q1,r1]
++                                ggml_transpose(ctx, // [p,m,qq,rr]
++                                    grad)));        // [m,p,qq,rr]
++                }
+             }
+         } break;
+         case GGML_OP_SCALE: {

data/vendor-patches/0009-sched-unsupported-node-diagnostic.patch

@@ -0,0 +1,26 @@
+diff --git a/src/ggml-backend.cpp b/src/ggml-backend.cpp
+index 4e36909..98fdf95 100644
+--- a/src/ggml-backend.cpp
++++ b/src/ggml-backend.cpp
+@@ -1239,6 +1239,21 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
+         for (int b = 0; b < sched->n_backends && *cur_backend_id == -1; b++) {
+             ggml_backend_sched_set_if_supported(sched, node, b, cur_backend_id);
+         }
++        if (*cur_backend_id == -1) {
++            // toy: fail loud, not bare-assert. No backend's supports_op() accepts
++            // this node — usually a dtype the op's kernel doesn't implement (e.g.
++            // a backward OUT_PROD/MUL_MAT on an f16/bf16 weight). Dump the node and
++            // its sources so the offending op+dtype is visible without a rebuild.
++            fprintf(stderr, "ggml_backend_sched: no backend supports node '%s' op=%s type=%s\n",
++                    node->name, ggml_op_name(node->op), ggml_type_name(node->type));
++            for (int j = 0; j < GGML_MAX_SRC; j++) {
++                if (node->src[j]) {
++                    fprintf(stderr, "  src[%d]: '%s' op=%s type=%s\n", j,
++                            node->src[j]->name, ggml_op_name(node->src[j]->op),
++                            ggml_type_name(node->src[j]->type));
++                }
++            }
++        }
+         GGML_ASSERT(*cur_backend_id != -1);
+     }
+