toy 0.8.0

data/lib/toy/io/toy_events.rb

@@ -0,0 +1,56 @@
+# lib/toy/io/toy_events.rb — Toy::Events, shared toy/v1 run_start provenance.
+#
+# WHY THIS EXISTS. Every runner/example run_start event carried the SAME three
+# provenance sub-objects in the SAME order — host{name,os,arch}, backend{kind},
+# git{sha,branch} — built inline (~12 lines + a Toy::Git.read) in 11 places. This
+# folds that into one call:
+#
+#   rs = SpinelKit::Json::Builder.new
+#   rs.add_str("kind", "run_start"); rs.add_str("schema", "toy/v1")
+#   rs.add_num("t", now); rs.add_str("started_at", iso); rs.add_str("run_id", rid)
+#   rs.add_str("phase", "train")
+#   Toy::Events.add_provenance(rs, host_name, host_os, host_arch, backend_kind)
+# (git{} comes from SpinelKit::Git.read — toy#44.)
+#   # ... caller appends model{} / config{} / schedule{} ...
+#   TinyNN.tnn_events_emit(rs.dump)
+#
+# The host/backend values are PASSED IN: they come from the backend-specific
+# TinyNN / TinyNNCuda / TinyNNMetal module the caller is bound to, which a shared
+# helper can't name. git is backend-agnostic (pure file read) so it's read here.
+# Key order is preserved exactly → byte-identical events for ASCII-clean values.
+#
+# Pure Ruby (no FFI) → compiles under Spinel, no mirror. Spinel naming discipline:
+# the module method's params carry an `ev_` prefix so they can't widen an
+# unrelated host_name/backend_kind elsewhere (landmines #12/#16).
+require_relative "../../../vendor/spinel/spinel_kit/lib/spinel_kit/json_builder"
+# SpinelKit::Git (toy#44) — git provenance, consolidated out of the former
+# lib/toy/io/toy_git.rb (identical .git/HEAD reader; plain sha/branch accessors
+# now that the Spinel gi_-prefix inference bug is fixed upstream). Vendored at
+# build time via `make vendor-tep` (spinel-compat). Required by-path (NOT via
+# vendor/spinel/deps) so the tep-free runners stay tep-free — deps.rb pulls tep
+# too. toy_events is a runner/example dep, never part of toy.rb's compute
+# surface, so this relative reach into vendor/spinel is build-local (same as
+# serve.rb's vendor/spinel/deps require).
+require_relative "../../../vendor/spinel/spinel_kit/lib/spinel_kit/git"
+
+module Toy
+  module Events
+    # Append host{}, backend{}, git{} (the canonical run_start provenance) to an
+    # in-progress SpinelKit::Json::Builder run_start builder, in order. Mutates ev_rs.
+    def self.add_provenance(ev_rs, ev_host_name, ev_host_os, ev_host_arch, ev_backend_kind)
+      ev_host = SpinelKit::Json::Builder.new
+      ev_host.add_str("name", ev_host_name)
+      ev_host.add_str("os",   ev_host_os)
+      ev_host.add_str("arch", ev_host_arch)
+      ev_rs.add_obj("host", ev_host)
+      ev_backend = SpinelKit::Json::Builder.new
+      ev_backend.add_str("kind", ev_backend_kind)
+      ev_rs.add_obj("backend", ev_backend)
+      ev_gp = SpinelKit::Git.read
+      ev_git = SpinelKit::Json::Builder.new
+      ev_git.add_str("sha",    ev_gp.sha)
+      ev_git.add_str("branch", ev_gp.branch)
+      ev_rs.add_obj("git", ev_git)
+    end
+  end
+end

data/lib/toy/io/toy_image_loader.rb

@@ -0,0 +1,48 @@
+# E1.5 / GH#13 — streaming image-dataset loader for ViT training.
+#
+# Reads pre-flattened patches + labels written by
+# prep/preprocess_images.py:
+#
+#   images.bin   raw f32, n_images × (patch_flat × n_patches)
+#   labels.bin   raw i32, n_images
+#
+# One record per call: read_image returns the patches as Array<Float>
+# (patch_flat × n_patches), and read_label returns the class index.
+# Caller maintains its own image-index cursor (wrap with modulo).
+#
+# Cheap: one syscall per training step at smoke shape. For real
+# Flowers-102 / Cars / CIFAR scale (~2k–8k images × 96×96 × 3) the
+# per-image record is ~30 KB; throughput is way more than enough for
+# CPU training. If we ever need batch>1 throughput, the right move is
+# mmap the file once + slice — a small follow-up.
+
+module ToyImageLoader
+  F32_BYTES = 4
+  I32_BYTES = 4
+
+  # Read the patches for image at `image_idx`. `record_floats` =
+  # patch_flat * n_patches (the number of f32s per image, written by
+  # the preprocessor).
+  def self.read_image(images_path, image_idx, record_floats)
+    buf = Array.new(record_floats, 0.0)
+    byte_offset = image_idx * record_floats * F32_BYTES
+    got = TinyNN.tnn_read_f32_file(images_path, byte_offset, record_floats, buf)
+    if got != record_floats
+      puts "WARN: ToyImageLoader.read_image short read " + got.to_s +
+           "/" + record_floats.to_s + " at idx=" + image_idx.to_s
+    end
+    buf
+  end
+
+  # Read the label (class index) for image `image_idx`. Returns Integer.
+  def self.read_label(labels_path, image_idx)
+    buf = [0]
+    byte_offset = image_idx * I32_BYTES
+    got = TinyNN.tnn_read_i32_file(labels_path, byte_offset, 1, buf)
+    if got != 1
+      puts "WARN: ToyImageLoader.read_label short read at idx=" + image_idx.to_s
+      return -1
+    end
+    buf[0]
+  end
+end

data/lib/toy/llm/adamw.rb

@@ -0,0 +1,169 @@
+# lib/toy/llm/adamw.rb — Toy::AdamW: NAMED AdamW hyper-parameters.
+#
+# A pure-Ruby value object that NAMES the 7 slots of the anonymous
+# `m_hp` Mat(1,7) that every training recipe hands to the FFI
+# optimizer step. It builds the SAME Mat the runners used to build by
+# hand, slot-for-slot, byte-identically. NO TinyNN / FFI calls — it
+# only touches Mat (already loaded transitively by every caller via
+# require_relative "../../toy"), so this file adds NO require and is
+# SHARED (not mirrored) across the CPU / CUDA / Metal runners.
+#
+# Spinel hygiene (mirrors RopeScaling / SmolLM2Config in
+# lib/toy_smollm2.rb): a PLAIN class (NO Struct.new), initialize takes
+# NO args and sets every default in the BODY (NO kwargs, NO default
+# args — landmine #4 widens RbVal across the compiled program), named
+# attr_accessor, NO #{} interpolation.
+#
+# ── LOUD FINDING: slots 5/6 mean DIFFERENT things per recipe family ──
+# The SAME m_hp slot index feeds DIFFERENT quantities into the
+# underlying FFI optimizer-step graph, depending on which training
+# graph family consumes it:
+#
+#   from-scratch / warm-start / vit graphs read
+#       flat[5] = beta1 (0.9), flat[6] = beta2 (0.95)
+#   as CONSTANT betas — literally a SECOND copy of the betas already in
+#   slots 1/2. Bias correction (if any) happens INSIDE those graphs.
+#
+#   the lora graph reads
+#       flat[5] = 1/(1 - beta1^t), flat[6] = 1/(1 - beta2^t)
+#   as PER-STEP BIAS-CORRECTION DENOMINATORS (and beta2 = 0.999, NOT
+#   0.95). Bias correction is computed Ruby-side and passed IN.
+#
+# This is a GENUINE inconsistency in how the two FFI training-graph
+# families interpret slots 5/6 — NOT a bug we get to "fix" here.
+# train.rb:306-307 warns that using lora-style hp in the from-scratch
+# graph BREAKS the byte gate. So `beta2` is a REAL field with no single
+# correct default, and `bias_correct` selects the slot-5/6 convention.
+# DO NOT unify the two conventions: that would change behaviour and
+# break byte-exactness. (Worth an upstream issue: the two graphs should
+# agree on what slots 5/6 carry. Flagged, not silently smoothed over.)
+
+module Toy
+  class AdamW
+    # EXPLICIT MODE (toy#64 item 4) — names which slot-5/6 convention
+    # this hp object feeds (the loud finding above). 0 = unset: hp()
+    # FAILS LOUD until the caller declares a mode, because feeding the
+    # wrong convention into a training graph silently changes numerics
+    # (train.rb's gate breaks with lora-style hp). Use the factories:
+    #   Toy::AdamW.for_from_scratch  — slots 5/6 = constant betas
+    #                                  (from-scratch / warm-start / vit
+    #                                  graph family; bias_correct=false)
+    #   Toy::AdamW.for_lora          — slots 5/6 = 1/(1-beta^t) per-step
+    #                                  bias correction, beta2=0.999
+    #                                  (lora-family graphs — ALSO the
+    #                                  gpt2 + full-finetune graphs,
+    #                                  which read the lora convention;
+    #                                  bias_correct=true)
+    # or set `mode` explicitly after .new. The mode names the HP-SLOT
+    # CONVENTION (graph family), not the recipe name.
+    MODE_UNSET        = 0
+    MODE_FROM_SCRATCH = 1
+    MODE_LORA         = 2
+
+    attr_accessor :lr, :beta1, :beta2, :eps, :weight_decay,
+                  :bias_correct, :mode
+
+    # FROM-SCRATCH defaults — the most common / blessed case. All set
+    # in the body (NO kwargs / default args — Spinel landmine #4).
+    # Callers that differ (lora) set the differing fields AFTER .new via
+    # the accessors (or use the .for_from_scratch / .for_lora factories,
+    # which also set the REQUIRED mode). warm / vit set adamw.lr per
+    # step. mode starts UNSET: hp() raises until it is declared.
+    def initialize
+      @lr           = 0.001
+      @beta1        = 0.9
+      @beta2        = 0.95
+      @eps          = 1.0e-8
+      @weight_decay = 0.0
+      @bias_correct = false
+      @mode         = MODE_UNSET
+    end
+
+    # The from-scratch/warm/vit-family factory: defaults + mode set.
+    # Slots 5/6 = constant betas (bias_correct=false).
+    def self.for_from_scratch
+      a = Toy::AdamW.new
+      a.mode = MODE_FROM_SCRATCH
+      a
+    end
+
+    # The lora-family factory (ALSO the gpt2 + full-finetune graphs):
+    # beta2=0.999 + per-step bias correction (slots 5/6 = 1/(1-beta^t)).
+    def self.for_lora
+      a = Toy::AdamW.new
+      a.beta2        = 0.999
+      a.bias_correct = true
+      a.mode = MODE_LORA
+      a
+    end
+
+    # Build the Mat(1,7) the recipe hands to step!. Byte-identical to
+    # the hand-built m_hp the runners used to fill inline.
+    #
+    # `step` is ignored when bias_correct == false (from-scratch / warm
+    # / vit pass slots 5/6 = constant betas, so hp(step) is step-agnostic
+    # in that mode). When bias_correct == true (lora) it is the CALLER's
+    # 1-indexed step (>= 1); the `** step.to_f` reproduces
+    # train_lora.rb:174-175 / smoke_recipe_lora.rb:88-89 VERBATIM.
+    #
+    # FAILS LOUD (toy#64 item 4) when mode is unset, or when mode and
+    # bias_correct disagree — both are silent-numerics traps (the
+    # slot-5/6 dual meaning above).
+    def hp(step)
+      if @mode == MODE_UNSET
+        raise "Toy::AdamW#hp: mode unset — declare which slot-5/6 " +
+              "convention this hp feeds (use Toy::AdamW.for_from_scratch " +
+              "or Toy::AdamW.for_lora, or set adamw.mode)"
+      end
+      if @mode == MODE_FROM_SCRATCH && @bias_correct
+        raise "Toy::AdamW#hp: mode=from_scratch but bias_correct=true — " +
+              "the from-scratch/warm/vit graphs read slots 5/6 as " +
+              "CONSTANT BETAS; lora-style hp breaks their byte gates"
+      end
+      if @mode == MODE_LORA && !@bias_correct
+        raise "Toy::AdamW#hp: mode=lora but bias_correct=false — the " +
+              "lora-family graphs read slots 5/6 as per-step " +
+              "1/(1-beta^t) bias-correction denominators"
+      end
+      m = Mat.new(1, 7)            # Mat.new zero-fills @flat (transformer.rb:74)
+      m.flat[0] = @lr
+      m.flat[1] = @beta1
+      m.flat[2] = @beta2
+      m.flat[3] = @eps
+      m.flat[4] = @weight_decay
+      # SLOT 5/6 DUAL MEANING (see the loud finding at the top of this
+      # file): bias_correct selects which of the two FFI conventions we
+      # feed. Do NOT unify — the from-scratch/warm/vit graphs read these
+      # as constant betas; the lora graph reads them as 1/(1-beta^t).
+      if @bias_correct
+        m.flat[5] = 1.0 / (1.0 - (@beta1 ** step.to_f))
+        m.flat[6] = 1.0 / (1.0 - (@beta2 ** step.to_f))
+      else
+        m.flat[5] = @beta1
+        m.flat[6] = @beta2
+      end
+      m
+    end
+
+    # MODE-AWARE per-step hp builder (toy#73 item 5): takes the CALLER's
+    # 0-INDEXED loop step (the convention every recipe loop already uses
+    # for its `step == 0` is_first branch) and applies the mode's step
+    # convention internally:
+    #
+    #   MODE_FROM_SCRATCH — step-agnostic (slots 5/6 = constant betas);
+    #                       hp_for_step(k) == hp(k).
+    #   MODE_LORA         — the lora-family graphs want a 1-INDEXED t in
+    #                       1/(1-beta^t); hp_for_step(k) == hp(k + 1).
+    #
+    # This removes the lora paths' off-by-one ceremony (a 1-indexed loop
+    # carried solely to feed hp(step)). Byte-identical to the hp calls it
+    # replaces. The #64 mode guard STAYS: both arms delegate to hp, so
+    # mode-unset / mode-vs-bias_correct mismatch still fail loud.
+    def hp_for_step(step)
+      if @mode == MODE_LORA
+        return hp(step + 1)
+      end
+      hp(step)
+    end
+  end
+end

data/lib/toy/llm/archs/llama_arch.rb

@@ -0,0 +1,233 @@
+# lib/toy/llm/archs/llama_arch.rb — L3 arch: the llama/qwen-family
+# sequence-mode forward orchestration (token-embed get_rows → optional
+# projection-lens matmul → stacked L2 transformer blocks → final RMSNorm
+# → tied/untied logits matmul).
+#
+# Extracted from lib/llama_seq_forward_ffi.rb (P2.5). This is the
+# MINIMAL faithful lift of Toy::LLM::Engine::LlamaSeqEngine#build_forward_in_current_ctx:
+# the orchestration body is moved VERBATIM (op order unchanged →
+# bit-identical graph). The arch now OWNS the arch-level persistent
+# tensor handles (token_embed, final_norm_gamma, output, w_proj, the
+# blocks array); the cache's four realize paths still ALLOCATE and
+# ASSIGN them through cache delegators (exactly the L2 pattern: the
+# block owns its weights, the cache realize paths allocate+assign them).
+#
+# DIVERGENCE from archs/README sketch: build_forward takes NO cfg and
+# does NO per_layer/with_hyper/build_initial_state/learned-position-
+# embedding. Those are forward-looking sketch only. Reality:
+# positions/RoPE live on the L2 block via the shared read-only
+# TransformerBlockCtx; one ctx is shared across all blocks; the LM head
+# is tied/untied via @seq_has_untied_output; an optional E2.3
+# projection-lens matmul sits between embed and blocks. This is the
+# minimal faithful lift of Toy::LLM::Engine::LlamaSeqEngine#build_forward_in_current_ctx
+# (P2.5, orchestration only).
+#
+# Spinel hygiene: NEVER Struct.new (landmine #16 / matz/spinel#1043) —
+# LlamaArchForwardOut is a hand-written plain class with an explicit
+# positional ctor; LlamaArch#initialize takes NO args and has NO
+# default-arg ctor (default-arg poisoning, landmine #4). No
+# Card/step_bind/FFI :str args at class load (step_bind :str landmine
+# 2026-05-28). Member names keep the VERBATIM `@t_seq_*` / `@seq_*`
+# prefixes from the cache for type-isolation and so cache-side
+# realize/train/decode walkers keep working by accessor name.
+#
+# This file does NOT `require_relative "tinynn"`: the loading module
+# (lib/llama_seq_forward_ffi.rb) already loads the correct backend's
+# TinyNN before requiring this arch, exactly like the L1 primitives and
+# the L2 block. The mirror generator picks the backend via the
+# monolith's require rewrite.
+
+module Toy; module LLM; module Archs
+  # Carries the three per-graph OUTPUT handles back to the cache caller,
+  # which spreads them onto its OWN ivars so every existing downstream
+  # reader (@t_seq_logits accessor, build_training_step CE-loss consumer,
+  # examples/06 fcache.t_seq_logits) is untouched.
+  #
+  # Hand-written plain class with a positional ctor — NEVER Struct.new
+  # (landmine #16 / matz/spinel#1043): a Struct's synthesized accessors
+  # would unify across modules and mis-compile unrelated callers. The
+  # member names reuse the cache's `t_seq_*` prefixes (type-isolated, no
+  # collision, same rationale as TransformerBlockCtx). Carries values,
+  # no behavior.
+  class LlamaArchForwardOut
+    attr_accessor :t_seq_x_embed, :t_seq_x_final, :t_seq_logits
+
+    def initialize(t_seq_x_embed, t_seq_x_final, t_seq_logits)
+      @t_seq_x_embed = t_seq_x_embed
+      @t_seq_x_final = t_seq_x_final
+      @t_seq_logits  = t_seq_logits
+    end
+  end
+
+  # The llama-family sequence-mode arch. Owns the arch-level persistent
+  # handles (the cache realize paths allocate+assign them via cache
+  # delegators). Field names are UNCHANGED from the former cache ivars so
+  # the cache-side realize / train / decode / tap walkers keep working by
+  # accessor name.
+  class LlamaArch
+    attr_accessor :t_seq_token_embed, :t_seq_final_norm_gamma, :t_seq_output,
+                  :t_seq_w_proj, :seq_blocks_ffi,
+                  # Orchestration-gating carriers — bare cache ivars with
+                  # no accessor before P2.5. The lens-branch guard reads
+                  # seq_donor_d_in; the shared ctx reads seq_rope_cfg.
+                  # The cache wrapper sets both from the realize-set
+                  # values before build_forward runs.
+                  :seq_donor_d_in, :seq_rope_cfg
+
+    def initialize
+      @t_seq_token_embed      = TinyNN.tnn_null_ptr
+      @t_seq_final_norm_gamma = TinyNN.tnn_null_ptr
+      @t_seq_output           = TinyNN.tnn_null_ptr
+      @t_seq_w_proj           = TinyNN.tnn_null_ptr
+      # Seed with one block — matches the former cache init (L112).
+      @seq_blocks_ffi         = [Toy::LLM::Blocks::TransformerBlock.new]
+      @seq_donor_d_in         = 0
+      # The cache overwrites seq_rope_cfg with the real RoPE::Cfg before
+      # build_forward runs (each realize prologue rebuilds it).
+      @seq_rope_cfg           = TinyNN.tnn_null_ptr
+    end
+
+    # Reset @seq_blocks_ffi and fill it with exactly n_layers fresh
+    # TransformerBlocks. The four cache realize paths each ran this
+    # identical loop verbatim (P2.6 Step 2) — seed one block, then push
+    # n_layers-1 more — so the SHAPE here matches the former cache loop
+    # byte-for-byte (length == n_layers; first element is a fresh block,
+    # exactly like the cache's `[TransformerBlock.new]` seed). The arch
+    # already OWNS this array (ctor seeds it with one block at L83) and
+    # already constructs TransformerBlock.new there, so no new class /
+    # Struct / FFI :str at class load. Each realize path now calls this
+    # via the cache's seq_blocks_ffi delegator chain (self.seq_arch).
+    def seed_blocks!(n_layers)
+      @seq_blocks_ffi = [Toy::LLM::Blocks::TransformerBlock.new]
+      li_init = 1
+      while li_init < n_layers
+        @seq_blocks_ffi.push(Toy::LLM::Blocks::TransformerBlock.new)
+        li_init = li_init + 1
+      end
+    end
+
+    # Allocate the three arch-owned PERSISTENT global tensors from the
+    # mmap'd GGUF: token_embd.weight (2d, native type), output_norm.weight
+    # (1d f32), and — when untied — output.weight (2d, native type). The
+    # cache's realize_for_mmap formerly ran this block inline (P2.6 pass-2
+    # Step 1); it is moved VERBATIM here (same FFI primitives, same
+    # find_index/file_offset/type LITERAL string lookups at runtime, same
+    # untied conditional which the GGUF round-trip gate exercises true).
+    # The arch already OWNS these accessors (L68), so no new class / Struct
+    # / FFI :str at class load. Called ONLY from realize_for_mmap — the
+    # random_init globals and full_finetune's else-branch globals are
+    # structurally different and are NOT routed through this helper.
+    # Mirrors the seed_blocks! / alloc_trainable_f32_weights! precedents.
+    def load_globals_from_gguf_mmap!(sess, gguf_handle, vocab, d_model, untied)
+      eidx = TinyNN.tnn_gguf_find_index(gguf_handle, "token_embd.weight")
+      eoff = TinyNN.tnn_gguf_tensor_file_offset(gguf_handle, eidx)
+      etyp = TinyNN.tnn_gguf_tensor_type(gguf_handle, eidx)
+      @t_seq_token_embed = TinyNN.tnn_input_2d_persistent_mmap(sess,
+                             vocab, d_model, etyp, eoff)
+
+      fnidx = TinyNN.tnn_gguf_find_index(gguf_handle, "output_norm.weight")
+      fnoff = TinyNN.tnn_gguf_tensor_file_offset(gguf_handle, fnidx)
+      @t_seq_final_norm_gamma = TinyNN.tnn_input_1d_persistent_mmap(sess,
+                                  d_model, 0, fnoff)
+
+      if untied
+        oidx = TinyNN.tnn_gguf_find_index(gguf_handle, "output.weight")
+        ooff = TinyNN.tnn_gguf_tensor_file_offset(gguf_handle, oidx)
+        otyp = TinyNN.tnn_gguf_tensor_type(gguf_handle, oidx)
+        @t_seq_output = TinyNN.tnn_input_2d_persistent_mmap(sess,
+                          vocab, d_model, otyp, ooff)
+      end
+    end
+
+    # P2-finish — the RANDOM-INIT (+ projection-lens) trainable-F32 GLOBAL alloc,
+    # lifted VERBATIM from Toy::LLM::Engine::LlamaSeqEngine#realize_for_random_init
+    # (alloc + ft_add_global / ft_name_last_global ORDER unchanged → bit-identical
+    # graph; gated by train_gate from-scratch + smoke_projection_lens). The arch
+    # already OWNS these handles (the load_globals_from_gguf_mmap! precedent); the
+    # engine's @ft_globals_* recorders + the frozen-embed :str namer are back-called
+    # through `cache` (the tnn_tensor_set_name :str FFI stays on the cache realize
+    # runtime path — same discipline as ft_name_last / lora_name_q!). donor_d_in>0 =
+    # projection lens (frozen donor-width embed + trainable lens.proj); 0 = standard.
+    def alloc_globals_trainable_f32!(sess, cache, vocab, d_model, donor_d_in, untied)
+      if donor_d_in > 0
+        self.t_seq_token_embed = TinyNN.tnn_input_2d_f32_persistent(sess, vocab, donor_d_in)
+        cache.name_global!(self.t_seq_token_embed, "token_embd.weight")
+        self.t_seq_w_proj = TinyNN.tnn_input_2d_f32_persistent(sess, d_model, donor_d_in)
+        cache.ft_add_global_2d(self.t_seq_w_proj, d_model, donor_d_in)
+        cache.ft_name_last_global("lens.proj.weight")
+      else
+        self.t_seq_token_embed = TinyNN.tnn_input_2d_f32_persistent(sess, vocab, d_model)
+        cache.ft_add_global_2d(self.t_seq_token_embed, vocab, d_model)
+        cache.ft_name_last_global("token_embd.weight")
+      end
+
+      self.t_seq_final_norm_gamma = TinyNN.tnn_input_1d_f32_persistent(sess, d_model)
+      cache.ft_add_global_1d(self.t_seq_final_norm_gamma)
+      cache.ft_name_last_global("output_norm.weight")
+
+      if untied
+        self.t_seq_output = TinyNN.tnn_input_2d_f32_persistent(sess, vocab, d_model)
+        cache.ft_add_global_2d(self.t_seq_output, vocab, d_model)
+        cache.ft_name_last_global("output.weight")
+      end
+    end
+
+    # SEQ-MODE forward orchestration. The per-graph INPUT handles
+    # (token_ids, positions) are ALLOCATED BY THE CACHE before this call
+    # (cache-owned graph I/O, read by forward() and the uploaders) and
+    # passed in; ditto t_rope_freq_factors and t_attn_mask. The arch
+    # builds: get_rows(token_embed, token_ids) → x_embed (tap), optional
+    # projection-lens matmul(w_proj, x_embed) when seq_donor_d_in>0 (tap),
+    # the shared TransformerBlockCtx built ONCE, the block-stacking loop,
+    # final RMSNorm (tap), tied/untied logits matmul (tap). Returns the
+    # three per-graph output handles in a LlamaArchForwardOut.
+    def build_forward(sess, t_token_ids, t_positions, t_rope_freq_factors,
+                      t_attn_mask, seq_eps, seq_d_head, seq_n_kv, seq_n_heads,
+                      seq_group_size, seq_has_qkv_bias, seq_weight_dtype,
+                      seq_lora_q_enabled, seq_t, seq_b, seq_n_layers,
+                      seq_has_untied_output)
+      eps   = seq_eps
+      scale = 1.0 / Math.sqrt(seq_d_head.to_f)
+
+      # Per-forward block context: the 14 config/handle values the block
+      # body reads. Positional class (no keyword_init) — matches the
+      # TransformerBlockCtx member order exactly. Built once before the
+      # block-stacking loop; shared (read-only) across all blocks.
+      ctx = Toy::LLM::Blocks::TransformerBlockCtx.new(
+        scale, eps, seq_n_kv, seq_n_heads, seq_group_size,
+        seq_has_qkv_bias, seq_weight_dtype, seq_lora_q_enabled,
+        t_positions, t_rope_freq_factors, self.seq_rope_cfg,
+        seq_t, seq_b, t_attn_mask)
+
+      x_embed = TinyNN.tnn_get_rows(sess, self.t_seq_token_embed, t_token_ids)
+      TinyNN.tnn_set_output(x_embed)
+
+      # E2.3 — projection lens. ggml matmul(W, x) with W=[donor_d_in, d_model]
+      # and x=[donor_d_in, T] gives [d_model, T] (contraction on ne[0]).
+      if self.seq_donor_d_in > 0
+        t_proj = TinyNN.tnn_matmul(sess, self.t_seq_w_proj, x_embed)
+        TinyNN.tnn_set_output(t_proj)
+        t_cur = t_proj
+      else
+        t_cur = x_embed
+      end
+      li_g = 0
+      while li_g < seq_n_layers
+        t_cur = self.seq_blocks_ffi[li_g].build_forward(sess, t_cur, ctx)
+        li_g = li_g + 1
+      end
+
+      x_final = Toy::LLM::Primitives::RMSNorm.build(sess, t_cur, self.t_seq_final_norm_gamma, eps)
+      TinyNN.tnn_set_output(x_final)
+
+      if seq_has_untied_output
+        logits = TinyNN.tnn_matmul(sess, self.t_seq_output, x_final)
+      else
+        logits = TinyNN.tnn_matmul(sess, self.t_seq_token_embed, x_final)
+      end
+      TinyNN.tnn_set_output(logits)
+
+      Toy::LLM::Archs::LlamaArchForwardOut.new(x_embed, x_final, logits)
+    end
+  end
+end; end; end