toy 0.8.0

data/lib/toy/run/eval_lmc.rb

@@ -0,0 +1,334 @@
+# lib/toy/run/eval_lmc.rb — Spinel-compiled LMC (Linear Mode Connectivity)
+# COMPUTE runner. Given two toy checkpoints A + B, evaluates loss along the
+# interpolation θ_α = (1-α)·θ_A + α·θ_B on a fixed sequence, emits one `eval`
+# event per α (kind="eval", name="lmc", extra.alpha=α), and prints a
+# deterministic α→loss curve to STDOUT.
+#
+# This is the lib-side home of `toy eval lmc`'s compute, mirroring the
+# CRuby→runner COMPUTE BRIDGE of the other eval/infer/train runners. The CRuby
+# CLI shell (lib/toy/core/cli/eval.rb) cannot compute in-process — every
+# ffi_lib-bearing lib crashes under MRI — so it locates the toy root, builds
+# this runner (`make libexec/toy-eval-lmc`), and shells out to it via Open3
+# with a CONTROLLED ENV.
+#
+# It is a FUSED-FORMAT-AWARE port of examples/legacy/08_lmc.rb. 08_lmc predates the
+# #3 fused-llama checkpoint format; `toy train` now writes a STANDARD FUSED
+# llama GGUF (projection-lens folded + per-head attention FUSED to
+# blk.N.attn_q.weight at write time). The session realized here is the
+# random-init per-HEAD layout (blk.N.attn_q.head_H.weight), so attention is
+# blended by SLICING the H-th block out of the fused GGUF tensor. Non-attention
+# tensors match the GGUF name directly.
+#
+# CONTRACT (read from ENV only — lib-vs-example scope, no experiment config
+# baked in):
+#   LMC_A, LMC_B   — required .gguf paths (the two checkpoints to interpolate)
+#   LMC_ALPHAS     — comma-separated α grid (default "0,0.25,0.5,0.75,1.0")
+#   CONTEXT        — SEQ_LEN (default "32"; MUST equal train.rb CONTEXT so the
+#                    endpoints match each ckpt's trained loss)
+#   SEED           — random-init seed for the eval session (default "0")
+#   TAO_RUN_DIR    — when set, emit events.jsonl HERE (FILE only); when empty,
+#                    stdout-only.
+#   TOY_RUN_ID     — run_id string for events (default "lmc")
+#
+# Backend: CPU only. A cuda LMC twin is a later slice. This file is deliberately
+# ABSENT from MIRRORABLE in prep/gen_cuda_mirror.rb (exactly like the other
+# CPU runners). `make verify-mirrors` stays green.
+#
+# DETERMINISM: pinned ckpts + fixed seq + lr=0 forward + ggml-internal CE (no
+# Ruby libm) → byte-exact cross-platform, like the train loss curves. This is
+# what prep/lmc_gate.rb gates against prep/fixtures/lmc_baseline.txt.
+#
+# OUTPUT (byte-exact prefix line the CLI + gate parse): one flat line per α, in
+# α-grid order:
+#   "lmc: <alpha>=<loss>"
+#
+# Spinel hygiene (landmine #16): hand-built String-concat JSON (no #{}
+# interpolation, no Math.*); no Struct.new; .split(",")+.to_f into a
+# popped-empty array literal.
+
+require_relative "../../toy"
+require_relative "../../../vendor/spinel/spinel_kit/lib/spinel_kit/json_builder"
+require_relative "../models/toy_smollm2"
+require_relative "../llm/engine/llama_seq_engine"
+require_relative "../llm/adamw"
+require_relative "../train/toy_drift_grad"
+
+LMC_A    = ENV["LMC_A"]      || ""
+LMC_B    = ENV["LMC_B"]      || ""
+ALPHAS_S = ENV["LMC_ALPHAS"] || "0,0.25,0.5,0.75,1.0"
+SEQ_LEN  = (ENV["CONTEXT"] || "32").to_i
+SEED     = (ENV["SEED"]    || "0").to_i
+RUN_DIR  = ENV["TAO_RUN_DIR"] || ""
+RUN_ID   = ENV["TOY_RUN_ID"]  || "lmc"
+
+if LMC_A == "" || LMC_B == ""
+  puts "toy-eval-lmc: LMC_A and LMC_B are required"
+  exit 1
+end
+# Explicit existence checks BEFORE the gguf open (spinel-dev#17 class:
+# a missing user-suppliable path must fail loud and name the path).
+if !File.exist?(LMC_A)
+  puts "toy-eval-lmc: no such file: LMC_A=" + LMC_A +
+       " (pass a from-scratch checkpoint GGUF, e.g. runs/<id>/weights/latest)"
+  exit 1
+end
+if !File.exist?(LMC_B)
+  puts "toy-eval-lmc: no such file: LMC_B=" + LMC_B +
+       " (pass a from-scratch checkpoint GGUF, e.g. runs/<id>/weights/latest)"
+  exit 1
+end
+
+EVENTS = RUN_DIR.length > 0 ? (RUN_DIR + "/events.jsonl") : ""
+if EVENTS != ""
+  TinyNN.tnn_events_open(EVENTS)
+end
+
+# Parse α-grid (Spinel-safe: popped-empty literal + .split + .to_f).
+alphas_arr = [0.0]; alphas_arr.pop
+ALPHAS_S.split(",").each do |a|
+  alphas_arr.push(a.to_f)
+end
+
+# Load both checkpoints (config read from A; the two must share architecture).
+ggA = TinyNN.tnn_gguf_load(LMC_A)
+ggB = TinyNN.tnn_gguf_load(LMC_B)
+if ggA == nil || ggA == TinyNN.tnn_null_ptr
+  puts "toy-eval-lmc: cannot open LMC_A=" + LMC_A; exit 1
+end
+if ggB == nil || ggB == TinyNN.tnn_null_ptr
+  puts "toy-eval-lmc: cannot open LMC_B=" + LMC_B; exit 1
+end
+
+vocab     = TinyNN.tnn_gguf_get_u32(ggA, "llama.vocab_size")
+d_model   = TinyNN.tnn_gguf_get_u32(ggA, "llama.embedding_length")
+d_ff      = TinyNN.tnn_gguf_get_u32(ggA, "llama.feed_forward_length")
+n_heads   = TinyNN.tnn_gguf_get_u32(ggA, "llama.attention.head_count")
+n_kv      = TinyNN.tnn_gguf_get_u32(ggA, "llama.attention.head_count_kv")
+n_layers  = TinyNN.tnn_gguf_get_u32(ggA, "llama.block_count")
+ctx_len   = TinyNN.tnn_gguf_get_u32(ggA, "llama.context_length")
+rope_base = TinyNN.tnn_gguf_get_f32(ggA, "llama.rope.freq_base")
+rms_eps   = TinyNN.tnn_gguf_get_f32(ggA, "llama.attention.layer_norm_rms_epsilon")
+
+cfg = Toy::SmolLM2Config.new(vocab, d_model, n_heads, n_kv, d_ff,
+                             n_layers, ctx_len, rope_base, rms_eps)
+
+# D_HEAD = d_model / n_heads. Per-head fused-tensor slice geometry.
+D_HEAD = d_model / n_heads
+
+# Fixed eval sequence — first TinyStories line, truncated/padded to SEQ_LEN.
+# Byte-identical to train.rb:282-291 so endpoints match each ckpt's loss.
+raw        = File.read("data/ts_seqs.txt")
+first_line = raw.split("\n")[0]
+parts      = first_line.split(" ")
+seq_ids    = [0]; seq_ids.pop
+k = 0
+while k < parts.length && k < SEQ_LEN
+  seq_ids.push(parts[k].to_i)
+  k = k + 1
+end
+while seq_ids.length < SEQ_LEN; seq_ids.push(0); end
+
+# Emit run_start (FILE only).
+if EVENTS != ""
+  t_open = TinyNN.tnn_events_now_seconds
+  rs = SpinelKit::Json::Builder.new
+  rs.add_str("kind",  "run_start")
+  rs.add_str("phase", "eval")
+  rs.add_num("t",          t_open)
+  rs.add_str("started_at", TinyNN.tnn_events_iso8601_now)
+  rs.add_str("run_id",     RUN_ID)
+  rs.add_str("name",       "lmc")
+  host = SpinelKit::Json::Builder.new
+  host.add_str("name", TinyNN.tnn_provenance_host_name)
+  host.add_str("os",   TinyNN.tnn_provenance_host_os)
+  host.add_str("arch", TinyNN.tnn_provenance_host_arch)
+  rs.add_obj("host", host)
+  backend = SpinelKit::Json::Builder.new
+  backend.add_str("kind", "cpu")
+  rs.add_obj("backend", backend)
+  TinyNN.tnn_events_emit(rs.dump)
+end
+
+# Cache for the FULL fused Q/K/V buffers — read each fused GGUF tensor exactly
+# ONCE (not n_heads times). Keyed by fused_name; α-invariant so it can live
+# across all α (the GGUF buffers never change). Two parallel arrays
+# (Spinel-safe; no Hash<String,Mat>): names + their Mat buffers, for A and B.
+fused_names = [""]; fused_names.pop
+fused_a     = [Mat.new(1, 1)]; fused_a.pop
+fused_b     = [Mat.new(1, 1)]; fused_b.pop
+
+# Look up (or read+cache) the full fused tensor for `fused_name` from ckpt A.
+def fused_lookup_a(fused_names, fused_a, fused_b, ggA, ggB, fused_name, nel_full)
+  fi = 0
+  while fi < fused_names.length
+    if fused_names[fi] == fused_name
+      return fi
+    end
+    fi = fi + 1
+  end
+  # Miss — read both A and B once.
+  idx_a = TinyNN.tnn_gguf_find_index(ggA, fused_name)
+  idx_b = TinyNN.tnn_gguf_find_index(ggB, fused_name)
+  if idx_a < 0 || idx_b < 0
+    puts "toy-eval-lmc: missing fused " + fused_name +
+                " in A=" + idx_a.to_s + " B=" + idx_b.to_s
+    return -1
+  end
+  ma = Mat.new(1, nel_full)
+  mb = Mat.new(1, nel_full)
+  TinyNN.tnn_gguf_read_f32_to_doubles(ggA, idx_a, ma.flat, nel_full)
+  TinyNN.tnn_gguf_read_f32_to_doubles(ggB, idx_b, mb.flat, nel_full)
+  fused_names.push(fused_name)
+  fused_a.push(ma)
+  fused_b.push(mb)
+  return fused_names.length - 1
+end
+
+# For each α, build a fresh session, blend weights, run forward+loss.
+ai = 0
+while ai < alphas_arr.length
+  alpha = alphas_arr[ai]
+
+  fcache = Toy::LLM::Engine::LlamaSeqEngine.new
+  # untied=TRUE (arg5) so the session allocates output.weight (the fused
+  # ckpt carries output.weight; train.rb folds with untied=true). We do NOT
+  # set cfg.donor_d_in (it stays 0) so the session embed is the STANDARD
+  # [vocab, d_model] tensor matching the folded fused ckpt.
+  fcache.realize_for_random_init(cfg, SEQ_LEN, 1, 0, true, false, SEED, 1.0)
+
+  result   = fcache.build_training_step
+  t_loss   = result[0]
+  t_labels = result[1]
+  t_hp     = result[2]
+
+  # Blend every PARAM tensor: θ = (1-α)·θ_A + α·θ_B.
+  plist = ToyDriftGrad.params(fcache.sess)
+  pi = 0
+  while pi < plist.length
+    t    = plist[pi]
+    name = TinyNN.tnn_tensor_name(t)
+    nel  = TinyNN.tnn_tensor_nelements(t)
+
+    if name.include?(".head_")
+      # Per-head attention weight: blk.<N>.attn_<q|k|v>.head_<H>.weight.
+      # The fused GGUF carries blk.<N>.attn_<q|k|v>.weight (full matrix);
+      # slice the H-th block out of it. Parse via .split (no regex captures).
+      pre_parts = name.split(".head_")          # ["blk.N.attn_X", "H.weight"]
+      fused_name = pre_parts[0] + ".weight"
+      tail_parts = pre_parts[1].split(".")        # ["H", "weight"]
+      head_h = tail_parts[0].to_i
+
+      # nel_full = (q? n_heads : n_kv) * D_HEAD * d_model. Detect q vs k/v by
+      # the fused_name's attn_ component (q has n_heads rows; k/v have n_kv).
+      is_q = fused_name.include?(".attn_q.")
+      rows = is_q ? n_heads : n_kv
+      nel_full = rows * D_HEAD * d_model
+
+      fci = fused_lookup_a(fused_names, fused_a, fused_b, ggA, ggB,
+                           fused_name, nel_full)
+      if fci < 0
+        pi = pi + 1
+      else
+        fa = fused_a[fci]
+        fb = fused_b[fci]
+        off = head_h * D_HEAD * d_model
+        blended = Mat.new(1, nel)
+        j = 0
+        while j < nel
+          blended.flat[j] = (1.0 - alpha) * fa.flat[off + j] +
+                            alpha * fb.flat[off + j]
+          j = j + 1
+        end
+        TinyNN.tnn_upload_from_float_array(fcache.sess, t, blended.flat, nel)
+        pi = pi + 1
+      end
+    else
+      # Non-attention tensor: DIRECT GGUF name match (08_lmc unchanged).
+      idx_a = TinyNN.tnn_gguf_find_index(ggA, name)
+      idx_b = TinyNN.tnn_gguf_find_index(ggB, name)
+      if idx_a < 0 || idx_b < 0
+        puts "toy-eval-lmc: missing " + name +
+                    " in A=" + idx_a.to_s + " B=" + idx_b.to_s
+        pi = pi + 1
+      else
+        ta = Mat.new(1, nel); tb = Mat.new(1, nel)
+        TinyNN.tnn_gguf_read_f32_to_doubles(ggA, idx_a, ta.flat, nel)
+        TinyNN.tnn_gguf_read_f32_to_doubles(ggB, idx_b, tb.flat, nel)
+        blended = Mat.new(1, nel)
+        j = 0
+        while j < nel
+          blended.flat[j] = (1.0 - alpha) * ta.flat[j] + alpha * tb.flat[j]
+          j = j + 1
+        end
+        TinyNN.tnn_upload_from_float_array(fcache.sess, t, blended.flat, nel)
+        pi = pi + 1
+      end
+    end
+  end
+
+  # Eval: forward + CE. build_training_step with lr=0 → AdamW no-op (pure
+  # forward; the Adam moment update mutates m/v but the session is fresh).
+  positions = [0]; positions.pop
+  p = 0; while p < SEQ_LEN; positions.push(p); p = p + 1; end
+  TinyNN.upload_int_array(fcache.sess, fcache.t_seq_token_ids, seq_ids)
+  TinyNN.upload_int_array(fcache.sess, fcache.t_seq_positions, positions)
+
+  # One-hot label rows (shift-by-one targets).
+  m_labels = Mat.new(SEQ_LEN, vocab)
+  j = 0; while j < SEQ_LEN * vocab; m_labels.flat[j] = 0.0; j = j + 1; end
+  k = 0
+  while k < SEQ_LEN
+    target = (k + 1 < SEQ_LEN) ? seq_ids[k + 1] : seq_ids[k]
+    m_labels.flat[k * vocab + target] = 1.0
+    k = k + 1
+  end
+  TinyNN.upload_row_major(fcache.sess, t_labels, m_labels)
+
+  # NAMED AdamW with lr=0 → no weight update (eval is forward + CE only).
+  # Defaults (β1=0.9, β2=0.95, eps=1e-8, wd=0, bias_correct=false → slots 5/6
+  # = constant betas) match the old hand-filled m_hp slot-for-slot.
+  adamw_eval = Toy::AdamW.for_from_scratch
+  adamw_eval.lr = 0.0
+  m_hp = adamw_eval.hp(0)
+  TinyNN.upload_row_major(fcache.sess, t_hp, m_hp)
+
+  TinyNN.tnn_compute_backward(fcache.sess)
+  loss_mat = TinyNN.download_row_major(fcache.sess, t_loss, 1, 1)
+  loss = loss_mat.flat[0]
+
+  # The byte-gated line (plain ASCII prefix, NOT 08_lmc's non-ASCII α).
+  puts "lmc: " + alpha.to_s + "=" + loss.to_s
+
+  # Emit eval event (FILE only).
+  if EVENTS != ""
+    t_now = TinyNN.tnn_events_now_seconds
+    ev = SpinelKit::Json::Builder.new
+    ev.add_str("kind",  "eval")
+    ev.add_str("phase", "eval")
+    ev.add_num("t",    t_now)
+    ev.add_str("name", "lmc")
+    ev.add_num("loss", loss)
+    extra = SpinelKit::Json::Builder.new
+    extra.add_num("alpha", alpha)
+    ev.add_obj("extra", extra)
+    TinyNN.tnn_events_emit(ev.dump)
+  end
+
+  ai = ai + 1
+end
+
+# Run-end (FILE only).
+if EVENTS != ""
+  t_close = TinyNN.tnn_events_now_seconds
+  re = SpinelKit::Json::Builder.new
+  re.add_str("kind",  "run_end")
+  re.add_str("phase", "eval")
+  re.add_num("t",        t_close)
+  re.add_str("reason",   "completed")
+  re.add_num("n_alphas", alphas_arr.length)
+  TinyNN.tnn_events_emit(re.dump)
+  TinyNN.tnn_events_close
+end
+
+TinyNN.tnn_gguf_free(ggA)
+TinyNN.tnn_gguf_free(ggB)

data/lib/toy/run/eval_metal.rb

@@ -0,0 +1,67 @@
+# lib/toy/run/eval_metal.rb — Spinel-compiled Metal evaluation COMPUTE runner.
+#
+# Metal sibling of lib/toy/run/eval.rb (libexec/toy-eval). The CRuby CLI shell
+# (lib/toy/core/cli/eval.rb) selects this binary when invoked with
+# `--device metal` on macOS: it builds libexec/toy-eval-metal and shells out
+# with the SAME controlled ENV (GGUF/TOP_K). DEVICE is irrelevant here — this
+# binary IS the metal path.
+#
+# HAND-WRITTEN, NOT mechanically mirrored: ToyLMMetal's ctor arity is 1 and it
+# has NO decode_step_with_logprobs (unlike ToyLM). So this file is DELIBERATELY
+# ABSENT from MIRRORABLE in prep/gen_cuda_mirror.rb, exactly like the CPU runner.
+#
+# OPTION (B): we inline ToyLogProbs here (decode_step → log_softmax → top_k)
+# rather than add decode_step_with_logprobs to ToyLMMetal. This (a) avoids
+# touching the Metal class, and (b) keeps this a SINGLE-TYPE file (only
+# ToyLMMetal referenced) so the Spinel polymorphic-lm-var miscompile
+# (landmine #16) cannot fire. transformer_lm_metal.rb does NOT pull
+# toy_logprobs, so we require it explicitly. ToyLogProbs is backend-agnostic CPU.
+#
+# This reproduces ToyLM#decode_step_with_logprobs (transformer_lm.rb:182-190)
+# EXACTLY: decode_step at the frozen pos → ToyLogProbs.log_softmax →
+# ToyLogProbs.top_k → the same "logprob: <id> <val>" line shape.
+#
+# DETERMINISM: pure Metal F32 forward + ToyLogProbs.log_softmax (max-shift
+# stable) + manual top-K (strict-< first-seen tie-break). No sampler/seed.
+# prep/metal_gate.rb gates the top-K id ORDERING metal-vs-cpu.
+# RUNTIME-UNVERIFIED on gx10 (Linux, no Apple frameworks) — gate on the Mac.
+
+require_relative "../models/arch"
+require_relative "../models/transformer_lm_metal"
+require_relative "../dev/toy_logprobs"
+
+GGUF  = ENV["GGUF"] || "data/smollm2-135m-f32.gguf"
+TOP_K = (ENV["TOP_K"] || "5").to_i
+
+# Explicit existence check BEFORE any GGUF read (spinel-dev#17 class).
+if !File.exist?(GGUF)
+  puts "toy-eval-metal: no such file: " + GGUF +
+       " (set GGUF= to a llama-family model; `toy list` shows local caches)"
+  exit 1
+end
+
+arch = Arch.load_or_fail(GGUF, "toy-eval")
+
+lm = ToyLMMetal.new(arch)
+lm.load(GGUF)
+
+# Frozen eval point: EXACT same prefill order as the CPU runner. KV-cache state
+# is op-order load-bearing — the sequential decode_step calls must match.
+ids = [1, 100, 200]
+i = 0
+while i < ids.length
+  lm.decode_step(ids[i], i)
+  i = i + 1
+end
+
+logits   = lm.decode_step(0, ids.length)
+logprobs = ToyLogProbs.log_softmax(logits)
+pair     = ToyLogProbs.top_k(logprobs, TOP_K)
+top_ids  = pair[0]
+top_vals = pair[1]
+
+k = 0
+while k < top_ids.length
+  puts "logprob: " + top_ids[k].to_s + " " + top_vals[k].to_s
+  k = k + 1
+end

data/lib/toy/run/infer.rb

@@ -0,0 +1,130 @@
+# lib/toy/run/infer.rb — Spinel-compiled inference COMPUTE runner.
+#
+# This is the lib-side home of `toy infer`'s compute. The CRuby CLI shell
+# (lib/toy/core/cli/infer.rb) cannot compute in-process — every ffi_lib-
+# bearing lib crashes under MRI — so it locates the toy root, builds this
+# runner (`make libexec/toy-infer`), and shells out to it via Open3 with a
+# controlled ENV. This is the PATTERN train/eval/serve will follow.
+#
+# CONTRACT (read from ENV only — lib-vs-example scope, no experiment config
+# baked in):
+#   GGUF   — path to the model (required; the CLI always passes it)
+#   PROMPT — input text (used only when the GGUF embeds a tokenizer)
+#   N_NEW  — number of tokens to generate (default 16)
+#
+# Backend: CPU only. CUDA/Metal inference lives in separate hand-written
+# classes (ToyLMCuda / ToyLMMetal) with different ctor arity, so this runner
+# is NOT mechanically mirrorable; a --device runner is a later slice. Hence
+# this file is deliberately ABSENT from MIRRORABLE in prep/gen_cuda_mirror.rb.
+#
+# DETERMINISM: generate(..) with no sampler_config → Sampler.argmax (greedy,
+# first-max-wins, no rand/temperature/seed). The output is byte-for-byte
+# reproducible — this is what prep/infer_gate.rb gates against a recorded
+# baseline.
+#
+# OUTPUT (byte-exact prefixes the CLI parses):
+#   "text: <decoded>"  when the GGUF embeds a tokenizer
+#   "ids: <id> <id>…"  otherwise (raw IDs, space-prefixed)
+#
+# DELIBERATELY NOT inherited from the retired examples/01_inference.rb: its
+# ModelIndex auto-select fallback (a silent fallback the never-mask rule
+# forbids). This runner requires an explicit GGUF and fails loud on a bad one.
+
+require_relative "../models/arch"
+require_relative "../models/transformer_lm"
+require_relative "../io/tokenizer"
+
+GGUF  = ENV["GGUF"] || "data/smollm2-135m-f32.gguf"
+PROMPT = ENV["PROMPT"] || "Once upon a time"
+N_NEW  = (ENV["N_NEW"] || "16").to_i
+# PROMPT_IDS — whitespace-separated NUMERIC token ids. Used for
+# tokenizer-less models (e.g. from-scratch checkpoints, vocab=627, no
+# embedded tokenizer): a string prompt cannot be tokenized, so the
+# caller passes raw ids. Empty when unset.
+PROMPT_IDS = ENV["PROMPT_IDS"] || ""
+
+# Explicit existence check BEFORE any GGUF read (spinel-dev#17 class:
+# missing user-suppliable paths fail loud and name the path + the fix).
+if !File.exist?(GGUF)
+  puts "toy-infer: no such file: " + GGUF +
+       " (set GGUF= to a llama-family model; `toy list` shows local caches)"
+  exit 1
+end
+
+arch = Arch.load_or_fail(GGUF, "toy-infer")
+puts arch.summary
+
+lm = ToyLM.new(arch, :cpu)
+lm.load(GGUF)
+
+# Try to load an embedded tokenizer. If absent (the converter was run
+# without --with-tokenizer), fall back to the hardcoded SmolLM2 prompt
+# IDs and emit raw IDs instead of decoded text.
+tok = Tokenizer.from_gguf(GGUF)
+
+# all_digits? — true iff `s` is non-empty and every char is 0..9. Explicit
+# char scan (NOT exception-based Integer(s)) per the Spinel landmines.
+def all_digits?(s)
+  return false if s.length == 0
+  i = 0
+  while i < s.length
+    c = s[i]
+    if c < "0" || c > "9"
+      return false
+    end
+    i = i + 1
+  end
+  true
+end
+
+if PROMPT_IDS.length > 0
+  # NUMERIC-ids path (tokenizer-less or explicitly id-driven). Parse +
+  # validate EVERY id is an integer in [0, vocab). Fail loud on any bad
+  # id (never silently mistokenize / clamp).
+  parts = PROMPT_IDS.split(" ")
+  parsed = [0]; parsed.pop
+  j = 0
+  while j < parts.length
+    tokstr = parts[j]
+    if tokstr.length > 0
+      if all_digits?(tokstr) == false
+        puts "toy-infer: PROMPT_IDS contains invalid token '" + tokstr +
+             "' (must be integer in [0," + arch.vocab_size.to_s + "))"
+        exit 1
+      end
+      idv = tokstr.to_i
+      if idv < 0 || idv >= arch.vocab_size
+        puts "toy-infer: PROMPT_IDS contains invalid token '" + tokstr +
+             "' (must be integer in [0," + arch.vocab_size.to_s + "))"
+        exit 1
+      end
+      parsed.push(idv)
+    end
+    j = j + 1
+  end
+  if parsed.length == 0
+    puts "toy-infer: PROMPT_IDS parsed to no token ids"
+    exit 1
+  end
+  out_ids = lm.generate(parsed, N_NEW)
+  print "ids:"
+  k = 0
+  while k < out_ids.length
+    print " " + out_ids[k].to_s
+    k = k + 1
+  end
+  puts ""
+elsif tok.present
+  in_ids = tok.encode(PROMPT)
+  puts "prompt: " + PROMPT.inspect + " → " + in_ids.length.to_s + " tokens"
+  out_ids = lm.generate(in_ids, N_NEW)
+  # The model returns prompt + continuation; decode the whole thing.
+  puts "text: " + tok.decode(out_ids)
+else
+  # Tokenizer-less model AND no PROMPT_IDS: a string prompt cannot be
+  # tokenized. Fail loud (never the old silent hardcoded-id fallback).
+  puts "toy-infer: model has no embedded tokenizer; a string prompt cannot " +
+       "be tokenized. Pass numeric token IDs via --prompt-ids (PROMPT_IDS=...) " +
+       "or re-convert with --with-tokenizer."
+  exit 1
+end

data/lib/toy/run/infer_cuda.rb

@@ -0,0 +1,118 @@
+# lib/toy/run/infer_cuda.rb — Spinel-compiled CUDA inference COMPUTE runner.
+#
+# CUDA sibling of lib/toy/run/infer.rb (libexec/toy-infer). The CRuby CLI shell
+# (lib/toy/core/cli/infer.rb) selects this binary when invoked with
+# `--device cuda`: it builds libexec/toy-infer-cuda and shells out with the
+# SAME controlled ENV (GGUF/PROMPT/N_NEW). DEVICE is irrelevant in here — this
+# binary IS the cuda path.
+#
+# HAND-WRITTEN, NOT mechanically mirrored: ToyLMCuda's ctor arity is 1
+# (ToyLMCuda.new(arch)), unlike ToyLM.new(arch, :cpu), so gen_cuda_mirror.rb
+# cannot derive this file from the CPU runner. It is therefore DELIBERATELY
+# ABSENT from MIRRORABLE in prep/gen_cuda_mirror.rb — exactly like the CPU
+# runner is, for the inverse reason.
+#
+# SINGLE-TYPE FILE: only ToyLMCuda is referenced (no ToyLM), so the Spinel
+# polymorphic-lm-var miscompile (landmine #16) cannot fire. Each device has its
+# own self-contained runner; the CLI does the dispatch by picking the binary.
+#
+# OUTPUT: byte-identical prefixes to the CPU runner ("text: …" / "ids: …") so
+# the CLI's scan_line and prep/infer_gate.rb parse both branches identically.
+# Hand-built output only — no #{} interpolation in any emitted line.
+#
+# DETERMINISM: generate(..) with no sampler_config → Sampler.argmax (greedy,
+# first-max-wins, no rand/seed). The CUDA F32 forward is expected to match the
+# CPU runner's generated token IDs; prep/infer_gate.rb (TOY_GATE_CUDA=1) gates
+# that discrete invariant cuda-vs-cpu in the same run.
+
+require_relative "../models/arch"
+require_relative "../models/transformer_lm_cuda"
+require_relative "../io/tokenizer"
+
+GGUF  = ENV["GGUF"] || "data/smollm2-135m-f32.gguf"
+PROMPT = ENV["PROMPT"] || "Once upon a time"
+N_NEW  = (ENV["N_NEW"] || "16").to_i
+# PROMPT_IDS — whitespace-separated NUMERIC token ids, for tokenizer-less
+# models (parity with the CPU runner). Empty when unset.
+PROMPT_IDS = ENV["PROMPT_IDS"] || ""
+
+# all_digits? — true iff `s` is non-empty and every char is 0..9 (explicit
+# char scan, not exception-based Integer(s), per the Spinel landmines).
+def all_digits?(s)
+  return false if s.length == 0
+  i = 0
+  while i < s.length
+    c = s[i]
+    if c < "0" || c > "9"
+      return false
+    end
+    i = i + 1
+  end
+  true
+end
+
+# Explicit existence check BEFORE any GGUF read (spinel-dev#17 class).
+if !File.exist?(GGUF)
+  puts "toy-infer-cuda: no such file: " + GGUF +
+       " (set GGUF= to a llama-family model; `toy list` shows local caches)"
+  exit 1
+end
+
+arch = Arch.load_or_fail(GGUF, "toy-infer")
+puts arch.summary
+
+lm = ToyLMCuda.new(arch)
+lm.load(GGUF)
+
+tok = Tokenizer.from_gguf(GGUF)
+
+if PROMPT_IDS.length > 0
+  # NUMERIC-ids path (tokenizer-less or explicitly id-driven). Parse +
+  # validate EVERY id is an integer in [0, vocab). Fail loud on any bad id
+  # (never silently mistokenize / clamp) — parity with the CPU runner.
+  parts = PROMPT_IDS.split(" ")
+  parsed = [0]; parsed.pop
+  j = 0
+  while j < parts.length
+    tokstr = parts[j]
+    if tokstr.length > 0
+      if all_digits?(tokstr) == false
+        puts "toy-infer: PROMPT_IDS contains invalid token '" + tokstr +
+             "' (must be integer in [0," + arch.vocab_size.to_s + "))"
+        exit 1
+      end
+      idv = tokstr.to_i
+      if idv < 0 || idv >= arch.vocab_size
+        puts "toy-infer: PROMPT_IDS contains invalid token '" + tokstr +
+             "' (must be integer in [0," + arch.vocab_size.to_s + "))"
+        exit 1
+      end
+      parsed.push(idv)
+    end
+    j = j + 1
+  end
+  if parsed.length == 0
+    puts "toy-infer: PROMPT_IDS parsed to no token ids"
+    exit 1
+  end
+  out_ids = lm.generate(parsed, N_NEW)
+  print "ids:"
+  k = 0
+  while k < out_ids.length
+    print " " + out_ids[k].to_s
+    k = k + 1
+  end
+  puts ""
+elsif tok.present
+  in_ids = tok.encode(PROMPT)
+  puts "prompt: " + PROMPT.inspect + " → " + in_ids.length.to_s + " tokens"
+  out_ids = lm.generate(in_ids, N_NEW)
+  puts "text: " + tok.decode(out_ids)
+else
+  # Tokenizer-less AND no PROMPT_IDS: fail loud (never the old silent
+  # hardcoded-id fallback). Parity with the CPU runner.
+  puts "toy-infer: model has no embedded tokenizer; a string prompt cannot " +
+       "be tokenized. Pass numeric token IDs via --prompt-ids (PROMPT_IDS=...) " +
+       "or re-convert with --with-tokenizer."
+  exit 1
+end