toy 0.8.0

data/lib/toy/run/train_vit.rb

@@ -0,0 +1,251 @@
+# lib/toy/run/train_vit.rb — Spinel-compiled ViT-Tiny from-scratch TRAINING
+# compute runner (→ libexec/toy-train-vit).
+#
+# This is the lib-side home of `toy train vit-tiny`'s compute. The CRuby CLI
+# shell (lib/toy/core/cli/train.rb) cannot compute in-process (every
+# ffi_lib-bearing lib crashes under MRI), so it locates the toy root, builds
+# this runner, creates runs/<id>/, and shells out via Open3 with a CONTROLLED
+# ENV — the same CRuby→runner COMPUTE BRIDGE that train.rb uses.
+#
+# SEPARATE BINARY (landmine #16): ViTTinyConfig must NOT share a Spinel
+# compilation unit with SmolLM2Config (cfg type-merge miscompile). Hence this
+# distinct source/binary, and it never requires toy_gguf_writer/toy_gguf_fuse
+# — their cfg.vocab/cfg.d_ff style getters poly-collide with ViTTinyConfig
+# (07_train_vit_tiny.rb:22-26). The ViT cache has no GGUF writer + there is no
+# ViT infer consumer yet, so NO checkpoint is written here (#169 follow-up).
+#
+# REAL DATA, NOT SYNTHETIC: this trains random-init on the COMMITTED/pinned
+# corpus data/vit_smoke/{images.bin,labels.bin} via ToyImageLoader (mirrors
+# 07_train_vit_tiny.rb's data path verbatim), NOT a synthetic-fill smoke. The
+# 224/16/196/10 timm shape below is the shape data/vit_smoke matches.
+#
+# DETERMINISM: random init is self-contained (upload_random_init! is
+# Box-Muller xorshift64 seeded by SEED — no donor). All loss math is
+# ggml-internal (RMSNorm + SiLU + cross-entropy, no Ruby libm), so the printed
+# "step N: loss=" curve is byte-for-byte reproducible AND cross-platform
+# portable — STRICT byte-exact, the same class as the llama train gate
+# (prep/train_gate.rb's vit-tiny case vs prep/fixtures/train_vit_baseline.txt).
+#
+# OUTPUT (byte-exact line the CLI + gate parse):
+#   "step <N>: loss=<float>"   one per step, to STDOUT (NO lr/idx/label
+# decoration — that matches the llama gate line format). events.jsonl goes to
+# runs/<id>/ (file-side, never stdout).
+#
+# Spinel hygiene (landmine #16): hand-built String-concat JSON (no #{}
+# interpolation); ENV reads are TOP-LEVEL constants (a constant assigned
+# inside a conditional arm reads back empty at runtime); no Struct.new.
+
+require_relative "../../../vendor/spinel/spinel_kit/lib/spinel_kit/json_builder"
+require_relative "../dev/toy_describe_flow"
+require_relative "../io/toy_events"
+require_relative "../models/toy_vit"
+require_relative "../llm/engine/vit_tiny_engine"
+require_relative "../io/toy_image_loader"
+require_relative "../train/toy_lr_schedule"
+require_relative "../train/toy_drift_grad"
+require_relative "../llm/recipes/vit_tiny"
+require_relative "../llm/adamw"
+
+# ENV reads — TOP-LEVEL constants (Spinel constant-in-conditional caveat).
+STEPS       = (ENV["STEPS"]      || "5").to_i
+SEED        = (ENV["SEED"]       || "0").to_i
+IMG_DIR     = (ENV["IMG_DIR"]    || "data/vit_smoke")
+TAO_RUN_DIR = (ENV["TAO_RUN_DIR"] || "")
+TOY_RUN_ID  = (ENV["TOY_RUN_ID"]  || "vit-tiny")
+EVENTS      = TAO_RUN_DIR.length > 0 ? (TAO_RUN_DIR + "/events.jsonl") : ""
+
+# Gate-fixed timm ViT-Tiny SHAPE — hardcoded (NOT env/flags). This is the
+# shape data/vit_smoke matches (224/16/196/10); the 16x16 ENV defaults in
+# 07_train_vit_tiny.rb's header are the REJECTED synthetic shape.
+IMAGE_SIZE  = 224
+PATCH_SIZE  = 16
+NUM_CHAN    = 3
+D_MODEL     = 192
+N_HEADS     = 3
+D_FF        = 768
+N_LAYERS    = 12
+NUM_CLASSES = 10
+LN_EPS      = 1.0e-5
+N_IMAGES    = 1
+
+# LR schedule PINNED to 07's defaults (confirmed produce the recorded
+# baseline curve). With WARMUP=10 > STEPS=5, every step is on the linear
+# warmup ramp.
+LR_MAX = 0.003
+LR_MIN = 0.0001
+WARMUP = 10
+
+# d_head = 192/3 = 64 derived inside ViTTinyConfig.
+cfg = ViTTinyConfig.new(IMAGE_SIZE, PATCH_SIZE, NUM_CHAN, D_MODEL,
+                        N_HEADS, D_FF, N_LAYERS, NUM_CLASSES, LN_EPS)
+
+# Named realize options (toy#64): the ViT path consumes seed +
+# init_scale only (init_scale keeps the canonical 1.0 default).
+opts = Toy::LLM::RecipeOptions.new
+opts.seed = SEED
+
+recipe = Toy::LLM::Recipes::VitTiny.new
+recipe.realize!(cfg, opts)
+# tao#flow-json-emit (#25): self-describing run bundle, parallel to events.jsonl.
+ToyDescribeFlow.emit_flow_json(TAO_RUN_DIR, recipe.vt_cache.sess)
+
+# Pre-allocate buffers used every step (07_train_vit_tiny.rb:223-237).
+n_patches  = (IMAGE_SIZE / PATCH_SIZE) * (IMAGE_SIZE / PATCH_SIZE)  # 196
+patch_flat = NUM_CHAN * PATCH_SIZE * PATCH_SIZE                     # 768
+record_f   = patch_flat * n_patches                                # 150528 (=602112/4)
+
+m_image  = Mat.new(patch_flat, n_patches)   # [768, 196]; upload_row_major matches t_image input_2d(196,768)
+m_labels = Mat.new(1, NUM_CLASSES)          # [1, 10]
+cls_idx  = [0]
+# NAMED AdamW. Defaults (beta2=0.95, bias_correct=false) → slots5/6 =
+# constant betas, byte-identical to the historical inline hp. lr is set
+# per step from the cosine schedule (vit never reads m_hp before the
+# per-step assignment), and the hp Mat is rebuilt per step below — the
+# original implicit flat[0]=0.0-until-set is reproduced by Mat.new's
+# zero-init inside hp(). Single-class labels stay inline (NOT shift-by-one).
+adamw_vit = Toy::AdamW.for_from_scratch
+
+images_path = IMG_DIR + "/images.bin"
+labels_path = IMG_DIR + "/labels.bin"
+
+# FAIL LOUD on a missing/truncated corpus. ToyImageLoader.read_image
+# zero-fills a short read (returning Array.new(record_f, 0.0)) and only puts a
+# warning — so a missing corpus would silently train on zeros and reproduce the
+# baseline curve, masking the failure (never-mask rule). The vit-tiny gate
+# corpus is COMMITTED at data/vit_smoke/ (un-gitignored). Probe the first image
+# record with the loader's OWN primitive (tnn_read_f32_file returns the actual
+# count read) and abort non-zero if missing/short — NO synthetic zero-fill.
+# (Use `puts`, not STDERR.puts: STDERR is a no-op in the Spinel runner; the
+# from-scratch runner emits its errors via `puts` too, and the CRuby CLI
+# capture2e merges stdout+stderr and prints the tail on a non-zero exit.)
+if !File.exist?(images_path) || !File.exist?(labels_path)
+  puts "toy-train-vit: corpus missing under " + IMG_DIR +
+       " (need images.bin + labels.bin; the vit-tiny gate corpus is committed at data/vit_smoke/)."
+  exit 1
+end
+probe_buf = Array.new(record_f, 0.0)
+probe_got = TinyNN.tnn_read_f32_file(images_path, 0, record_f, probe_buf)
+if probe_got != record_f
+  puts "toy-train-vit: corpus short read " + probe_got.to_s + "/" + record_f.to_s +
+       " from " + images_path + " — refusing to train on a zero-filled read."
+  exit 1
+end
+
+# git provenance read pure-Ruby from .git/HEAD (07:140-167).
+
+# --- run_start event (FILE only; arch=vit). ---
+if EVENTS.length > 0
+  rc = TinyNN.tnn_events_open(EVENTS)
+  if rc == 0
+    rid = TOY_RUN_ID.length > 0 ? TOY_RUN_ID : "anonymous"
+    rs = SpinelKit::Json::Builder.new
+    rs.add_str("kind", "run_start")
+    rs.add_str("schema", "toy/v1")
+    rs.add_num("t", TinyNN.tnn_events_now_seconds)
+    rs.add_str("started_at", TinyNN.tnn_events_iso8601_now)
+    rs.add_str("run_id", rid)
+    rs.add_str("phase", "train")
+    rs.add_str("name", "vit-tiny")
+    Toy::Events.add_provenance(rs,
+      TinyNN.tnn_provenance_host_name, TinyNN.tnn_provenance_host_os,
+      TinyNN.tnn_provenance_host_arch,
+      TinyNN.tnn_backend_name(recipe.vt_cache.sess))
+    model = SpinelKit::Json::Builder.new
+    model.add_str("arch", "vit")
+    model.add_str("name", "vit-tiny")
+    model.add_num("image_size",  cfg.image_size)
+    model.add_num("patch_size",  cfg.patch_size)
+    model.add_num("d_model",     cfg.d_model)
+    model.add_num("n_layers",    cfg.n_layers)
+    model.add_num("n_heads",     cfg.n_heads)
+    model.add_num("d_ff",        cfg.d_ff)
+    model.add_num("num_classes", cfg.num_classes)
+    rs.add_obj("model", model)
+    config = SpinelKit::Json::Builder.new
+    config.add_str("image_dir", IMG_DIR)
+    config.add_num("n_images",  N_IMAGES)
+    config.add_num("steps",     STEPS)
+    config.add_num("seed",      SEED)
+    config.add_str("init",      "scratch")
+    rs.add_obj("config", config)
+    schedule = SpinelKit::Json::Builder.new
+    schedule.add_num("lr_max",  LR_MAX)
+    schedule.add_num("lr_min",  LR_MIN)
+    schedule.add_num("warmup",  WARMUP)
+    schedule.add_num("n_steps", STEPS)
+    rs.add_obj("schedule", schedule)
+    TinyNN.tnn_events_emit(rs.dump)
+  else
+    puts "events_open failed: rc=" + rc.to_s + " (path=" + EVENTS + ")"
+  end
+end
+
+# --- Training loop (0-indexed; cosine LR + REAL data from data/vit_smoke). ---
+final_loss = 0.0
+step = 0
+while step < STEPS
+  step_wall_start = TinyNN.tnn_events_now_seconds
+  lr = ToyLR.cosine(step, STEPS, LR_MAX, LR_MIN, WARMUP)
+  adamw_vit.lr = lr
+  m_hp = adamw_vit.hp(step)   # bias_correct=false → slots5/6=betas
+
+  # Single-image memorisation: idx=0 every step (N_IMAGES=1, deterministic).
+  idx = step % N_IMAGES
+  patches = ToyImageLoader.read_image(images_path, idx, record_f)
+  label   = ToyImageLoader.read_label(labels_path, idx)
+  if label < 0 || label >= NUM_CLASSES
+    puts "skip step " + step.to_s + " (bad label=" + label.to_s + ")"
+    step = step + 1
+    next
+  end
+
+  # Copy patches into the pre-allocated image Mat.
+  i = 0
+  while i < record_f
+    m_image.flat[i] = patches[i]
+    i = i + 1
+  end
+  # One-hot label.
+  j = 0
+  while j < NUM_CLASSES
+    m_labels.flat[j] = (j == label) ? 1.0 : 0.0
+    j = j + 1
+  end
+
+  loss = recipe.step!(m_image, cls_idx, m_labels, m_hp, step == 0)
+  final_loss = loss
+  # The byte-gated line — to STDOUT, NO decoration (llama gate line format).
+  puts "step " + (step + 1).to_s + ": loss=" + loss.to_s
+
+  if EVENTS.length > 0
+    step_wall_us = ((TinyNN.tnn_events_now_seconds - step_wall_start) * 1.0e6).to_i
+    es = SpinelKit::Json::Builder.new
+    es.add_str("kind",  "step")
+    es.add_str("phase", "train")
+    es.add_num("t",       TinyNN.tnn_events_now_seconds)
+    es.add_num("step",    step + 1)
+    es.add_num("loss",    loss)
+    es.add_num("lr",      lr)
+    es.add_num("wall_us", step_wall_us)
+    TinyNN.tnn_events_emit(es.dump)
+  end
+  step = step + 1
+end
+
+# --- run_end (FILE only). NO checkpoint write: the ViT cache has no GGUF
+# writer + ToyGGUFWriter/Fuser poly-collide with ViTTinyConfig (#169
+# follow-up). Document the deferral in a note field (events, NOT stdout). ---
+if EVENTS.length > 0 && TinyNN.tnn_events_active == 1
+  re = SpinelKit::Json::Builder.new
+  re.add_str("kind", "run_end")
+  re.add_num("t",          TinyNN.tnn_events_now_seconds)
+  re.add_str("ended_at",   TinyNN.tnn_events_iso8601_now)
+  re.add_str("reason",     "completed")
+  re.add_num("final_step", STEPS)
+  re.add_num("final_loss", final_loss)
+  re.add_str("checkpoint", "deferred-169")
+  re.add_str("note",       "ViT cache has no GGUF writer; checkpoint is the #169 follow-up")
+  re.add_raw("exit_code",  "0")
+  TinyNN.tnn_events_emit(re.dump)
+  TinyNN.tnn_events_close
+end

data/lib/toy/serve/openai/embeddings_handler.rb

@@ -0,0 +1,92 @@
+# lib/toy/serve/openai/embeddings_handler.rb -- OpenAI-shape
+# /v1/embeddings handler.
+#
+# MOVED from tep_demo/embeddings_handler.rb (P4 toy serve). Decodes the input
+# token-id array with SpinelKit::Json.get_int_array (toy#44; the former local
+# ApiJson shim is retired — SpinelKit's decoder replaces it). Constructed with
+# (STATE, MODEL_NAME) so the handler doesn't need cross-file constant resolution.
+#
+# Contract:
+# - Body: {"input": [int, int, ...], "model": "..."} -- IDs only,
+#   tokenize client-side (matches the server's overall "IDs only"
+#   policy).
+# - Lookup: per-token embedding via tnn_embed_lookup_to_doubles
+#   (dequantize-aware; works on f32 + Q8 + Q4 tables).
+# - Pooling: mean over the input tokens -> single d_model vector.
+# - Response shape: OpenAI /v1/embeddings v1.
+#
+# Spinel notes:
+# - State is passed in at construction so the handler doesn't need
+#   cross-file constant resolution (STATE is per-server).
+# - The pooling loop uses pure while loops + Array<Float> buffers
+#   seeded with pop-to-empty (the [0.0]; arr.pop type-pin pattern).
+
+class EmbeddingsHandler < Tep::Handler
+  def initialize(state, model_name)
+    @state      = state
+    @model_name = model_name
+  end
+
+  def handle(req, res)
+    res.headers["Content-Type"] = "application/json"
+    body = req.body
+
+    ids = SpinelKit::Json.get_int_array(body, "input")
+    if ids.length == 0
+      res.set_status(400)
+      return "{\"error\":{\"message\":\"input must be a non-empty int array " +
+             "(this server speaks IDs only; tokenize client-side)\"," +
+             "\"type\":\"invalid_request_error\"}}\n"
+    end
+
+    d_model = @state.cfg.d_model
+    sess    = @state.kv.sess
+    t_embed = @state.kv.t_token_embed
+
+    # Mean-pool buffer: sum across tokens, then divide.
+    sum_buf = [0.0]; sum_buf.pop
+    j0 = 0
+    while j0 < d_model
+      sum_buf.push(0.0)
+      j0 = j0 + 1
+    end
+    tok_buf = [0.0]; tok_buf.pop
+    j1 = 0
+    while j1 < d_model
+      tok_buf.push(0.0)
+      j1 = j1 + 1
+    end
+
+    i = 0
+    while i < ids.length
+      rc = TinyNN.tnn_embed_lookup_to_doubles(sess, t_embed, ids[i], tok_buf, d_model)
+      if rc != 0
+        # Per the "never mask, fail loud" rule, surface as an error
+        # response rather than silently returning a partial vector.
+        res.set_status(500)
+        return "{\"error\":{\"message\":\"embed_lookup rc=" + rc.to_s +
+               " at token index " + i.to_s + " (id=" + ids[i].to_s +
+               ")\",\"type\":\"server_error\"}}\n"
+      end
+      k = 0
+      while k < d_model
+        sum_buf[k] = sum_buf[k] + tok_buf[k]
+        k = k + 1
+      end
+      i = i + 1
+    end
+
+    inv_n = 1.0 / ids.length.to_f
+    out = "{\"object\":\"list\",\"data\":[{\"object\":\"embedding\",\"index\":0,\"embedding\":["
+    kk = 0
+    while kk < d_model
+      if kk > 0; out = out + ","; end
+      out = out + (sum_buf[kk] * inv_n).to_s
+      kk = kk + 1
+    end
+    out = out + "]}],\"model\":\"" + @model_name +
+          "\",\"usage\":{\"prompt_tokens\":" + ids.length.to_s +
+          ",\"total_tokens\":" + ids.length.to_s + "}}\n"
+    out
+  end
+end

data/lib/toy/serve/openai/handlers.rb

@@ -0,0 +1,143 @@
+# lib/toy/serve/openai/handlers.rb -- Tep::Handler subclasses for the
+# OpenAI-compatible endpoints.
+#
+# MOVED from tep_demo/openai_api_llama.rb:268-365 (P4 toy serve).
+# Handlers: Health (/health), Models (/v1/models), Completions
+# (/v1/completions -- choices[0].ids is the gate field), ChatCompletions
+# (/v1/chat/completions -> 501), Index (/). The /v1/embeddings handler
+# lives in embeddings_handler.rb.
+#
+# FIX during the move: the pre-existing HTML bug in the index page
+# (openai_api_llama.rb:359-360) -- an orphaned String literal with no
+# leading `+` followed by a stray trailing `++`. It is not gated either
+# way (the gate hits /v1/completions, not /), so fixing it during the
+# lift is safe and removes a latent syntax landmine.
+#
+# Tep is consumed purely as transport. Spinel hygiene (#16): String-concat
+# JSON, no #{} interpolation, no Struct.new.
+
+class HealthHandler < Tep::Handler
+  def handle(req, res)
+    res.headers["Content-Type"] = "text/plain"
+    if STATE.ready
+      "ok\n"
+    else
+      res.set_status(503)
+      "loading\n"
+    end
+  end
+end
+
+class ModelsHandler < Tep::Handler
+  def handle(req, res)
+    res.headers["Content-Type"] = "application/json"
+    "{\"object\":\"list\",\"data\":[{" +
+      SpinelKit::Json.encode_pair_str("id", STATE.model_name) + "," +
+      SpinelKit::Json.encode_pair_str("object", "model") + "," +
+      SpinelKit::Json.encode_pair_int("created", api_now_unix) + "," +
+      SpinelKit::Json.encode_pair_str("owned_by", "toy") +
+    "}]}\n"
+  end
+end
+
+class CompletionsHandler < Tep::Handler
+  def handle(req, res)
+    res.headers["Content-Type"] = "application/json"
+    body = req.body
+
+    # Accept the prompt as a JSON int array. OpenAI spec allows
+    # `prompt: <int-array>` for pre-tokenized input; we require it.
+    prompt_ids = SpinelKit::Json.get_int_array(body, "prompt")
+    if prompt_ids.length == 0
+      prompt_ids = SpinelKit::Json.get_int_array(body, "prompt_ids")
+    end
+
+    if prompt_ids.length == 0
+      res.set_status(400)
+      return "{\"error\":{\"message\":\"prompt must be a non-empty int array " +
+             "(this server speaks IDs only; tokenize client-side)\"," +
+             "\"type\":\"invalid_request_error\"}}\n"
+    end
+
+    n_new = 16
+    if SpinelKit::Json.has_key?(body, "max_tokens")
+      n_new = SpinelKit::Json.get_int(body, "max_tokens")
+    end
+    if n_new <= 0; n_new = 16; end
+    if n_new > 256; n_new = 256; end
+
+    # Measure generation latency across the single compute call (cheap: two
+    # tnn_events_now_seconds reads regardless of whether events are active).
+    t_start = TinyNN.tnn_events_now_seconds
+    new_ids = api_generate_ids(prompt_ids, n_new)
+    t_end   = TinyNN.tnn_events_now_seconds
+    prompt_len = prompt_ids.length
+    completion_len = new_ids.length
+    latency_us = ((t_end - t_start) * 1.0e6).to_i
+
+    # ONE eval/serve/request event per completion (toy/v1). FILE-only side
+    # effect: the response String below is byte-UNCHANGED. Guarded by the
+    # process-global C events state (tnn_events_active==1 only when the runner
+    # opened events.jsonl, i.e. TAO_RUN_DIR was set), so the handler needs no
+    # EVENTS-path knowledge. NO emit on the 400 empty-prompt early-return.
+    if TinyNN.tnn_events_active == 1
+      STATE.req_seq = STATE.req_seq + 1
+      req_id = "req-" + STATE.req_seq.to_s
+      ev  = "{\"kind\":\"eval\",\"phase\":\"serve\""
+      ev = ev + ",\"t\":"    + TinyNN.tnn_events_now_seconds.to_s
+      ev = ev + ",\"name\":\"request\""
+      ev = ev + ",\"extra\":{"
+      ev = ev +   "\"model\":\"" + STATE.model_name + "\""
+      ev = ev +   ",\"prompt_tokens\":"     + prompt_len.to_s
+      ev = ev +   ",\"completion_tokens\":" + completion_len.to_s
+      ev = ev +   ",\"latency_us\":"        + latency_us.to_s
+      ev = ev +   ",\"sampling\":{\"max_tokens\":" + n_new.to_s + "}"
+      ev = ev +   ",\"request_id\":\"" + req_id + "\""
+      ev = ev + "}}"
+      TinyNN.tnn_events_emit(ev)
+    end
+
+    "{" +
+      SpinelKit::Json.encode_pair_str("id", api_gen_id("cmpl")) + "," +
+      SpinelKit::Json.encode_pair_str("object", "text_completion") + "," +
+      SpinelKit::Json.encode_pair_int("created", api_now_unix) + "," +
+      SpinelKit::Json.encode_pair_str("model", STATE.model_name) + "," +
+      "\"choices\":[{\"index\":0," +
+        SpinelKit::Json.encode_pair_str("text", "") + "," +
+        "\"ids\":" + SpinelKit::Json.from_int_array(new_ids) + "," +
+        "\"finish_reason\":\"length\"}]," +
+      "\"usage\":{" +
+        SpinelKit::Json.encode_pair_int("prompt_tokens", prompt_len) + "," +
+        SpinelKit::Json.encode_pair_int("completion_tokens", completion_len) + "," +
+        SpinelKit::Json.encode_pair_int("total_tokens", prompt_len + completion_len) +
+      "}}\n"
+  end
+end
+
+class ChatCompletionsHandler < Tep::Handler
+  def handle(req, res)
+    res.headers["Content-Type"] = "application/json"
+    res.set_status(501)
+    "{\"error\":{\"message\":\"chat/completions requires a tokenizer; " +
+    "this server speaks IDs only. Use POST /v1/completions with " +
+    "prompt as an int array.\",\"type\":\"not_implemented\"}}\n"
+  end
+end
+
+class IndexHandler < Tep::Handler
+  def handle(req, res)
+    res.headers["Content-Type"] = "text/html; charset=utf-8"
+    "<!doctype html><html><head><title>toy openai-compat (IDs only)</title></head>" +
+    "<body><h1>toy openai-compat API (ID-only)</h1>" +
+    "<p>Model: <code>" + STATE.model_name + "</code> via direct GGUF&rarr;FFI loader.</p>" +
+    "<p>Tokenize client-side; this server speaks integer token IDs only. " +
+    "Run <code>prep/qwen25_tokens.py encode \"...\"</code> to get IDs.</p>" +
+    "<p>Endpoints:</p><ul>" +
+    "<li><code>POST /v1/completions</code> &mdash; body <code>{\"prompt\":[int,...],\"max_tokens\":N}</code></li>" +
+    "<li><code>POST /v1/chat/completions</code> &mdash; 501 (tokenizer required)</li>" +
+    "<li><code>POST /v1/embeddings</code> &mdash; body <code>{\"input\":[int,...]}</code> &rarr; mean-pooled vector</li>" +
+    "<li><code>GET /v1/models</code></li>" +
+    "<li><code>GET /health</code></li>" +
+    "</ul></body></html>"
+  end
+end

data/lib/toy/serve/openai/server.rb

@@ -0,0 +1,159 @@
+# lib/toy/serve/openai/server.rb -- the Server state + boot + greedy
+# decode for `toy serve` (OpenAI-compatible, llama-family GGUF).
+#
+# MOVED from tep_demo/openai_api_llama.rb:136-264 (P4 toy serve). The
+# endpoint logic that USED to live in the tep_demo binary now lives in
+# lib/toy/serve/openai/; the tep_demo source STAYS until the later
+# cleanup pass retires it (serve must be gated first).
+#
+# Tep is consumed purely as HTTP transport (Tep::Handler / Tep.get/post/
+# Tep.run!). JSON encode/decode is SpinelKit::Json now, NOT Tep::Json (toy#44):
+# serve's only remaining Tep coupling is the handler/server surface. Nothing
+# here patches Tep -- this is build-dep usage, not "Tep re-adaptation" (that =
+# Tep consuming Toy, DEFERRED per user directive until Toy is stable).
+#
+# CONTROLLED ENV contract (set by `toy serve` via lib/toy/run/serve.rb):
+#   MODEL_PATH   path to any llama-family GGUF (default
+#                data/smollm2-135m-native.gguf)
+#   MODEL_NAME   user-facing label in /v1/models + completion responses.
+#                Defaults to GGUF basename minus .gguf.
+#   MAX_T        max context for the KV cache (default 256)
+#   PORT         (read in the entrypoint, not here)
+#
+# Spinel hygiene (#16): no #{} interpolation, no Struct.new, String-concat
+# JSON, long-hand loops. KEEP the GH#188 landmine workarounds VERBATIM:
+#   - CONSTANT-held STATE (so Spinel emits a typed slot)
+#   - explicit STATE.model_name re-assign AFTER STATE.new
+#   - rindex-based basename derivation (the rindex landmine workaround)
+# Greedy api_generate_ids: pure argmax, first-max-wins, NO rand/seed/
+# temperature -- CONFIRMED deterministic (this is what the HTTP gate
+# reproduces byte-for-byte).
+
+# GH#188 -- model selection via env. Defaults to SmolLM2-135M for a
+# cheap-to-test smoke; override for any llama-family GGUF.
+# If MODEL_NAME isn't set, it defaults to the basename of MODEL_PATH
+# minus the ".gguf" suffix -- close enough for the /v1/models response.
+GGUF_PATH = ENV["MODEL_PATH"] || "data/smollm2-135m-native.gguf"
+MODEL_NAME_ENV = ENV["MODEL_NAME"] || ""
+# Derive a default model name from the GGUF basename when not given.
+# basename: strip everything before the last "/"; then strip ".gguf".
+_mn_last_slash = GGUF_PATH.rindex("/")
+_mn_basename   = _mn_last_slash == nil ? GGUF_PATH : GGUF_PATH[_mn_last_slash + 1..-1]
+_mn_dot_gguf   = _mn_basename.rindex(".gguf")
+_mn_default    = _mn_dot_gguf == nil ? _mn_basename : _mn_basename[0...(_mn_dot_gguf)]
+MODEL_NAME = MODEL_NAME_ENV.length > 0 ? MODEL_NAME_ENV : _mn_default
+MAX_T      = (ENV["MAX_T"] || "256").to_i
+
+# ---- Inference state. Class instance held as a CONSTANT so spinel
+#      emits a typed slot for it (same pattern as inference_api.rb /
+#      openai_api.rb). ----
+
+class State
+  # :req_seq is a monomorphic Integer slot (always Integer) for the
+  # per-request monotonic counter the serving events emitter uses as a
+  # deterministic-shaped request_id. Held on the already-CONSTANT-held STATE
+  # so Spinel emits a typed slot; NO new polymorphic accessor (landmine #16).
+  attr_accessor :cfg, :kv, :gguf, :model_name, :ready, :req_seq
+  def initialize
+    @cfg  = nil
+    @kv   = nil
+    @gguf = TinyNN.tnn_null_ptr
+    @model_name = MODEL_NAME
+    @ready      = false
+    @req_seq    = 0
+  end
+end
+STATE = State.new
+# GH#188 -- explicit re-assignment AFTER STATE.new. State#initialize
+# captures `@model_name = MODEL_NAME` but Spinel's ivar typing
+# appears to evaluate it before the MODEL_NAME ternary resolves
+# (the embeddings handler receives MODEL_NAME via constructor at
+# boot and prints it correctly; STATE.model_name accessed at
+# request time shows empty). Setting it explicitly here works
+# around it cleanly without touching the State class.
+STATE.model_name = MODEL_NAME
+
+puts "[openai_api_llama] loading config from " + GGUF_PATH
+STATE.cfg = SmolLM2ConfigLoader.read(GGUF_PATH)
+puts "[openai_api_llama] vocab=" + STATE.cfg.vocab.to_s +
+     " d=" + STATE.cfg.d_model.to_s +
+     " L=" + STATE.cfg.n_layers.to_s +
+     " n_heads=" + STATE.cfg.n_heads.to_s +
+     " n_kv=" + STATE.cfg.n_kv.to_s
+
+flags = GGUFLoad.detect_smollm2_flags(GGUF_PATH)
+puts "[openai_api_llama] flags: untied=" + flags.untied.to_s +
+     " qkv_bias=" + flags.qkv_bias.to_s
+
+# Auto-dispatch: native GGUFs get BYO-pointer mmap (Phase 2);
+# legacy GGUFs fall through to realize_for + load_weights copy.
+# Regenerate with `--ggml-native` (and optionally `--quantize q8_0`)
+# to unlock the mmap fast path on this binary.
+puts "[openai_api_llama] realising (MAX_T=" + MAX_T.to_s + ")..."
+STATE.kv = SmolLM2KVFFICache.new
+STATE.gguf = STATE.kv.realize_and_load_auto(GGUF_PATH, MAX_T, STATE.cfg, flags)
+
+STATE.ready = true
+puts "[openai_api_llama] ready; serving"
+
+# ---- Helpers ----
+
+def api_now_unix
+  Time.now.to_i
+end
+
+def api_gen_id(prefix)
+  t = Time.now
+  v = (t.to_i * 1_000_003) ^ ((t.to_f - t.to_i).to_f * 1.0e9).to_i
+  prefix + "-" + v.to_s
+end
+
+# Greedy generation from a pre-tokenized prompt. KV-cache decode:
+# prefill the prompt one step at a time, then sample greedily for
+# `n_new` more steps. Returns Array<Int> of the new token IDs (does
+# NOT include the prompt).
+#
+# This routine re-runs the prefill from position 0 every call; the
+# cache's t_K / t_V tensors are persistent and get overwritten in
+# place. (A future optimisation would be a fast prefix-cache for
+# shared prompts.)
+def api_generate_ids(prompt_ids, n_new)
+  out_ids = [0]
+  out_ids.pop
+
+  vocab = STATE.cfg.vocab
+  last_logits = Mat.new(1, vocab)
+  prefill_pos = 0
+  while prefill_pos < prompt_ids.length
+    last_logits = SmolLM2KV.decode_step(STATE.kv, prompt_ids[prefill_pos], prefill_pos)
+    prefill_pos = prefill_pos + 1
+  end
+
+  # First generated token comes from the last prefill step's logits.
+  best_idx = 0
+  best_val = last_logits.flat[0]
+  v_iter = 1
+  while v_iter < vocab
+    val = last_logits.flat[v_iter]
+    if val > best_val; best_val = val; best_idx = v_iter; end
+    v_iter = v_iter + 1
+  end
+  out_ids.push(best_idx)
+
+  step = 1
+  while step < n_new
+    last_logits = SmolLM2KV.decode_step(STATE.kv, out_ids[out_ids.length - 1],
+                                         prompt_ids.length + out_ids.length - 1)
+    best_idx = 0
+    best_val = last_logits.flat[0]
+    v_iter = 1
+    while v_iter < vocab
+      val = last_logits.flat[v_iter]
+      if val > best_val; best_val = val; best_idx = v_iter; end
+      v_iter = v_iter + 1
+    end
+    out_ids.push(best_idx)
+    step = step + 1
+  end
+  out_ids
+end