nanogpt 0.2.0 → 0.3.0

data/lib/nano_gpt/web/views/runs.erb

@@ -0,0 +1,434 @@
+<!-- Header -->
+<div class="flex items-center justify-between">
+  <h1 class="text-lg font-semibold">Runs</h1>
+  <div class="flex gap-2">
+    <button onclick="toggleImportModal()" class="px-3 py-2 bg-gray-700 hover:bg-gray-600 rounded-lg text-sm transition">
+      Import Dataset
+    </button>
+    <button onclick="openNewRunModal()" class="px-4 py-2 bg-green-600 hover:bg-green-700 rounded-lg font-medium text-sm transition">
+      + New Run
+    </button>
+  </div>
+</div>
+
+<!-- Runs List -->
+<div id="runs-list" class="space-y-2">
+  <div class="text-center text-gray-500 py-8">Loading runs...</div>
+</div>
+
+<!-- New Run Modal -->
+<div id="new-run-modal" class="fixed inset-0 bg-black/60 z-50 hidden flex items-start justify-center pt-16 px-4">
+  <div class="bg-gray-800 rounded-xl shadow-2xl w-full max-w-2xl max-h-[80vh] overflow-y-auto">
+    <div class="flex items-center justify-between px-6 py-4 border-b border-gray-700">
+      <h2 class="text-base font-semibold">New Training Run</h2>
+      <button onclick="closeNewRunModal()" class="text-gray-400 hover:text-white text-xl leading-none">&times;</button>
+    </div>
+
+    <div class="px-6 py-5 space-y-5">
+      <!-- Dataset -->
+      <div>
+        <label class="block text-xs text-gray-400 mb-1.5">Dataset</label>
+        <select id="cfg-dataset" class="w-full bg-gray-700 rounded px-3 py-2 text-sm"></select>
+      </div>
+
+      <!-- Basic -->
+      <div>
+        <div class="text-xs text-gray-400 uppercase tracking-wide mb-2">Training</div>
+        <div class="grid grid-cols-3 gap-3">
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Max Iters</label>
+            <input id="cfg-max_iters" type="number" value="5000" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Learning Rate</label>
+            <input id="cfg-learning_rate" type="number" value="0.001" step="0.0001" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Batch Size</label>
+            <input id="cfg-batch_size" type="number" value="64" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+        </div>
+      </div>
+
+      <!-- Model -->
+      <div>
+        <div class="text-xs text-gray-400 uppercase tracking-wide mb-2">Model</div>
+        <div class="grid grid-cols-3 gap-3">
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Block Size</label>
+            <input id="cfg-block_size" type="number" value="256" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Layers</label>
+            <input id="cfg-n_layer" type="number" value="6" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Heads</label>
+            <input id="cfg-n_head" type="number" value="6" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Embedding Dim</label>
+            <input id="cfg-n_embd" type="number" value="384" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Dropout</label>
+            <input id="cfg-dropout" type="number" value="0.2" step="0.05" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+        </div>
+      </div>
+
+      <!-- Advanced (collapsed) -->
+      <details>
+        <summary class="text-xs text-gray-400 uppercase tracking-wide cursor-pointer hover:text-gray-300">Advanced</summary>
+        <div class="grid grid-cols-3 gap-3 mt-3">
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Eval Interval</label>
+            <input id="cfg-eval_interval" type="number" value="250" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Log Interval</label>
+            <input id="cfg-log_interval" type="number" value="10" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Eval Iters</label>
+            <input id="cfg-eval_iters" type="number" value="20" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Grad Accumulation</label>
+            <input id="cfg-gradient_accumulation_steps" type="number" value="1" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Weight Decay</label>
+            <input id="cfg-weight_decay" type="number" value="0.1" step="0.01" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Grad Clip</label>
+            <input id="cfg-grad_clip" type="number" value="1.0" step="0.1" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Warmup Iters</label>
+            <input id="cfg-warmup_iters" type="number" value="100" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">LR Decay Iters</label>
+            <input id="cfg-lr_decay_iters" type="number" value="5000" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Min LR</label>
+            <input id="cfg-min_lr" type="number" value="0.0001" step="0.0001" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Bias</label>
+            <select id="cfg-bias" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+              <option value="false" selected>false</option>
+              <option value="true">true</option>
+            </select>
+          </div>
+          <div>
+            <label class="block text-xs text-gray-400 mb-1">Always Save Ckpt</label>
+            <select id="cfg-always_save_checkpoint" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+              <option value="false" selected>false</option>
+              <option value="true">true</option>
+            </select>
+          </div>
+        </div>
+      </details>
+    </div>
+
+    <div class="px-6 py-4 border-t border-gray-700 flex justify-end gap-3">
+      <button onclick="closeNewRunModal()" class="px-4 py-2 bg-gray-700 hover:bg-gray-600 rounded-lg text-sm transition">Cancel</button>
+      <button id="btn-start-run" onclick="startRun()" class="px-5 py-2 bg-green-600 hover:bg-green-700 rounded-lg font-medium text-sm transition">
+        Start Training
+      </button>
+    </div>
+  </div>
+</div>
+
+<!-- Import Dataset Modal -->
+<div id="import-modal" class="fixed inset-0 bg-black/60 z-50 hidden flex items-start justify-center pt-16 px-4">
+  <div class="bg-gray-800 rounded-xl shadow-2xl w-full max-w-lg">
+    <div class="flex items-center justify-between px-6 py-4 border-b border-gray-700">
+      <h2 class="text-base font-semibold">Import Dataset</h2>
+      <button onclick="toggleImportModal()" class="text-gray-400 hover:text-white text-xl leading-none">&times;</button>
+    </div>
+    <form id="upload-form" enctype="multipart/form-data" class="px-6 py-5 space-y-4">
+      <div>
+        <label class="block text-xs text-gray-400 mb-1">Text File</label>
+        <input id="import-file" name="file" type="file" accept=".txt,.text" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm file:mr-3 file:py-1 file:px-3 file:rounded file:border-0 file:text-sm file:bg-gray-600 file:text-gray-200">
+      </div>
+      <div>
+        <label class="block text-xs text-gray-400 mb-1">Dataset Name</label>
+        <input id="import-name" name="name" type="text" placeholder="auto from filename" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+      </div>
+      <div>
+        <label class="block text-xs text-gray-400 mb-1">Tokenizer</label>
+        <div class="flex gap-4 mt-1">
+          <label class="flex items-center gap-1.5 text-sm"><input type="radio" name="tokenizer" value="char" checked class="accent-green-500"> Char</label>
+          <label class="flex items-center gap-1.5 text-sm"><input type="radio" name="tokenizer" value="bpe" class="accent-green-500"> BPE (GPT-2)</label>
+        </div>
+      </div>
+      <div>
+        <label class="block text-xs text-gray-400 mb-1">Val Split</label>
+        <input id="import-val-ratio" name="val_ratio" type="number" value="0.1" step="0.05" min="0" max="0.5" class="w-full bg-gray-700 rounded px-2 py-1.5 text-sm">
+      </div>
+    </form>
+    <div class="px-6 py-4 border-t border-gray-700 flex items-center justify-between">
+      <span id="import-status" class="text-xs text-gray-400"></span>
+      <button id="btn-upload" onclick="uploadDataset()" class="px-4 py-2 bg-blue-600 hover:bg-blue-700 rounded-lg font-medium text-sm transition">
+        Upload & Prepare
+      </button>
+    </div>
+  </div>
+</div>
+
+<script>
+  let runsData = [];
+  let currentTrainingRunId = null;
+
+  // ---- Load Runs ----
+  async function loadRuns() {
+    try {
+      const res = await fetch('/api/runs');
+      const data = await res.json();
+      runsData = data.runs || [];
+      renderRuns();
+    } catch (e) {
+      document.getElementById('runs-list').innerHTML =
+        '<div class="text-center text-red-400 py-8">Failed to load runs</div>';
+    }
+  }
+
+  function renderRuns() {
+    const container = document.getElementById('runs-list');
+    if (runsData.length === 0) {
+      container.innerHTML = '<div class="text-center text-gray-500 py-12">No runs yet. Click "+ New Run" to start training.</div>';
+      return;
+    }
+
+    container.innerHTML = runsData.map(r => {
+      const valLoss = r.best_val_loss ? parseFloat(r.best_val_loss).toFixed(4) : '--';
+      const iters = r.current_iter || 0;
+      const config = JSON.parse(r.config_json || '{}');
+      const maxIters = config.max_iters || '?';
+      const isActive = r.status === 'running' && currentTrainingRunId === r.id;
+
+      return `
+        <div class="bg-gray-800 rounded-lg p-4 hover:bg-gray-750 transition cursor-pointer flex items-center justify-between group"
+             onclick="window.location='/runs/${r.id}'">
+          <div class="flex items-center gap-4 min-w-0">
+            <div class="text-sm font-mono text-gray-500 w-8">#${r.id}</div>
+            <div class="min-w-0">
+              <div class="flex items-center gap-2">
+                <span class="font-medium text-sm">${r.dataset}</span>
+                ${statusBadge(r.status)}
+              </div>
+              <div class="text-xs text-gray-500 mt-0.5">
+                ${formatTime(r.started_at)}
+                &middot; ${iters}/${maxIters} iters
+                &middot; val_loss: ${valLoss}
+              </div>
+            </div>
+          </div>
+          <div class="flex items-center gap-2" onclick="event.stopPropagation()">
+            ${r.status === 'running' ? `
+              <button onclick="stopRun()" class="px-3 py-1.5 bg-red-600/80 hover:bg-red-600 rounded text-xs font-medium transition">Stop</button>
+            ` : ''}
+            ${(r.status === 'stopped' || r.status === 'completed') && r.checkpoint_path ? `
+              <button onclick="resumeRun(${r.id}, '${r.dataset}', '${(r.checkpoint_path || '').replace(/'/g, "\\'")}')" class="px-3 py-1.5 bg-yellow-600/80 hover:bg-yellow-600 rounded text-xs font-medium transition">Resume</button>
+            ` : ''}
+            <a href="/runs/${r.id}" class="px-3 py-1.5 bg-gray-700 hover:bg-gray-600 rounded text-xs font-medium transition">View</a>
+          </div>
+        </div>
+      `;
+    }).join('');
+  }
+
+  // ---- Training Status Listener ----
+  window.addEventListener('training-status', (e) => {
+    const state = e.detail;
+    const oldId = currentTrainingRunId;
+    currentTrainingRunId = state.run_id;
+
+    // Re-render if status changed
+    if (runsData.length > 0) {
+      const activeRun = runsData.find(r => r.id === state.run_id);
+      if (activeRun && activeRun.status !== state.status) {
+        activeRun.status = state.status;
+        activeRun.current_iter = state.current_iter;
+        activeRun.best_val_loss = state.best_val_loss;
+        renderRuns();
+      }
+      // If status changed from running to something else, reload to get final state
+      if (oldId && state.status !== 'running') {
+        loadRuns();
+      }
+    }
+  });
+
+  // ---- Modal Controls ----
+  function openNewRunModal() {
+    loadDatasets();
+    document.getElementById('new-run-modal').classList.remove('hidden');
+  }
+
+  function closeNewRunModal() {
+    document.getElementById('new-run-modal').classList.add('hidden');
+  }
+
+  function toggleImportModal() {
+    document.getElementById('import-modal').classList.toggle('hidden');
+  }
+
+  // Close modals on backdrop click
+  document.getElementById('new-run-modal')?.addEventListener('click', (e) => {
+    if (e.target === e.currentTarget) closeNewRunModal();
+  });
+  document.getElementById('import-modal')?.addEventListener('click', (e) => {
+    if (e.target === e.currentTarget) toggleImportModal();
+  });
+
+  // Close modals on Escape
+  document.addEventListener('keydown', (e) => {
+    if (e.key === 'Escape') {
+      closeNewRunModal();
+      document.getElementById('import-modal').classList.add('hidden');
+    }
+  });
+
+  // ---- Config Helpers ----
+  function getConfig() {
+    return {
+      dataset: document.getElementById('cfg-dataset').value,
+      batch_size: parseInt(document.getElementById('cfg-batch_size').value),
+      block_size: parseInt(document.getElementById('cfg-block_size').value),
+      n_layer: parseInt(document.getElementById('cfg-n_layer').value),
+      n_head: parseInt(document.getElementById('cfg-n_head').value),
+      n_embd: parseInt(document.getElementById('cfg-n_embd').value),
+      learning_rate: parseFloat(document.getElementById('cfg-learning_rate').value),
+      max_iters: parseInt(document.getElementById('cfg-max_iters').value),
+      eval_interval: parseInt(document.getElementById('cfg-eval_interval').value),
+      log_interval: parseInt(document.getElementById('cfg-log_interval').value),
+      dropout: parseFloat(document.getElementById('cfg-dropout').value),
+      gradient_accumulation_steps: parseInt(document.getElementById('cfg-gradient_accumulation_steps').value),
+      eval_iters: parseInt(document.getElementById('cfg-eval_iters').value),
+      weight_decay: parseFloat(document.getElementById('cfg-weight_decay').value),
+      grad_clip: parseFloat(document.getElementById('cfg-grad_clip').value),
+      warmup_iters: parseInt(document.getElementById('cfg-warmup_iters').value),
+      lr_decay_iters: parseInt(document.getElementById('cfg-lr_decay_iters').value),
+      min_lr: parseFloat(document.getElementById('cfg-min_lr').value),
+      bias: document.getElementById('cfg-bias').value === 'true',
+      always_save_checkpoint: document.getElementById('cfg-always_save_checkpoint').value === 'true'
+    };
+  }
+
+  // ---- Actions ----
+  async function startRun() {
+    const btn = document.getElementById('btn-start-run');
+    btn.disabled = true;
+    btn.textContent = 'Starting...';
+
+    try {
+      const res = await fetch('/train/start', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(getConfig())
+      });
+      const data = await res.json();
+      if (res.ok) {
+        closeNewRunModal();
+        // Navigate to the new run
+        window.location = '/runs/' + data.run_id;
+      } else {
+        alert(data.error);
+      }
+    } finally {
+      btn.disabled = false;
+      btn.textContent = 'Start Training';
+    }
+  }
+
+  async function stopRun() {
+    await fetch('/train/stop', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: '{}'
+    });
+  }
+
+  async function resumeRun(runId, dataset, checkpointPath) {
+    // Open modal pre-filled, but just start immediately with the same config
+    const run = runsData.find(r => r.id === runId);
+    const config = run ? JSON.parse(run.config_json || '{}') : {};
+    config.checkpoint_path = checkpointPath;
+
+    const res = await fetch('/train/resume', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(config)
+    });
+    const data = await res.json();
+    if (res.ok) {
+      window.location = '/runs/' + data.run_id;
+    } else {
+      alert(data.error);
+    }
+  }
+
+  // ---- Datasets ----
+  async function loadDatasets() {
+    try {
+      const res = await fetch('/datasets');
+      const data = await res.json();
+      const select = document.getElementById('cfg-dataset');
+      select.innerHTML = '';
+      data.datasets.forEach(d => {
+        const opt = document.createElement('option');
+        opt.value = d;
+        opt.textContent = d;
+        select.appendChild(opt);
+      });
+    } catch (e) {}
+  }
+
+  // ---- Import ----
+  document.getElementById('import-file')?.addEventListener('change', function() {
+    const nameInput = document.getElementById('import-name');
+    if (!nameInput.value && this.files[0]) {
+      nameInput.value = this.files[0].name.replace(/\.[^/.]+$/, '').replace(/[^a-zA-Z0-9_-]/g, '_');
+    }
+  });
+
+  async function uploadDataset() {
+    const btn = document.getElementById('btn-upload');
+    const status = document.getElementById('import-status');
+    btn.disabled = true;
+    status.textContent = 'Uploading and preparing...';
+    status.className = 'text-xs text-yellow-400';
+
+    const form = document.getElementById('upload-form');
+    const formData = new FormData(form);
+    try {
+      const res = await fetch('/datasets/upload', { method: 'POST', body: formData });
+      const data = await res.json();
+      if (data.ok) {
+        status.textContent = 'Dataset "' + data.dataset + '" ready!';
+        status.className = 'text-xs text-green-400';
+        setTimeout(() => {
+          document.getElementById('import-modal').classList.add('hidden');
+          status.textContent = '';
+        }, 1500);
+      } else {
+        status.textContent = 'Error: ' + data.error;
+        status.className = 'text-xs text-red-400';
+      }
+    } catch (err) {
+      status.textContent = 'Upload failed: ' + err.message;
+      status.className = 'text-xs text-red-400';
+    } finally {
+      btn.disabled = false;
+    }
+  }
+
+  // ---- Init ----
+  loadRuns();
+</script>

data/lib/nano_gpt/web/web_trainer.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module NanoGPT
+  module Web
+    # Training loop with web dashboard hooks
+    # Composes with the same model/data_loader/config as Trainer
+    # but adds stop flag checking, metric recording, and SSE broadcasting
+    class WebTrainer
+      attr_reader :model, :config, :iter_num, :best_val_loss
+
+      def initialize(model:, data_loader:, config:, training_state:, metrics_store:, sse_notifier:)
+        @model = model
+        @data_loader = data_loader
+        @config = config.is_a?(Hash) ? config.transform_keys(&:to_sym) : config.to_h
+        @training_state = training_state
+        @metrics_store = metrics_store
+        @sse_notifier = sse_notifier
+
+        @iter_num = 0
+        @best_val_loss = Float::INFINITY
+
+        setup_optimizer
+        setup_lr_scheduler
+      end
+
+      def train(run_id)
+        @run_id = run_id
+        @training_state.update(status: "running", run_id: run_id, max_iters: @config[:max_iters])
+
+        puts "Starting training... max_iters=#{@config[:max_iters]} eval_interval=#{@config[:eval_interval]} eval_iters=#{@config[:eval_iters]} device=#{@config[:device]}"
+
+        @model.train
+        x, y = @data_loader.get_batch(:train)
+        t0 = Time.now
+
+        while @iter_num <= @config[:max_iters]
+          # Check stop flag
+          if @training_state.stop_requested?
+            save_checkpoint("final")
+            @training_state.update(status: "stopped")
+            @metrics_store.update_run(run_id, status: "stopped", stopped_at: Time.now.iso8601,
+              checkpoint_path: checkpoint_path("final"))
+            @sse_notifier.broadcast(type: "status", data: @training_state.to_h)
+            return
+          end
+
+          lr = @config[:decay_lr] ? @lr_scheduler.step(@optimizer, @iter_num) : @config[:learning_rate]
+
+          if @iter_num % @config[:eval_interval] == 0
+            puts "iter #{@iter_num}: running eval..."
+            losses = estimate_loss
+            val_loss = losses[:val]
+            train_loss = losses[:train]
+            puts "iter #{@iter_num}: eval done - train_loss=#{format('%.4f', train_loss)} val_loss=#{format('%.4f', val_loss)}"
+
+            @training_state.update(best_val_loss: [@best_val_loss, val_loss].min)
+            @metrics_store.record_metrics(run_id, @iter_num, { val_loss: val_loss, eval_train_loss: train_loss })
+            @sse_notifier.broadcast(type: "eval", data: {
+              iteration: @iter_num, val_loss: val_loss, train_loss: train_loss
+            })
+
+            if val_loss < @best_val_loss || @config[:always_save_checkpoint]
+              @best_val_loss = [val_loss, @best_val_loss].min
+              save_checkpoint("best") if @iter_num > 0
+              @metrics_store.update_run(run_id,
+                best_val_loss: @best_val_loss,
+                checkpoint_path: checkpoint_path("best")
+              )
+            end
+          end
+
+          break if @iter_num == 0 && @config[:eval_only]
+
+          @optimizer.zero_grad
+
+          accumulated_loss = 0.0
+          @config[:gradient_accumulation_steps].times do
+            _logits, loss = @model.call(x, targets: y)
+            loss = loss / @config[:gradient_accumulation_steps]
+            accumulated_loss += loss.item
+            loss.backward
+            x, y = @data_loader.get_batch(:train)
+          end
+
+          clip_grad_norm(@model.parameters, @config[:grad_clip]) if @config[:grad_clip] > 0.0
+          @optimizer.step
+
+          t1 = Time.now
+          dt = t1 - t0
+          t0 = t1
+
+          if @iter_num % @config[:log_interval] == 0
+            @training_state.update(
+              current_iter: @iter_num,
+              current_loss: accumulated_loss
+            )
+            @metrics_store.record_metrics(run_id, @iter_num, { train_loss: accumulated_loss, lr: lr, iter_time_ms: dt * 1000 })
+            @metrics_store.update_run(run_id, current_iter: @iter_num)
+            @sse_notifier.broadcast(type: "train", data: {
+              iteration: @iter_num, loss: accumulated_loss, lr: lr, time_ms: (dt * 1000).round(2)
+            })
+          end
+
+          @iter_num += 1
+        end
+
+        save_checkpoint("final")
+        @training_state.update(status: "completed")
+        @metrics_store.update_run(run_id, status: "completed", stopped_at: Time.now.iso8601,
+          checkpoint_path: checkpoint_path("best"))
+        @sse_notifier.broadcast(type: "status", data: @training_state.to_h)
+      end
+
+      def load_checkpoint(path)
+        checkpoint = Torch.load(path)
+        @model.load_state_dict(checkpoint["model"])
+        @iter_num = checkpoint["iter_num"]
+        @best_val_loss = checkpoint["best_val_loss"]
+        setup_optimizer
+        checkpoint
+      end
+
+      private
+
+      def estimate_loss
+        @model.eval
+        out = {}
+
+        [:train, :val].each do |split|
+          losses = []
+          @config[:eval_iters].times do |i|
+            break if @training_state.stop_requested?
+            x, y = @data_loader.get_batch(split)
+            Torch.no_grad do
+              _logits, loss = @model.call(x, targets: y)
+              losses << loss.item
+            end
+          end
+          out[split] = losses.empty? ? Float::INFINITY : losses.sum / losses.size
+        end
+
+        @model.train
+        out
+      end
+
+      def save_checkpoint(suffix = "best")
+        FileUtils.mkdir_p(@config[:out_dir])
+        path = checkpoint_path(suffix)
+        checkpoint = {
+          "model" => @model.state_dict,
+          "model_args" => stringify_keys(@model.config.to_h),
+          "iter_num" => @iter_num,
+          "best_val_loss" => @best_val_loss,
+          "config" => stringify_keys(@config)
+        }
+        Torch.save(checkpoint, path)
+        @metrics_store.record_checkpoint(@run_id,
+          path: path, suffix: suffix, iteration: @iter_num, val_loss: @best_val_loss)
+      end
+
+      def checkpoint_path(suffix = "best")
+        File.join(@config[:out_dir], "ckpt_run#{@run_id}_#{suffix}.pt")
+      end
+
+      def stringify_keys(hash)
+        hash.transform_keys(&:to_s).transform_values do |v|
+          v.is_a?(Hash) ? stringify_keys(v) : v
+        end
+      end
+
+      def clip_grad_norm(parameters, max_norm)
+        total_norm = 0.0
+        parameters.each do |p|
+          next unless p.grad
+          param_norm = p.grad.data.norm(2).item
+          total_norm += param_norm**2
+        end
+        total_norm = Math.sqrt(total_norm)
+
+        clip_coef = max_norm / (total_norm + 1e-6)
+        if clip_coef < 1
+          parameters.each do |p|
+            next unless p.grad
+            p.grad.data.mul!(clip_coef)
+          end
+        end
+      end
+
+      def setup_optimizer
+        @optimizer = @model.configure_optimizers(
+          weight_decay: @config[:weight_decay] || 1e-1,
+          learning_rate: @config[:learning_rate],
+          betas: [@config[:beta1] || 0.9, @config[:beta2] || 0.99],
+          device_type: @config[:device]
+        )
+      end
+
+      def setup_lr_scheduler
+        @lr_scheduler = LRScheduler.new(
+          learning_rate: @config[:learning_rate],
+          min_lr: @config[:min_lr] || 1e-4,
+          warmup_iters: @config[:warmup_iters] || 100,
+          lr_decay_iters: @config[:lr_decay_iters] || 5000
+        )
+      end
+    end
+  end
+end

data/lib/nano_gpt/web.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "bpe_textfile_preparer"
+require_relative "web/metrics_store"
+require_relative "web/training_state"
+require_relative "web/sse_notifier"
+require_relative "web/web_trainer"
+require_relative "web/training_worker"
+require_relative "web/server"

data/lib/nano_gpt.rb

@@ -16,3 +16,4 @@ require_relative "nano_gpt/data_loader"
 require_relative "nano_gpt/lr_scheduler"
 require_relative "nano_gpt/trainer"
 require_relative "nano_gpt/train_config"
+require_relative "nano_gpt/textfile_preparer"

data/nanogpt.gemspec

@@ -32,6 +32,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency "torch-rb", "~> 0.14"
   spec.add_dependency "numo-narray", "~> 0.9"
   spec.add_dependency "tiktoken_ruby", "~> 0.0"
+  spec.add_dependency "sinatra", "~> 4.0"
+  spec.add_dependency "rackup", "~> 2.0"
+  spec.add_dependency "webrick", "~> 1.8"
+  spec.add_dependency "sqlite3", "~> 2.0"
 
   spec.add_development_dependency "rspec", "~> 3.12"
 end