nanogpt 0.2.0 → 0.3.0

data/exe/nanogpt

@@ -6,7 +6,7 @@ $stdout.sync = true
 require "nano_gpt"
 
 class NanoGPTCLI
-  COMMANDS = %w[prepare train sample bench version help].freeze
+  COMMANDS = %w[prepare train sample bench web check version help].freeze
 
   def initialize(args)
     @command = args.shift
@@ -23,6 +23,10 @@ class NanoGPTCLI
       sample
     when "bench"
       bench
+    when "web"
+      web
+    when "check"
+      check
     when "version", "-v", "--version"
       version
     when "help", "-h", "--help", nil
@@ -40,7 +44,6 @@ class NanoGPTCLI
   def prepare
     dataset = @args.first
 
-    # Find available datasets
     data_dir = File.join(File.dirname(__FILE__), "..", "data")
     available = Dir.glob(File.join(data_dir, "*", "prepare.rb")).map do |path|
       File.basename(File.dirname(path))
@@ -77,7 +80,6 @@ class NanoGPTCLI
       exit 1
     end
 
-    # Set output directory to current working directory
     output_dir = File.join(Dir.pwd, "data", dataset)
     ENV["NANOGPT_DATA_DIR"] = output_dir
 
@@ -87,10 +89,6 @@ class NanoGPTCLI
   end
 
   def prepare_textfile
-    require "numo/narray"
-    require "json"
-    require "fileutils"
-
     input_path = nil
     output_name = nil
     val_ratio = 0.1
@@ -116,144 +114,15 @@ class NanoGPTCLI
       exit 1
     end
 
-    unless File.exist?(input_path)
-      puts "Error: File not found: #{input_path}"
-      exit 1
-    end
-
-    output_name ||= File.basename(input_path, ".*").gsub(/[^a-zA-Z0-9_-]/, "_")
-    output_dir = File.join(Dir.pwd, "data", output_name)
-    FileUtils.mkdir_p(output_dir)
-
-    file_size = File.size(input_path)
-    puts "Preparing text file: #{input_path}"
-    puts "File size: #{(file_size / 1_000_000.0).round(2)} MB"
-    puts "Output directory: #{output_dir}"
-    puts "Validation ratio: #{val_ratio}"
-    puts ""
-
-    # Phase 1: Build vocabulary by reading entire file
-    # For very large files, we read line by line to avoid memory issues
-    puts "Phase 1: Building vocabulary..."
-    char_set = Set.new
-    char_count = 0
-
-    # Detect encoding: check if file is valid UTF-8, otherwise assume Windows-1252
-    sample = File.binread(input_path, 100_000)
-    encoding = sample.force_encoding("UTF-8").valid_encoding? ? "UTF-8" : "Windows-1252:UTF-8"
-    puts "  Detected encoding: #{encoding.split(':').first}"
-
-    File.foreach(input_path, encoding: encoding) do |line|
-      line.each_char { |c| char_set.add(c) }
-      char_count += line.length
-      print "\r  Scanned #{char_count} characters, #{char_set.size} unique..." if (char_count % 100_000) < 1000
-    end
-    puts "\r  Scanned #{char_count} characters, #{char_set.size} unique..."
-
-    chars = char_set.to_a.sort
-    vocab_size = chars.size
-    puts "Vocabulary size: #{vocab_size}"
-
-    stoi = chars.each_with_index.to_h
-    itos = chars.each_with_index.map { |c, i| [i, c] }.to_h
-
-    # Phase 2: Calculate split point
-    total_chars = char_count
-    val_chars = (total_chars * val_ratio).to_i
-    train_chars = total_chars - val_chars
-    puts ""
-    puts "Train: #{train_chars} characters"
-    puts "Val: #{val_chars} characters"
-
-    # Phase 3: Encode and write train.bin (streaming line by line)
-    puts ""
-    puts "Phase 2: Encoding and writing train.bin..."
-    train_path = File.join(output_dir, "train.bin")
-    chars_written = 0
-    buffer = []
-    buffer_size = 100_000
-
-    File.open(train_path, "wb") do |output|
-      File.foreach(input_path, encoding: encoding) do |line|
-        line.each_char do |c|
-          break if chars_written >= train_chars
-
-          buffer << stoi[c]
-          chars_written += 1
-
-          if buffer.size >= buffer_size
-            arr = Numo::UInt16.cast(buffer)
-            output.write(arr.to_binary)
-            buffer.clear
-            print "\r  Written #{chars_written}/#{train_chars} characters..."
-          end
-        end
-        break if chars_written >= train_chars
-      end
-
-      unless buffer.empty?
-        arr = Numo::UInt16.cast(buffer)
-        output.write(arr.to_binary)
-        buffer.clear
-      end
-    end
-    puts ""
-
-    # Phase 4: Encode and write val.bin (streaming line by line)
-    puts "Phase 3: Encoding and writing val.bin..."
-    val_path = File.join(output_dir, "val.bin")
-    chars_written = 0
-    skipped = 0
-    buffer = []
-
-    File.open(val_path, "wb") do |output|
-      File.foreach(input_path, encoding: encoding) do |line|
-        line.each_char do |c|
-          if skipped < train_chars
-            skipped += 1
-            next
-          end
-
-          buffer << stoi[c]
-          chars_written += 1
-
-          if buffer.size >= buffer_size
-            arr = Numo::UInt16.cast(buffer)
-            output.write(arr.to_binary)
-            buffer.clear
-            print "\r  Written #{chars_written}/#{val_chars} characters..."
-          end
-        end
-      end
-
-      unless buffer.empty?
-        arr = Numo::UInt16.cast(buffer)
-        output.write(arr.to_binary)
-        buffer.clear
-      end
-    end
-    puts ""
-
-    # Phase 5: Save meta.json
-    puts "Phase 4: Saving meta.json..."
-    meta = {
-      "vocab_size" => vocab_size,
-      "itos" => itos.transform_keys(&:to_s),
-      "stoi" => stoi
-    }
-    File.write(File.join(output_dir, "meta.json"), JSON.pretty_generate(meta))
-
-    train_size_mb = File.size(train_path) / 1_000_000.0
-    val_size_mb = File.size(val_path) / 1_000_000.0
-
-    puts ""
-    puts "Done!"
-    puts "  train.bin: #{train_chars} tokens (#{train_size_mb.round(2)} MB)"
-    puts "  val.bin: #{val_chars} tokens (#{val_size_mb.round(2)} MB)"
-    puts "  meta.json: vocab_size=#{vocab_size}"
-    puts ""
-    puts "To train:"
-    puts "  nanogpt train --dataset=#{output_name}"
+    preparer = NanoGPT::TextfilePreparer.new(
+      input_path: input_path,
+      output_name: output_name,
+      val_ratio: val_ratio
+    )
+    preparer.prepare
+  rescue RuntimeError => e
+    puts "Error: #{e.message}"
+    exit 1
   end
 
   def train
@@ -359,7 +228,7 @@ class NanoGPTCLI
     puts "Generating #{config[:num_samples]} samples..."
     puts "=" * 50
 
-    config[:num_samples].times do |k|
+    config[:num_samples].times do |_k|
       y = model.generate(
         x,
         config[:max_new_tokens],
@@ -374,110 +243,58 @@ class NanoGPTCLI
   end
 
   def bench
-    bench_config = {
-      batch_size: 12,
-      block_size: 1024,
-      n_layer: 12,
-      n_head: 12,
-      n_embd: 768,
-      dropout: 0.0,
-      bias: false,
-      real_data: true,
-      dataset: "openwebtext",
-      seed: 1337,
-      device: "auto"
-    }
-
-    # Parse args
-    @args.each do |arg|
-      next unless arg.start_with?("--") && arg.include?("=")
-
-      key, val = arg[2..].split("=", 2)
-      key = key.to_sym
-
-      next unless bench_config.key?(key)
-
-      bench_config[key] = case bench_config[key]
-                          when Integer then val.to_i
-                          when Float then val.to_f
-                          when TrueClass, FalseClass then val.downcase == "true"
-                          else val
-                          end
-    end
+    config = NanoGPT::BenchConfig.load(@args)
 
     puts "=" * 60
     puts "NanoGPT Benchmark"
     puts "=" * 60
     puts ""
     puts "Configuration:"
-    puts "  batch_size: #{bench_config[:batch_size]}"
-    puts "  block_size: #{bench_config[:block_size]}"
-    puts "  n_layer: #{bench_config[:n_layer]}"
-    puts "  n_head: #{bench_config[:n_head]}"
-    puts "  n_embd: #{bench_config[:n_embd]}"
-    puts "  real_data: #{bench_config[:real_data]}"
+    puts "  batch_size: #{config[:batch_size]}"
+    puts "  block_size: #{config[:block_size]}"
+    puts "  n_layer: #{config[:n_layer]}"
+    puts "  n_head: #{config[:n_head]}"
+    puts "  n_embd: #{config[:n_embd]}"
+    puts "  real_data: #{config[:real_data]}"
     puts ""
 
-    if bench_config[:device] == "auto"
-      bench_config[:device] = NanoGPT::Device.auto
+    if config[:device] == "auto"
+      config[:device] = NanoGPT::Device.auto
     end
-    device = bench_config[:device]
+    device = config[:device]
     puts "Device: #{device}"
 
-    Torch.manual_seed(bench_config[:seed])
+    Torch.manual_seed(config[:seed])
 
-    if bench_config[:real_data]
-      data_dir = File.join("data", bench_config[:dataset])
+    if config[:real_data]
+      data_dir = File.join("data", config[:dataset])
       train_bin = File.join(data_dir, "train.bin")
 
       unless File.exist?(train_bin)
         puts ""
         puts "Warning: #{train_bin} not found, using random data instead."
-        puts "To use real data, run: bundle exec ruby data/#{bench_config[:dataset]}/prepare.rb"
+        puts "To use real data, run: bundle exec ruby data/#{config[:dataset]}/prepare.rb"
         puts ""
-        bench_config[:real_data] = false
+        config[:real_data] = false
       end
     end
 
-    if bench_config[:real_data]
-      bytes = File.binread(File.join("data", bench_config[:dataset], "train.bin"))
-      train_data = bytes.unpack("S<*")
-      puts "Loaded #{train_data.size} tokens from #{bench_config[:dataset]}"
-
-      get_batch = lambda do
-        max_start = train_data.size - bench_config[:block_size] - 1
-        indices = Array.new(bench_config[:batch_size]) { rand(0..max_start) }
-        x_arrays = indices.map { |i| train_data[i, bench_config[:block_size]] }
-        y_arrays = indices.map { |i| train_data[i + 1, bench_config[:block_size]] }
-        x = Torch.tensor(x_arrays, dtype: :long)
-        y = Torch.tensor(y_arrays, dtype: :long)
-        x = x.to(device) if device != "cpu"
-        y = y.to(device) if device != "cpu"
-        [x, y]
-      end
-    else
-      vocab_size = 50304
-      puts "Using random data (vocab_size=#{vocab_size})"
-
-      get_batch = lambda do
-        x = Torch.randint(vocab_size, [bench_config[:batch_size], bench_config[:block_size]], dtype: :long)
-        y = Torch.randint(vocab_size, [bench_config[:batch_size], bench_config[:block_size]], dtype: :long)
-        x = x.to(device) if device != "cpu"
-        y = y.to(device) if device != "cpu"
-        [x, y]
-      end
-    end
+    get_batch = if config[:real_data]
+                  create_real_data_batch_fn(config, device)
+                else
+                  create_random_data_batch_fn(config, device)
+                end
 
     puts ""
     puts "Initializing model..."
     model_config = NanoGPT::GPTConfig.new(
-      block_size: bench_config[:block_size],
+      block_size: config[:block_size],
       vocab_size: 50304,
-      n_layer: bench_config[:n_layer],
-      n_head: bench_config[:n_head],
-      n_embd: bench_config[:n_embd],
-      dropout: bench_config[:dropout],
-      bias: bench_config[:bias]
+      n_layer: config[:n_layer],
+      n_head: config[:n_head],
+      n_embd: config[:n_embd],
+      dropout: config[:dropout],
+      bias: config[:bias]
     )
 
     model = NanoGPT::GPT.new(model_config)
@@ -494,6 +311,41 @@ class NanoGPTCLI
     puts "Starting benchmark..."
     puts "-" * 60
 
+    run_benchmark_phases(model, optimizer, get_batch, config)
+  end
+
+  def create_real_data_batch_fn(config, device)
+    bytes = File.binread(File.join("data", config[:dataset], "train.bin"))
+    train_data = bytes.unpack("S<*")
+    puts "Loaded #{train_data.size} tokens from #{config[:dataset]}"
+
+    lambda do
+      max_start = train_data.size - config[:block_size] - 1
+      indices = Array.new(config[:batch_size]) { rand(0..max_start) }
+      x_arrays = indices.map { |i| train_data[i, config[:block_size]] }
+      y_arrays = indices.map { |i| train_data[i + 1, config[:block_size]] }
+      x = Torch.tensor(x_arrays, dtype: :long)
+      y = Torch.tensor(y_arrays, dtype: :long)
+      x = x.to(device) if device != "cpu"
+      y = y.to(device) if device != "cpu"
+      [x, y]
+    end
+  end
+
+  def create_random_data_batch_fn(config, device)
+    vocab_size = 50304
+    puts "Using random data (vocab_size=#{vocab_size})"
+
+    lambda do
+      x = Torch.randint(vocab_size, [config[:batch_size], config[:block_size]], dtype: :long)
+      y = Torch.randint(vocab_size, [config[:batch_size], config[:block_size]], dtype: :long)
+      x = x.to(device) if device != "cpu"
+      y = y.to(device) if device != "cpu"
+      [x, y]
+    end
+  end
+
+  def run_benchmark_phases(model, optimizer, get_batch, config)
     [{ name: "burn-in", steps: 10 }, { name: "benchmark", steps: 20 }].each do |phase|
       puts ""
       puts "Phase: #{phase[:name]} (#{phase[:steps]} steps)"
@@ -514,18 +366,140 @@ class NanoGPTCLI
       t1 = Time.now
       dt = t1 - t0
 
-      if phase[:name] == "benchmark"
-        mfu = model.estimate_mfu(bench_config[:batch_size] * phase[:steps], dt)
-        time_per_iter = dt / phase[:steps] * 1000
+      next unless phase[:name] == "benchmark"
 
-        puts ""
-        puts "=" * 60
-        puts "Results:"
-        puts "  Time per iteration: #{format('%.2f', time_per_iter)}ms"
-        puts "  MFU: #{format('%.2f', mfu * 100)}%"
-        puts "=" * 60
+      mfu = model.estimate_mfu(config[:batch_size] * phase[:steps], dt)
+      time_per_iter = dt / phase[:steps] * 1000
+
+      puts ""
+      puts "=" * 60
+      puts "Results:"
+      puts "  Time per iteration: #{format('%.2f', time_per_iter)}ms"
+      puts "  MFU: #{format('%.2f', mfu * 100)}%"
+      puts "=" * 60
+    end
+  end
+
+  def web
+    require "nano_gpt/web"
+
+    port = 4567
+    @args.each do |arg|
+      if arg.start_with?("--port=")
+        port = arg.split("=", 2).last.to_i
       end
     end
+
+    training_state = NanoGPT::Web::TrainingState.new
+    metrics_store = NanoGPT::Web::MetricsStore.new
+    sse_notifier = NanoGPT::Web::SSENotifier.new
+
+    worker = NanoGPT::Web::TrainingWorker.new(
+      training_state: training_state,
+      metrics_store: metrics_store,
+      sse_notifier: sse_notifier
+    )
+
+    NanoGPT::Web::Server.training_state = training_state
+    NanoGPT::Web::Server.metrics_store = metrics_store
+    NanoGPT::Web::Server.sse_notifier = sse_notifier
+    NanoGPT::Web::Server.training_worker = worker
+
+    puts "Starting nanoGPT web dashboard on http://localhost:#{port}"
+    puts "Device: #{NanoGPT::Device.auto}"
+
+    # Web server runs in a background thread; the main thread is reserved
+    # for Torch operations (processed via the training worker queue).
+    Thread.new { NanoGPT::Web::Server.run!(port: port, bind: "0.0.0.0") }
+    sleep 1
+    worker.run
+  end
+
+  def check
+    puts "nanoGPT environment check"
+    puts "=" * 40
+    puts ""
+
+    # Ruby version
+    puts "Ruby: #{RUBY_VERSION} (#{RUBY_PLATFORM})"
+    puts ""
+
+    # Torch availability
+    print "torch-rb: "
+    begin
+      require "torch"
+      puts "#{Torch::VERSION} OK"
+    rescue LoadError => e
+      puts "FAILED -- #{e.message}"
+      puts "  Install with: gem install torch-rb"
+      puts ""
+      puts "Check complete (with errors)."
+      return
+    end
+    puts ""
+
+    # Device detection
+    puts "Devices:"
+    puts "  CPU: always available"
+
+    mps_available = begin
+      Torch::Backends::MPS.available?
+    rescue
+      false
+    end
+    puts "  MPS: #{mps_available ? 'available' : 'not available'}"
+
+    cuda_available = begin
+      Torch::CUDA.available?
+    rescue
+      false
+    end
+    puts "  CUDA: #{cuda_available ? 'available' : 'not available'}"
+
+    device = NanoGPT::Device.auto
+    puts ""
+    puts "  Selected device: #{device}"
+    puts ""
+
+    # Basic tensor operation
+    print "Tensor ops (CPU): "
+    begin
+      a = Torch.tensor([1.0, 2.0, 3.0])
+      b = Torch.tensor([4.0, 5.0, 6.0])
+      c = a + b
+      raise "unexpected result" unless c.to_a == [5.0, 7.0, 9.0]
+      puts "OK"
+    rescue => e
+      puts "FAILED -- #{e.message}"
+    end
+
+    # Test on selected device
+    if device != "cpu"
+      print "Tensor ops (#{device}): "
+      begin
+        a = Torch.tensor([1.0, 2.0, 3.0], device: device)
+        b = Torch.tensor([4.0, 5.0, 6.0], device: device)
+        c = (a + b).cpu
+        raise "unexpected result" unless c.to_a == [5.0, 7.0, 9.0]
+        puts "OK"
+      rescue => e
+        puts "FAILED -- #{e.message}"
+      end
+    end
+
+    # Matmul test (more representative of model workload)
+    print "Matrix multiply (#{device}): "
+    begin
+      m = Torch.randn(64, 384, device: device)
+      w = Torch.randn(384, 384, device: device)
+      _result = Torch.matmul(m, w)
+      puts "OK"
+    rescue => e
+      puts "FAILED -- #{e.message}"
+    end
+
+    puts ""
+    puts "All checks passed. Ready to train!"
   end
 
   def version
@@ -543,6 +517,8 @@ class NanoGPTCLI
         train     Train a GPT model
         sample    Generate text from a trained model
         bench     Run performance benchmarks
+        web       Start the web dashboard
+        check     Verify environment (torch, CUDA/MPS, tensor ops)
         version   Show version
         help      Show this help message
 
@@ -551,6 +527,7 @@ class NanoGPTCLI
         nanogpt train --dataset=shakespeare_char --device=mps
         nanogpt sample --dataset=shakespeare_char --num_samples=3
         nanogpt bench --batch_size=8 --block_size=512
+        nanogpt web --port=4567
 
       For more information, visit: https://github.com/khasinski/nanogpt-rb
     HELP

data/lib/nano_gpt/bpe_textfile_preparer.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "numo/narray"
+require "fileutils"
+
+module NanoGPT
+  # Prepares custom text files for training with GPT-2 BPE tokenization
+  # Mirrors TextfilePreparer but uses tiktoken instead of character-level encoding.
+  # Does NOT write meta.json -- absence triggers GPT-2 auto-detection in Tokenizer.for_dataset.
+  class BPETextfilePreparer
+    BUFFER_SIZE = 100_000
+
+    attr_reader :input_path, :output_dir, :val_ratio
+
+    def initialize(input_path:, output_name: nil, val_ratio: 0.1)
+      @input_path = input_path
+      @val_ratio = val_ratio
+      @output_name = output_name || derive_output_name(input_path)
+      @output_dir = File.join(Dir.pwd, "data", @output_name)
+    end
+
+    def prepare
+      validate_input!
+      FileUtils.mkdir_p(@output_dir)
+
+      print_header
+      encoding = detect_encoding
+      tokens = tokenize_file(encoding)
+      train_tokens, val_tokens = split_tokens(tokens)
+
+      write_bin(File.join(@output_dir, "train.bin"), train_tokens, "train")
+      write_bin(File.join(@output_dir, "val.bin"), val_tokens, "val")
+
+      print_summary(train_tokens.size, val_tokens.size)
+      @output_name
+    end
+
+    private
+
+    def derive_output_name(path)
+      File.basename(path, ".*").gsub(/[^a-zA-Z0-9_-]/, "_")
+    end
+
+    def validate_input!
+      raise "File not found: #{@input_path}" unless File.exist?(@input_path)
+    end
+
+    def print_header
+      file_size = File.size(@input_path)
+      puts "Preparing text file (BPE): #{@input_path}"
+      puts "File size: #{(file_size / 1_000_000.0).round(2)} MB"
+      puts "Output directory: #{@output_dir}"
+      puts "Validation ratio: #{@val_ratio}"
+      puts ""
+    end
+
+    def detect_encoding
+      sample = File.binread(@input_path, 100_000)
+      encoding = sample.force_encoding("UTF-8").valid_encoding? ? "UTF-8" : "Windows-1252:UTF-8"
+      puts "  Detected encoding: #{encoding.split(':').first}"
+      encoding
+    end
+
+    def tokenize_file(encoding)
+      puts "Phase 1: Tokenizing with GPT-2 BPE..."
+      require "tiktoken_ruby"
+      enc = Tiktoken.get_encoding(:r50k_base)
+
+      text = File.read(@input_path, encoding: encoding)
+      tokens = enc.encode(text)
+      puts "  #{tokens.size} tokens from #{text.length} characters"
+      tokens
+    end
+
+    def split_tokens(tokens)
+      val_count = (tokens.size * @val_ratio).to_i
+      train_count = tokens.size - val_count
+      puts ""
+      puts "Train: #{train_count} tokens"
+      puts "Val: #{val_count} tokens"
+      [tokens[0...train_count], tokens[train_count..]]
+    end
+
+    def write_bin(path, tokens, label)
+      puts ""
+      puts "Phase 2: Writing #{label}.bin..."
+      arr = Numo::UInt16.cast(tokens)
+      File.binwrite(path, arr.to_binary)
+    end
+
+    def print_summary(train_count, val_count)
+      train_size_mb = File.size(File.join(@output_dir, "train.bin")) / 1_000_000.0
+      val_size_mb = File.size(File.join(@output_dir, "val.bin")) / 1_000_000.0
+
+      puts ""
+      puts "Done!"
+      puts "  train.bin: #{train_count} tokens (#{train_size_mb.round(2)} MB)"
+      puts "  val.bin: #{val_count} tokens (#{val_size_mb.round(2)} MB)"
+      puts "  No meta.json (GPT-2 tokenizer auto-detected)"
+      puts ""
+      puts "To train:"
+      puts "  nanogpt train --dataset=#{@output_name}"
+    end
+  end
+end