nanogpt 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b10e62747a63be10b519fdd8eb1eaabfc63214b01ab553d4230af865a0a8f327
-  data.tar.gz: 69db126e3d02c897045e543981223d1d29d72f7625bd381e3ba4badee88b6d20
+  metadata.gz: cf308fcec8ccec074200361b2327a2381a5412b92497391bb71ec5f154cd1283
+  data.tar.gz: 87dcc389df03af0ac59fc0e75bbef7be7071f00613a2662cecf365ba38bc853e
 SHA512:
-  metadata.gz: 97f5846373224b889f22b80302a05271c7ab0276037e519c19c58cc0954aa982a4d3ac9992766aa73f6d1eec1dfe2b96b8a750414dd02818529e37789671cd87
-  data.tar.gz: accd2112564f004cc763dbaac418f35ce24426d4d86e53935a32bdf6411659463ed6cb8e9415146136ee78fa79748e39a1e506b5c44848a7b6e503dfee86e310
+  metadata.gz: 2b6ceeb10236b639c82398c94d3c1a876eff549a17289ff45abae489e480a3d6a1db45f95a4b2e56f1d1df6afde28159965b188342d1e38c2482359dfb11e061
+  data.tar.gz: 0fd4653c2719d1d3c339a904437fd0657f17e49fa0a9d4361a3a259806fab8dc798ed7f179722b41913d3471e8799abf78823f207988987d86cba680d7f90f03

data/data/openwebtext/prepare.rb

@@ -33,7 +33,10 @@ require "rubygems/package"
 require "zlib"
 
 SCRIPT_DIR = File.dirname(__FILE__)
-RAW_DIR = File.join(SCRIPT_DIR, "raw")
+OUTPUT_DIR = ENV["NANOGPT_DATA_DIR"] || SCRIPT_DIR
+RAW_DIR = File.join(OUTPUT_DIR, "raw")
+
+FileUtils.mkdir_p(OUTPUT_DIR) if ENV["NANOGPT_DATA_DIR"]
 DEFAULT_VAL_RATIO = 0.0005  # ~0.5% for validation
 
 def parse_args
@@ -60,11 +63,17 @@ def parse_args
 end
 
 def find_data_files
-  # Look for various supported formats
+  # Look for various supported formats in both SCRIPT_DIR (gem) and OUTPUT_DIR (local)
   patterns = [
     File.join(RAW_DIR, "**", "*.parquet"),  # Parquet (from Python export)
     File.join(RAW_DIR, "**", "*.tar"),       # Original tar files
     File.join(RAW_DIR, "**", "*.txt"),       # Plain text files
+    File.join(OUTPUT_DIR, "*.parquet"),
+    File.join(OUTPUT_DIR, "*.tar"),
+    File.join(OUTPUT_DIR, "*.txt"),
+    File.join(SCRIPT_DIR, "raw", "**", "*.parquet"),
+    File.join(SCRIPT_DIR, "raw", "**", "*.tar"),
+    File.join(SCRIPT_DIR, "raw", "**", "*.txt"),
     File.join(SCRIPT_DIR, "*.parquet"),
     File.join(SCRIPT_DIR, "*.tar"),
     File.join(SCRIPT_DIR, "*.txt")
@@ -273,8 +282,8 @@ def main
 
   # Write binary files
   puts "Writing binary files..."
-  write_binary(train_tokens, File.join(SCRIPT_DIR, "train.bin"))
-  write_binary(val_tokens, File.join(SCRIPT_DIR, "val.bin"))
+  write_binary(train_tokens, File.join(OUTPUT_DIR, "train.bin"))
+  write_binary(val_tokens, File.join(OUTPUT_DIR, "val.bin"))
 
   puts ""
   puts "Done! OpenWebText dataset prepared."

data/data/shakespeare/prepare.rb

@@ -11,9 +11,13 @@ require "net/http"
 require "openssl"
 require "numo/narray"
 require "tiktoken_ruby"
+require "fileutils"
 
 DATA_URL = "https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt"
 SCRIPT_DIR = File.dirname(__FILE__)
+OUTPUT_DIR = ENV["NANOGPT_DATA_DIR"] || SCRIPT_DIR
+
+FileUtils.mkdir_p(OUTPUT_DIR) if ENV["NANOGPT_DATA_DIR"]
 
 def download_file(url)
   uri = URI(url)
@@ -53,8 +57,8 @@ puts "Val has #{val_ids.length} tokens"
 # Export to binary files (uint16)
 train_arr = Numo::UInt16.cast(train_ids)
 val_arr = Numo::UInt16.cast(val_ids)
-File.binwrite(File.join(SCRIPT_DIR, "train.bin"), train_arr.to_binary)
-File.binwrite(File.join(SCRIPT_DIR, "val.bin"), val_arr.to_binary)
+File.binwrite(File.join(OUTPUT_DIR, "train.bin"), train_arr.to_binary)
+File.binwrite(File.join(OUTPUT_DIR, "val.bin"), val_arr.to_binary)
 
 # No meta.json - indicates GPT-2 BPE tokenizer should be used
 puts "Done! Created train.bin and val.bin"

data/data/shakespeare_char/prepare.rb

@@ -8,9 +8,13 @@ require "net/http"
 require "openssl"
 require "numo/narray"
 require "json"
+require "fileutils"
 
 DATA_URL = "https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt"
 SCRIPT_DIR = File.dirname(__FILE__)
+OUTPUT_DIR = ENV["NANOGPT_DATA_DIR"] || SCRIPT_DIR
+
+FileUtils.mkdir_p(OUTPUT_DIR) if ENV["NANOGPT_DATA_DIR"]
 
 def download_file(url)
   uri = URI(url)
@@ -59,8 +63,8 @@ puts "Val has #{val_ids.length} tokens"
 # Export to binary files (uint16)
 train_arr = Numo::UInt16.cast(train_ids)
 val_arr = Numo::UInt16.cast(val_ids)
-File.binwrite(File.join(SCRIPT_DIR, "train.bin"), train_arr.to_binary)
-File.binwrite(File.join(SCRIPT_DIR, "val.bin"), val_arr.to_binary)
+File.binwrite(File.join(OUTPUT_DIR, "train.bin"), train_arr.to_binary)
+File.binwrite(File.join(OUTPUT_DIR, "val.bin"), val_arr.to_binary)
 
 # Save meta information as JSON
 meta = {
@@ -68,6 +72,6 @@ meta = {
   "itos" => itos.transform_keys(&:to_s),
   "stoi" => stoi
 }
-File.write(File.join(SCRIPT_DIR, "meta.json"), JSON.pretty_generate(meta))
+File.write(File.join(OUTPUT_DIR, "meta.json"), JSON.pretty_generate(meta))
 
 puts "Done! Created train.bin, val.bin, and meta.json"

data/exe/nanogpt

@@ -6,7 +6,7 @@ $stdout.sync = true
 require "nano_gpt"
 
 class NanoGPTCLI
-  COMMANDS = %w[train sample bench version help].freeze
+  COMMANDS = %w[prepare train sample bench version help].freeze
 
   def initialize(args)
     @command = args.shift
@@ -15,6 +15,8 @@ class NanoGPTCLI
 
   def run
     case @command
+    when "prepare"
+      prepare
     when "train"
       train
     when "sample"
@@ -35,6 +37,42 @@ class NanoGPTCLI
 
   private
 
+  def prepare
+    dataset = @args.first
+
+    # Find available datasets
+    data_dir = File.join(File.dirname(__FILE__), "..", "data")
+    available = Dir.glob(File.join(data_dir, "*", "prepare.rb")).map do |path|
+      File.basename(File.dirname(path))
+    end.sort
+
+    if dataset.nil?
+      puts "Usage: nanogpt prepare <dataset>"
+      puts ""
+      puts "Available datasets:"
+      available.each { |d| puts "  #{d}" }
+      exit 1
+    end
+
+    prepare_script = File.join(data_dir, dataset, "prepare.rb")
+
+    unless File.exist?(prepare_script)
+      puts "Error: Unknown dataset '#{dataset}'"
+      puts ""
+      puts "Available datasets:"
+      available.each { |d| puts "  #{d}" }
+      exit 1
+    end
+
+    # Set output directory to current working directory
+    output_dir = File.join(Dir.pwd, "data", dataset)
+    ENV["NANOGPT_DATA_DIR"] = output_dir
+
+    puts "Preparing dataset: #{dataset}"
+    puts "Output directory: #{output_dir}"
+    load prepare_script
+  end
+
   def train
     config = NanoGPT::TrainConfig.load(@args)
 
@@ -318,6 +356,7 @@ class NanoGPTCLI
       Usage: nanogpt <command> [options]
 
       Commands:
+        prepare   Download and prepare a dataset
         train     Train a GPT model
         sample    Generate text from a trained model
         bench     Run performance benchmarks
@@ -325,9 +364,9 @@ class NanoGPTCLI
         help      Show this help message
 
       Examples:
-        nanogpt train --config=config/train_shakespeare_char.json
-        nanogpt train --dataset=shakespeare_char --max_iters=1000
-        nanogpt sample --out_dir=out-shakespeare-char --num_samples=3
+        nanogpt prepare shakespeare_char
+        nanogpt train --dataset=shakespeare_char --device=mps
+        nanogpt sample --dataset=shakespeare_char --num_samples=3
         nanogpt bench --batch_size=8 --block_size=512
 
       For more information, visit: https://github.com/khasinski/nanogpt-rb

data/lib/nano_gpt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module NanoGPT
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nanogpt
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Chris Hasiński