nanogpt 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17c3921412aa2d70e303ba68f979f5ccf9bb3e9b815ddfe3fbfbfcc62b7a3be6
-  data.tar.gz: 1b0b97a5513a341ae93bd046a86df8237f3941d3cbe11dec480a6cb59c8ea597
+  metadata.gz: cf308fcec8ccec074200361b2327a2381a5412b92497391bb71ec5f154cd1283
+  data.tar.gz: 87dcc389df03af0ac59fc0e75bbef7be7071f00613a2662cecf365ba38bc853e
 SHA512:
-  metadata.gz: b3b7d07a8016bb7817f20bb22476d1d85bdaad42600234f8cf6a763316b6b128e473bccf17eef5135486c5b67051c3d1f10278214a28ec72cd26da18a4f68c8c
-  data.tar.gz: 90fe296ae42f058d9a918ba630781edca983ef22314797ef44a82e15c257bf06903c10b0442611e78a9ff654c08978720e1ece4bb3bb13b52f5d3cffe2331122
+  metadata.gz: 2b6ceeb10236b639c82398c94d3c1a876eff549a17289ff45abae489e480a3d6a1db45f95a4b2e56f1d1df6afde28159965b188342d1e38c2482359dfb11e061
+  data.tar.gz: 0fd4653c2719d1d3c339a904437fd0657f17e49fa0a9d4361a3a259806fab8dc798ed7f179722b41913d3471e8799abf78823f207988987d86cba680d7f90f03

data/data/openwebtext/prepare.rb

@@ -33,7 +33,10 @@ require "rubygems/package"
 require "zlib"
 
 SCRIPT_DIR = File.dirname(__FILE__)
-RAW_DIR = File.join(SCRIPT_DIR, "raw")
+OUTPUT_DIR = ENV["NANOGPT_DATA_DIR"] || SCRIPT_DIR
+RAW_DIR = File.join(OUTPUT_DIR, "raw")
+
+FileUtils.mkdir_p(OUTPUT_DIR) if ENV["NANOGPT_DATA_DIR"]
 DEFAULT_VAL_RATIO = 0.0005  # ~0.5% for validation
 
 def parse_args
@@ -60,11 +63,17 @@ def parse_args
 end
 
 def find_data_files
-  # Look for various supported formats
+  # Look for various supported formats in both SCRIPT_DIR (gem) and OUTPUT_DIR (local)
   patterns = [
     File.join(RAW_DIR, "**", "*.parquet"),  # Parquet (from Python export)
     File.join(RAW_DIR, "**", "*.tar"),       # Original tar files
     File.join(RAW_DIR, "**", "*.txt"),       # Plain text files
+    File.join(OUTPUT_DIR, "*.parquet"),
+    File.join(OUTPUT_DIR, "*.tar"),
+    File.join(OUTPUT_DIR, "*.txt"),
+    File.join(SCRIPT_DIR, "raw", "**", "*.parquet"),
+    File.join(SCRIPT_DIR, "raw", "**", "*.tar"),
+    File.join(SCRIPT_DIR, "raw", "**", "*.txt"),
     File.join(SCRIPT_DIR, "*.parquet"),
     File.join(SCRIPT_DIR, "*.tar"),
     File.join(SCRIPT_DIR, "*.txt")
@@ -273,8 +282,8 @@ def main
 
   # Write binary files
   puts "Writing binary files..."
-  write_binary(train_tokens, File.join(SCRIPT_DIR, "train.bin"))
-  write_binary(val_tokens, File.join(SCRIPT_DIR, "val.bin"))
+  write_binary(train_tokens, File.join(OUTPUT_DIR, "train.bin"))
+  write_binary(val_tokens, File.join(OUTPUT_DIR, "val.bin"))
 
   puts ""
   puts "Done! OpenWebText dataset prepared."

data/data/shakespeare/prepare.rb

@@ -11,9 +11,13 @@ require "net/http"
 require "openssl"
 require "numo/narray"
 require "tiktoken_ruby"
+require "fileutils"
 
 DATA_URL = "https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt"
 SCRIPT_DIR = File.dirname(__FILE__)
+OUTPUT_DIR = ENV["NANOGPT_DATA_DIR"] || SCRIPT_DIR
+
+FileUtils.mkdir_p(OUTPUT_DIR) if ENV["NANOGPT_DATA_DIR"]
 
 def download_file(url)
   uri = URI(url)
@@ -53,8 +57,8 @@ puts "Val has #{val_ids.length} tokens"
 # Export to binary files (uint16)
 train_arr = Numo::UInt16.cast(train_ids)
 val_arr = Numo::UInt16.cast(val_ids)
-File.binwrite(File.join(SCRIPT_DIR, "train.bin"), train_arr.to_binary)
-File.binwrite(File.join(SCRIPT_DIR, "val.bin"), val_arr.to_binary)
+File.binwrite(File.join(OUTPUT_DIR, "train.bin"), train_arr.to_binary)
+File.binwrite(File.join(OUTPUT_DIR, "val.bin"), val_arr.to_binary)
 
 # No meta.json - indicates GPT-2 BPE tokenizer should be used
 puts "Done! Created train.bin and val.bin"

data/data/shakespeare_char/prepare.rb

@@ -8,9 +8,13 @@ require "net/http"
 require "openssl"
 require "numo/narray"
 require "json"
+require "fileutils"
 
 DATA_URL = "https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt"
 SCRIPT_DIR = File.dirname(__FILE__)
+OUTPUT_DIR = ENV["NANOGPT_DATA_DIR"] || SCRIPT_DIR
+
+FileUtils.mkdir_p(OUTPUT_DIR) if ENV["NANOGPT_DATA_DIR"]
 
 def download_file(url)
   uri = URI(url)
@@ -59,8 +63,8 @@ puts "Val has #{val_ids.length} tokens"
 # Export to binary files (uint16)
 train_arr = Numo::UInt16.cast(train_ids)
 val_arr = Numo::UInt16.cast(val_ids)
-File.binwrite(File.join(SCRIPT_DIR, "train.bin"), train_arr.to_binary)
-File.binwrite(File.join(SCRIPT_DIR, "val.bin"), val_arr.to_binary)
+File.binwrite(File.join(OUTPUT_DIR, "train.bin"), train_arr.to_binary)
+File.binwrite(File.join(OUTPUT_DIR, "val.bin"), val_arr.to_binary)
 
 # Save meta information as JSON
 meta = {
@@ -68,6 +72,6 @@ meta = {
   "itos" => itos.transform_keys(&:to_s),
   "stoi" => stoi
 }
-File.write(File.join(SCRIPT_DIR, "meta.json"), JSON.pretty_generate(meta))
+File.write(File.join(OUTPUT_DIR, "meta.json"), JSON.pretty_generate(meta))
 
 puts "Done! Created train.bin, val.bin, and meta.json"

data/exe/nanogpt

@@ -64,7 +64,12 @@ class NanoGPTCLI
       exit 1
     end
 
+    # Set output directory to current working directory
+    output_dir = File.join(Dir.pwd, "data", dataset)
+    ENV["NANOGPT_DATA_DIR"] = output_dir
+
     puts "Preparing dataset: #{dataset}"
+    puts "Output directory: #{output_dir}"
     load prepare_script
   end
 

data/lib/nano_gpt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module NanoGPT
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nanogpt
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Chris Hasiński