fine 0.1.0

data/lib/fine/datasets/instruction_dataset.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+module Fine
+  module Datasets
+    # Dataset for instruction/chat fine-tuning
+    #
+    # Supports common formats:
+    # - Alpaca: {"instruction": "...", "input": "...", "output": "..."}
+    # - ShareGPT: {"conversations": [{"from": "human", "value": "..."}, {"from": "gpt", "value": "..."}]}
+    # - Simple: {"prompt": "...", "completion": "..."}
+    class InstructionDataset
+      attr_reader :examples, :tokenizer, :max_length
+
+      # Load from JSONL file
+      #
+      # @param path [String] Path to JSONL file
+      # @param tokenizer [Tokenizers::AutoTokenizer] Tokenizer
+      # @param format [Symbol] Data format (:alpaca, :sharegpt, :simple, :auto)
+      # @param max_length [Integer] Maximum sequence length
+      # @return [InstructionDataset]
+      def self.from_jsonl(path, tokenizer:, format: :auto, max_length: 2048)
+        examples = File.readlines(path).map { |line| JSON.parse(line, symbolize_names: true) }
+        new(examples, tokenizer: tokenizer, format: format, max_length: max_length)
+      end
+
+      def initialize(examples, tokenizer:, format: :auto, max_length: 2048)
+        @tokenizer = tokenizer
+        @max_length = max_length
+        @format = format == :auto ? detect_format(examples.first) : format
+
+        @examples = examples.map { |ex| normalize_example(ex) }
+      end
+
+      def size
+        @examples.size
+      end
+
+      def [](idx)
+        example = @examples[idx]
+
+        # Format as prompt + completion
+        text = format_example(example)
+
+        # Tokenize (without tensors for easier manipulation)
+        encoding = @tokenizer.encode(text, return_tensors: false)
+        input_ids = encoding[:input_ids].first
+
+        # Get prompt length before truncation
+        prompt_text = format_prompt_only(example)
+        prompt_encoding = @tokenizer.encode(prompt_text, return_tensors: false)
+        prompt_length = prompt_encoding[:input_ids].first.size
+
+        # Truncate if needed, but ensure at least some completion tokens remain
+        if input_ids.size > @max_length
+          # If prompt alone is too long, truncate from the left (keep completion)
+          if prompt_length >= @max_length - 10
+            # Keep last max_length tokens (includes completion)
+            input_ids = input_ids.last(@max_length)
+            # No masking since we dropped the prompt prefix
+            prompt_length = 0
+          else
+            input_ids = input_ids.first(@max_length)
+          end
+        end
+
+        # Labels are same as input_ids for causal LM (predict next token)
+        labels = input_ids.dup
+
+        # Mask prompt tokens with -100 (ignored in loss), but only if there's room for completion
+        if prompt_length > 0 && prompt_length < input_ids.size
+          labels[0...prompt_length] = [-100] * prompt_length
+        end
+        # If prompt_length >= input_ids.size, don't mask (train on full sequence)
+
+        {
+          input_ids: Torch.tensor([input_ids]),
+          labels: Torch.tensor([labels]),
+          attention_mask: Torch.ones(1, input_ids.size)
+        }
+      end
+
+      # Split dataset
+      def split(test_size: 0.1, seed: 42)
+        rng = Random.new(seed)
+        indices = (0...size).to_a.shuffle(random: rng)
+
+        split_idx = (size * (1 - test_size)).to_i
+        train_indices = indices[0...split_idx]
+        test_indices = indices[split_idx..]
+
+        train_examples = train_indices.map { |i| @examples[i] }
+        test_examples = test_indices.map { |i| @examples[i] }
+
+        [
+          self.class.new(train_examples, tokenizer: @tokenizer, format: @format, max_length: @max_length),
+          self.class.new(test_examples, tokenizer: @tokenizer, format: @format, max_length: @max_length)
+        ]
+      end
+
+      private
+
+      def detect_format(example)
+        if example.key?(:instruction)
+          :alpaca
+        elsif example.key?(:conversations)
+          :sharegpt
+        elsif example.key?(:prompt) || example.key?(:text)
+          :simple
+        else
+          raise DatasetError, "Cannot detect format. Keys: #{example.keys}"
+        end
+      end
+
+      def normalize_example(example)
+        case @format
+        when :alpaca
+          {
+            prompt: build_alpaca_prompt(example[:instruction], example[:input]),
+            completion: example[:output] || example[:response]
+          }
+        when :sharegpt
+          conversations = example[:conversations]
+          # Take first human/assistant pair
+          human = conversations.find { |c| c[:from] == "human" }
+          assistant = conversations.find { |c| c[:from] == "gpt" || c[:from] == "assistant" }
+          {
+            prompt: human[:value],
+            completion: assistant[:value]
+          }
+        when :simple
+          {
+            prompt: example[:prompt] || example[:text],
+            completion: example[:completion] || example[:response] || ""
+          }
+        else
+          raise DatasetError, "Unknown format: #{@format}"
+        end
+      end
+
+      def build_alpaca_prompt(instruction, input = nil)
+        if input && !input.empty?
+          "### Instruction:\n#{instruction}\n\n### Input:\n#{input}\n\n### Response:\n"
+        else
+          "### Instruction:\n#{instruction}\n\n### Response:\n"
+        end
+      end
+
+      def format_example(example)
+        "#{example[:prompt]}#{example[:completion]}"
+      end
+
+      def format_prompt_only(example)
+        example[:prompt]
+      end
+    end
+
+    # Data loader for instruction dataset with dynamic padding
+    class InstructionDataLoader
+      include Enumerable
+
+      def initialize(dataset, batch_size:, shuffle: true, pad_token_id: 0)
+        @dataset = dataset
+        @batch_size = batch_size
+        @shuffle = shuffle
+        @pad_token_id = pad_token_id
+      end
+
+      def each
+        indices = (0...@dataset.size).to_a
+        indices.shuffle! if @shuffle
+
+        indices.each_slice(@batch_size) do |batch_indices|
+          batch = batch_indices.map { |i| @dataset[i] }
+          yield collate_batch(batch)
+        end
+      end
+
+      def size
+        (@dataset.size.to_f / @batch_size).ceil
+      end
+
+      private
+
+      def collate_batch(batch)
+        max_len = batch.map { |b| b[:input_ids].size(-1) }.max
+
+        input_ids = []
+        labels = []
+        attention_masks = []
+
+        batch.each do |item|
+          seq_len = item[:input_ids].size(-1)
+          pad_len = max_len - seq_len
+
+          if pad_len > 0
+            # Pad on the right
+            input_ids << Torch.cat([
+              item[:input_ids],
+              Torch.full([1, pad_len], @pad_token_id)
+            ], dim: 1)
+
+            labels << Torch.cat([
+              item[:labels],
+              Torch.full([1, pad_len], -100)  # Ignore padding in loss
+            ], dim: 1)
+
+            attention_masks << Torch.cat([
+              item[:attention_mask],
+              Torch.zeros(1, pad_len)
+            ], dim: 1)
+          else
+            input_ids << item[:input_ids]
+            labels << item[:labels]
+            attention_masks << item[:attention_mask]
+          end
+        end
+
+        {
+          input_ids: Torch.cat(input_ids, dim: 0),
+          labels: Torch.cat(labels, dim: 0),
+          attention_mask: Torch.cat(attention_masks, dim: 0)
+        }
+      end
+    end
+  end
+end

data/lib/fine/datasets/text_data_loader.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Fine
+  module Datasets
+    # DataLoader for text datasets with dynamic padding
+    class TextDataLoader
+      include Enumerable
+
+      attr_reader :dataset, :batch_size, :shuffle, :drop_last
+
+      def initialize(dataset, batch_size:, shuffle: false, drop_last: false)
+        @dataset = dataset
+        @batch_size = batch_size
+        @shuffle = shuffle
+        @drop_last = drop_last
+      end
+
+      # Iterate over batches
+      def each_batch
+        return enum_for(:each_batch) unless block_given?
+
+        indices = (0...@dataset.size).to_a
+        indices.shuffle! if @shuffle
+
+        indices.each_slice(@batch_size) do |batch_indices|
+          next if @drop_last && batch_indices.size < @batch_size
+
+          yield collate(batch_indices)
+        end
+      end
+
+      alias each each_batch
+
+      # Number of batches
+      def size
+        n = @dataset.size / @batch_size
+        n += 1 unless @drop_last || (@dataset.size % @batch_size).zero?
+        n
+      end
+
+      alias num_batches size
+
+      private
+
+      def collate(indices)
+        samples = indices.map { |i| @dataset[i] }
+
+        # Find max length in this batch for dynamic padding
+        max_len = samples.map { |s| s[:input_ids].length }.max
+
+        # Pad and stack
+        input_ids = []
+        attention_mask = []
+        token_type_ids = []
+        labels = []
+
+        samples.each do |sample|
+          ids = sample[:input_ids]
+          mask = sample[:attention_mask]
+          type_ids = sample[:token_type_ids]
+
+          # Pad to max_len
+          pad_len = max_len - ids.length
+          if pad_len > 0
+            ids = ids + Array.new(pad_len, 0)
+            mask = mask + Array.new(pad_len, 0)
+            type_ids = type_ids + Array.new(pad_len, 0) if type_ids
+          end
+
+          input_ids << ids
+          attention_mask << mask
+          token_type_ids << type_ids if type_ids
+          labels << sample[:label]
+        end
+
+        result = {
+          input_ids: Torch.tensor(input_ids, dtype: :long),
+          attention_mask: Torch.tensor(attention_mask, dtype: :long),
+          labels: Torch.tensor(labels, dtype: :long)
+        }
+
+        result[:token_type_ids] = Torch.tensor(token_type_ids, dtype: :long) if token_type_ids.first
+
+        result
+      end
+    end
+  end
+end

data/lib/fine/datasets/text_dataset.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module Fine
+  module Datasets
+    # Dataset for text classification tasks
+    #
+    # Supports JSONL and CSV formats:
+    #   JSONL: {"text": "...", "label": "positive"}
+    #   CSV: text,label (with header)
+    #
+    class TextDataset
+      include Enumerable
+
+      attr_reader :texts, :labels, :label_map, :inverse_label_map, :tokenizer
+
+      # Load dataset from a JSONL file
+      #
+      # @param path [String] Path to JSONL file
+      # @param tokenizer [AutoTokenizer] Tokenizer to use
+      # @param text_column [String] Name of text field
+      # @param label_column [String] Name of label field
+      # @return [TextDataset]
+      def self.from_jsonl(path, tokenizer:, text_column: "text", label_column: "label")
+        raise DatasetError, "File not found: #{path}" unless File.exist?(path)
+
+        texts = []
+        labels = []
+
+        File.foreach(path) do |line|
+          next if line.strip.empty?
+
+          data = JSON.parse(line)
+          texts << data[text_column]
+          labels << data[label_column]
+        end
+
+        raise DatasetError, "No data found in #{path}" if texts.empty?
+
+        new(texts: texts, labels: labels, tokenizer: tokenizer)
+      end
+
+      # Load dataset from a CSV file
+      #
+      # @param path [String] Path to CSV file
+      # @param tokenizer [AutoTokenizer] Tokenizer to use
+      # @param text_column [String] Name of text column
+      # @param label_column [String] Name of label column
+      # @return [TextDataset]
+      def self.from_csv(path, tokenizer:, text_column: "text", label_column: "label")
+        require "csv"
+        raise DatasetError, "File not found: #{path}" unless File.exist?(path)
+
+        texts = []
+        labels = []
+
+        CSV.foreach(path, headers: true) do |row|
+          texts << row[text_column]
+          labels << row[label_column]
+        end
+
+        raise DatasetError, "No data found in #{path}" if texts.empty?
+
+        new(texts: texts, labels: labels, tokenizer: tokenizer)
+      end
+
+      # Load from file (auto-detect format)
+      #
+      # @param path [String] Path to data file
+      # @param tokenizer [AutoTokenizer] Tokenizer to use
+      # @return [TextDataset]
+      def self.from_file(path, tokenizer:, **kwargs)
+        case File.extname(path).downcase
+        when ".jsonl", ".json"
+          from_jsonl(path, tokenizer: tokenizer, **kwargs)
+        when ".csv"
+          from_csv(path, tokenizer: tokenizer, **kwargs)
+        else
+          # Try JSONL first, then CSV
+          begin
+            from_jsonl(path, tokenizer: tokenizer, **kwargs)
+          rescue JSON::ParserError
+            from_csv(path, tokenizer: tokenizer, **kwargs)
+          end
+        end
+      end
+
+      def initialize(texts:, labels:, tokenizer:, label_map: nil)
+        raise ArgumentError, "texts and labels must have same length" if texts.size != labels.size
+
+        @texts = texts
+        @tokenizer = tokenizer
+
+        # Build label map if not provided
+        if label_map
+          @label_map = label_map
+        else
+          unique_labels = labels.uniq.sort
+          @label_map = unique_labels.each_with_index.to_h
+        end
+
+        # Convert string labels to integers
+        @labels = labels.map do |label|
+          label.is_a?(Integer) ? label : @label_map[label]
+        end
+
+        # Build inverse mapping
+        @inverse_label_map = @label_map.invert
+      end
+
+      # Get a single item from the dataset
+      #
+      # @param index [Integer] Index of the item
+      # @return [Hash] Hash with tokenized inputs and label
+      def [](index)
+        text = @texts[index]
+        encoding = @tokenizer.encode(text, return_tensors: false)
+
+        {
+          input_ids: encoding[:input_ids].first,
+          attention_mask: encoding[:attention_mask].first,
+          token_type_ids: encoding[:token_type_ids]&.first,
+          label: @labels[index]
+        }.compact
+      end
+
+      # Number of items in the dataset
+      def size
+        @texts.size
+      end
+      alias length size
+
+      # Iterate over all items
+      def each
+        return enum_for(:each) unless block_given?
+
+        size.times { |i| yield self[i] }
+      end
+
+      # Number of classes
+      def num_classes
+        @label_map.size
+      end
+
+      # Get class names in order
+      def class_names
+        @inverse_label_map.sort.map(&:last)
+      end
+
+      # Split dataset into train and validation sets
+      #
+      # @param test_size [Float] Fraction of data for validation (0.0-1.0)
+      # @param shuffle [Boolean] Whether to shuffle before splitting
+      # @param stratify [Boolean] Whether to maintain class distribution
+      # @param seed [Integer, nil] Random seed
+      # @return [Array<TextDataset, TextDataset>] Train and validation datasets
+      def split(test_size: 0.2, shuffle: true, stratify: true, seed: nil)
+        rng = seed ? Random.new(seed) : Random.new
+
+        indices = (0...size).to_a
+        indices = indices.shuffle(random: rng) if shuffle && !stratify
+
+        if stratify
+          train_indices, val_indices = stratified_split(indices, test_size, rng)
+        else
+          split_idx = (size * (1 - test_size)).round
+          train_indices = indices[0...split_idx]
+          val_indices = indices[split_idx..]
+        end
+
+        train_set = subset(train_indices)
+        val_set = subset(val_indices)
+
+        [train_set, val_set]
+      end
+
+      private
+
+      def subset(indices)
+        TextDataset.new(
+          texts: indices.map { |i| @texts[i] },
+          labels: indices.map { |i| @labels[i] },
+          tokenizer: @tokenizer,
+          label_map: @label_map
+        )
+      end
+
+      def stratified_split(indices, test_size, rng)
+        train_indices = []
+        val_indices = []
+
+        # Group indices by label
+        by_label = indices.group_by { |i| @labels[i] }
+
+        by_label.each_value do |label_indices|
+          shuffled = label_indices.shuffle(random: rng)
+          split_idx = (shuffled.size * (1 - test_size)).round
+
+          train_indices.concat(shuffled[0...split_idx])
+          val_indices.concat(shuffled[split_idx..])
+        end
+
+        [train_indices.shuffle(random: rng), val_indices.shuffle(random: rng)]
+      end
+    end
+
+    # Dataset for text pair tasks (similarity, NLI, etc.)
+    class TextPairDataset
+      include Enumerable
+
+      attr_reader :texts_a, :texts_b, :labels, :tokenizer
+
+      # Load from JSONL with query/positive pairs
+      #
+      # @param path [String] Path to JSONL file
+      # @param tokenizer [AutoTokenizer] Tokenizer to use
+      # @return [TextPairDataset]
+      def self.from_jsonl(path, tokenizer:, text_a_column: "query", text_b_column: "positive")
+        raise DatasetError, "File not found: #{path}" unless File.exist?(path)
+
+        texts_a = []
+        texts_b = []
+        labels = []
+
+        File.foreach(path) do |line|
+          next if line.strip.empty?
+
+          data = JSON.parse(line)
+          texts_a << data[text_a_column]
+          texts_b << data[text_b_column]
+          labels << (data["label"] || 1.0)  # Default to positive pair
+        end
+
+        new(texts_a: texts_a, texts_b: texts_b, labels: labels, tokenizer: tokenizer)
+      end
+
+      def initialize(texts_a:, texts_b:, labels:, tokenizer:)
+        @texts_a = texts_a
+        @texts_b = texts_b
+        @labels = labels
+        @tokenizer = tokenizer
+      end
+
+      def [](index)
+        encoding = @tokenizer.encode_pair(@texts_a[index], @texts_b[index], return_tensors: false)
+
+        {
+          input_ids: encoding[:input_ids].first,
+          attention_mask: encoding[:attention_mask].first,
+          token_type_ids: encoding[:token_type_ids]&.first,
+          label: @labels[index]
+        }.compact
+      end
+
+      def size
+        @texts_a.size
+      end
+      alias length size
+
+      def each
+        return enum_for(:each) unless block_given?
+
+        size.times { |i| yield self[i] }
+      end
+    end
+  end
+end

data/lib/fine/error.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Fine
+  # Base error class for all Fine errors
+  class Error < StandardError; end
+
+  # Raised when a model cannot be found on Hugging Face Hub
+  class ModelNotFoundError < Error
+    attr_reader :model_id
+
+    def initialize(model_id, message = nil)
+      @model_id = model_id
+      super(message || "Model not found: #{model_id}")
+    end
+  end
+
+  # Raised when configuration is invalid
+  class ConfigurationError < Error; end
+
+  # Raised when there's an issue with dataset loading or processing
+  class DatasetError < Error; end
+
+  # Raised when training fails
+  class TrainingError < Error; end
+
+  # Raised when model weights cannot be loaded
+  class WeightLoadingError < Error
+    attr_reader :missing_keys, :unexpected_keys
+
+    def initialize(message, missing_keys: [], unexpected_keys: [])
+      @missing_keys = missing_keys
+      @unexpected_keys = unexpected_keys
+      super(message)
+    end
+  end
+
+  # Raised when image processing fails
+  class ImageProcessingError < Error
+    attr_reader :path
+
+    def initialize(path, message = nil)
+      @path = path
+      super(message || "Failed to process image: #{path}")
+    end
+  end
+
+  # Raised when model export fails
+  class ExportError < Error; end
+end