fine 0.1.0 → 0.2.0

data/lib/fine/validators.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+module Fine
+  # Data validation with helpful error messages
+  module Validators
+    class ValidationError < Error
+      attr_reader :line_number, :expected_format
+
+      def initialize(message, line_number: nil, expected_format: nil)
+        @line_number = line_number
+        @expected_format = expected_format
+        super(build_message(message))
+      end
+
+      private
+
+      def build_message(message)
+        parts = [message]
+        parts << "Line #{@line_number}" if @line_number
+        parts << "\nExpected format:\n#{@expected_format}" if @expected_format
+        parts.join(" ")
+      end
+    end
+
+    class << self
+      # Validate text classification data
+      #
+      # @param path [String] Path to JSONL file
+      # @raise [ValidationError] if validation fails
+      def validate_text_classification!(path)
+        validate_file_exists!(path)
+        validate_jsonl!(path) do |data, line_num|
+          unless data.key?("text") || data.key?(:text)
+            raise ValidationError.new(
+              "Missing 'text' field",
+              line_number: line_num,
+              expected_format: TEXT_CLASSIFICATION_FORMAT
+            )
+          end
+          unless data.key?("label") || data.key?(:label)
+            raise ValidationError.new(
+              "Missing 'label' field",
+              line_number: line_num,
+              expected_format: TEXT_CLASSIFICATION_FORMAT
+            )
+          end
+        end
+      end
+
+      # Validate text pairs for embedding training
+      #
+      # @param path [String] Path to JSONL file
+      # @raise [ValidationError] if validation fails
+      def validate_text_pairs!(path)
+        validate_file_exists!(path)
+        validate_jsonl!(path) do |data, line_num|
+          has_text_a = data.key?("text_a") || data.key?(:text_a) ||
+                       data.key?("anchor") || data.key?(:anchor) ||
+                       data.key?("sentence1") || data.key?(:sentence1) ||
+                       data.key?("query") || data.key?(:query)
+          has_text_b = data.key?("text_b") || data.key?(:text_b) ||
+                       data.key?("positive") || data.key?(:positive) ||
+                       data.key?("sentence2") || data.key?(:sentence2)
+
+          unless has_text_a && has_text_b
+            raise ValidationError.new(
+              "Missing text pair fields",
+              line_number: line_num,
+              expected_format: TEXT_PAIRS_FORMAT
+            )
+          end
+        end
+      end
+
+      # Validate instruction data for LLM fine-tuning
+      #
+      # @param path [String] Path to JSONL file
+      # @param format [Symbol] Expected format (:alpaca, :sharegpt, :simple, :auto)
+      # @raise [ValidationError] if validation fails
+      def validate_instructions!(path, format: :auto)
+        validate_file_exists!(path)
+
+        first_line = File.open(path, &:readline)
+        first_data = JSON.parse(first_line, symbolize_names: true)
+        detected_format = format == :auto ? detect_instruction_format(first_data) : format
+
+        validate_jsonl!(path) do |data, line_num|
+          case detected_format
+          when :alpaca
+            validate_alpaca_format!(data, line_num)
+          when :sharegpt
+            validate_sharegpt_format!(data, line_num)
+          when :simple
+            validate_simple_format!(data, line_num)
+          end
+        end
+
+        detected_format
+      end
+
+      # Validate image directory structure
+      #
+      # @param path [String] Path to directory
+      # @raise [ValidationError] if validation fails
+      def validate_image_directory!(path)
+        unless File.directory?(path)
+          raise ValidationError.new(
+            "Directory not found: #{path}",
+            expected_format: IMAGE_DIRECTORY_FORMAT
+          )
+        end
+
+        subdirs = Dir.entries(path).reject { |e| e.start_with?(".") }
+        subdirs = subdirs.select { |e| File.directory?(File.join(path, e)) }
+
+        if subdirs.empty?
+          raise ValidationError.new(
+            "No class subdirectories found in #{path}",
+            expected_format: IMAGE_DIRECTORY_FORMAT
+          )
+        end
+
+        # Check each subdirectory has images
+        subdirs.each do |subdir|
+          subdir_path = File.join(path, subdir)
+          images = Dir.glob(File.join(subdir_path, "*.{jpg,jpeg,png,gif,webp}"))
+          if images.empty?
+            raise ValidationError.new(
+              "No images found in class directory: #{subdir_path}",
+              expected_format: IMAGE_DIRECTORY_FORMAT
+            )
+          end
+        end
+
+        subdirs
+      end
+
+      # Quick check if file looks valid (non-blocking, for warnings)
+      #
+      # @param path [String] Path to file
+      # @param type [Symbol] Type of data (:text_classification, :text_pairs, :instructions)
+      # @return [Hash] { valid: true/false, warnings: [...], line_count: N }
+      def check(path, type:)
+        result = { valid: true, warnings: [], line_count: 0 }
+
+        begin
+          case type
+          when :text_classification
+            validate_text_classification!(path)
+          when :text_pairs
+            validate_text_pairs!(path)
+          when :instructions
+            validate_instructions!(path)
+          when :image_directory
+            validate_image_directory!(path)
+          end
+
+          result[:line_count] = File.readlines(path).count if File.file?(path)
+        rescue ValidationError => e
+          result[:valid] = false
+          result[:warnings] << e.message
+        rescue StandardError => e
+          result[:valid] = false
+          result[:warnings] << "Unexpected error: #{e.message}"
+        end
+
+        result
+      end
+
+      private
+
+      def validate_file_exists!(path)
+        unless File.exist?(path)
+          raise ValidationError.new("File not found: #{path}")
+        end
+
+        if File.empty?(path)
+          raise ValidationError.new("File is empty: #{path}")
+        end
+      end
+
+      def validate_jsonl!(path)
+        File.foreach(path).with_index(1) do |line, line_num|
+          next if line.strip.empty?
+
+          begin
+            data = JSON.parse(line, symbolize_names: true)
+          rescue JSON::ParserError => e
+            raise ValidationError.new(
+              "Invalid JSON: #{e.message}",
+              line_number: line_num
+            )
+          end
+
+          yield(data, line_num) if block_given?
+        end
+      end
+
+      def detect_instruction_format(data)
+        if data.key?(:instruction)
+          :alpaca
+        elsif data.key?(:conversations)
+          :sharegpt
+        elsif data.key?(:prompt) || data.key?(:text)
+          :simple
+        else
+          raise ValidationError.new(
+            "Cannot detect instruction format",
+            expected_format: INSTRUCTION_FORMATS
+          )
+        end
+      end
+
+      def validate_alpaca_format!(data, line_num)
+        unless data.key?(:instruction)
+          raise ValidationError.new(
+            "Missing 'instruction' field for Alpaca format",
+            line_number: line_num,
+            expected_format: ALPACA_FORMAT
+          )
+        end
+        unless data.key?(:output) || data.key?(:response)
+          raise ValidationError.new(
+            "Missing 'output' or 'response' field for Alpaca format",
+            line_number: line_num,
+            expected_format: ALPACA_FORMAT
+          )
+        end
+      end
+
+      def validate_sharegpt_format!(data, line_num)
+        unless data.key?(:conversations)
+          raise ValidationError.new(
+            "Missing 'conversations' field for ShareGPT format",
+            line_number: line_num,
+            expected_format: SHAREGPT_FORMAT
+          )
+        end
+        unless data[:conversations].is_a?(Array)
+          raise ValidationError.new(
+            "'conversations' must be an array",
+            line_number: line_num,
+            expected_format: SHAREGPT_FORMAT
+          )
+        end
+      end
+
+      def validate_simple_format!(data, line_num)
+        unless data.key?(:prompt) || data.key?(:text)
+          raise ValidationError.new(
+            "Missing 'prompt' or 'text' field for simple format",
+            line_number: line_num,
+            expected_format: SIMPLE_FORMAT
+          )
+        end
+      end
+
+      # Format examples for error messages
+      TEXT_CLASSIFICATION_FORMAT = <<~FORMAT
+        {"text": "This product is great!", "label": "positive"}
+        {"text": "Terrible experience", "label": "negative"}
+      FORMAT
+
+      TEXT_PAIRS_FORMAT = <<~FORMAT
+        {"text_a": "How do I reset my password?", "text_b": "Click forgot password on login page"}
+
+        Alternative field names: query/positive, anchor/positive, sentence1/sentence2
+      FORMAT
+
+      ALPACA_FORMAT = <<~FORMAT
+        {"instruction": "Summarize this text", "input": "Long text here...", "output": "Summary here"}
+        {"instruction": "Translate to French", "output": "Bonjour"}
+      FORMAT
+
+      SHAREGPT_FORMAT = <<~FORMAT
+        {"conversations": [
+          {"from": "human", "value": "Hello"},
+          {"from": "assistant", "value": "Hi there!"}
+        ]}
+      FORMAT
+
+      SIMPLE_FORMAT = <<~FORMAT
+        {"prompt": "Question here", "completion": "Answer here"}
+        {"text": "Full text for language modeling"}
+      FORMAT
+
+      INSTRUCTION_FORMATS = <<~FORMAT
+        Alpaca: {"instruction": "...", "output": "..."}
+        ShareGPT: {"conversations": [{"from": "human", "value": "..."}, ...]}
+        Simple: {"prompt": "...", "completion": "..."}
+      FORMAT
+
+      IMAGE_DIRECTORY_FORMAT = <<~FORMAT
+        data/
+          cats/
+            cat1.jpg
+            cat2.jpg
+          dogs/
+            dog1.jpg
+            dog2.jpg
+      FORMAT
+    end
+  end
+end

data/lib/fine/version.rb

@@ -1,3 +1,3 @@
 module Fine
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/fine.rb

@@ -12,6 +12,7 @@ require "fileutils"
 require_relative "fine/version"
 require_relative "fine/error"
 require_relative "fine/configuration"
+require_relative "fine/validators"
 
 # Hub
 require_relative "fine/hub/config_loader"
@@ -68,6 +69,9 @@ require_relative "fine/llm"
 # Export
 require_relative "fine/export"
 
+# LoRA
+require_relative "fine/lora"
+
 module Fine
   class << self
     attr_accessor :configuration

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fine
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Chris Hasinski
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-01-26 00:00:00.000000000 Z
+date: 2026-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: torch-rb
@@ -151,8 +151,12 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- docs/examples/image-classification-shapes.md
+- docs/examples/text-embeddings-faq.md
 - docs/installation.md
+- docs/quickstart.md
 - docs/tutorials/llm-fine-tuning.md
+- docs/tutorials/lora-tool-calling.md
 - docs/tutorials/model-export.md
 - docs/tutorials/siglip2-image-classification.md
 - docs/tutorials/siglip2-object-recognition.md
@@ -160,17 +164,56 @@ files:
 - docs/tutorials/text-classification.md
 - docs/tutorials/text-embeddings.md
 - examples/basic_classification.rb
+- examples/data/generate_tool_data.rb
+- examples/data/ollama_tool_calls.jsonl
+- examples/data/sentiment_reviews.jsonl
+- examples/data/shapes/circle/circle_1.jpg
+- examples/data/shapes/circle/circle_10.jpg
+- examples/data/shapes/circle/circle_2.jpg
+- examples/data/shapes/circle/circle_3.jpg
+- examples/data/shapes/circle/circle_4.jpg
+- examples/data/shapes/circle/circle_5.jpg
+- examples/data/shapes/circle/circle_6.jpg
+- examples/data/shapes/circle/circle_7.jpg
+- examples/data/shapes/circle/circle_8.jpg
+- examples/data/shapes/circle/circle_9.jpg
+- examples/data/shapes/square/square_1.jpg
+- examples/data/shapes/square/square_10.jpg
+- examples/data/shapes/square/square_2.jpg
+- examples/data/shapes/square/square_3.jpg
+- examples/data/shapes/square/square_4.jpg
+- examples/data/shapes/square/square_5.jpg
+- examples/data/shapes/square/square_6.jpg
+- examples/data/shapes/square/square_7.jpg
+- examples/data/shapes/square/square_8.jpg
+- examples/data/shapes/square/square_9.jpg
+- examples/data/shapes/triangle/triangle_1.jpg
+- examples/data/shapes/triangle/triangle_10.jpg
+- examples/data/shapes/triangle/triangle_2.jpg
+- examples/data/shapes/triangle/triangle_3.jpg
+- examples/data/shapes/triangle/triangle_4.jpg
+- examples/data/shapes/triangle/triangle_5.jpg
+- examples/data/shapes/triangle/triangle_6.jpg
+- examples/data/shapes/triangle/triangle_7.jpg
+- examples/data/shapes/triangle/triangle_8.jpg
+- examples/data/shapes/triangle/triangle_9.jpg
+- examples/data/support_faq_pairs.jsonl
 - examples/data/tool_calls.jsonl
 - examples/demo_training.rb
 - examples/finetune_gemma3_tools.rb
+- examples/generate_shape_images.rb
 - examples/real_llm_test.rb
 - examples/real_text_classification_test.rb
 - examples/real_text_embedder_test.rb
 - examples/real_training_test.rb
+- examples/sentiment_classification.rb
+- examples/shape_classification.rb
+- examples/support_faq_embeddings.rb
 - examples/test_export.rb
 - examples/test_image_classifier.rb
 - examples/test_llm.rb
 - examples/test_text_classifier.rb
+- examples/train_lora_tools.rb
 - lib/fine.rb
 - lib/fine/callbacks/base.rb
 - lib/fine/callbacks/progress_bar.rb
@@ -189,6 +232,7 @@ files:
 - lib/fine/hub/safetensors_loader.rb
 - lib/fine/image_classifier.rb
 - lib/fine/llm.rb
+- lib/fine/lora.rb
 - lib/fine/models/base.rb
 - lib/fine/models/bert_encoder.rb
 - lib/fine/models/bert_for_sequence_classification.rb
@@ -209,6 +253,7 @@ files:
 - lib/fine/transforms/normalize.rb
 - lib/fine/transforms/resize.rb
 - lib/fine/transforms/to_tensor.rb
+- lib/fine/validators.rb
 - lib/fine/version.rb
 - mise.toml
 homepage: https://github.com/khasinski/fine