classifier 2.0.0 → 2.2.0

data/lib/classifier/storage/base.rb

@@ -0,0 +1,50 @@
+# rbs_inline: enabled
+
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+module Classifier
+  module Storage
+    # Abstract base class for storage backends.
+    # Implement this protocol to create custom storage (Redis, PostgreSQL, etc.)
+    #
+    # Example:
+    #   class RedisStorage < Classifier::Storage::Base
+    #     def initialize(redis:, key:)
+    #       @redis, @key = redis, key
+    #     end
+    #
+    #     def write(data) = @redis.set(@key, data)
+    #     def read = @redis.get(@key)
+    #     def delete = @redis.del(@key)
+    #     def exists? = @redis.exists?(@key)
+    #   end
+    #
+    class Base
+      # Save classifier data
+      # @rbs (String) -> void
+      def write(data)
+        raise NotImplementedError, "#{self.class}#write must be implemented"
+      end
+
+      # Load classifier data
+      # @rbs () -> String?
+      def read
+        raise NotImplementedError, "#{self.class}#read must be implemented"
+      end
+
+      # Delete classifier data
+      # @rbs () -> void
+      def delete
+        raise NotImplementedError, "#{self.class}#delete must be implemented"
+      end
+
+      # Check if data exists
+      # @rbs () -> bool
+      def exists?
+        raise NotImplementedError, "#{self.class}#exists? must be implemented"
+      end
+    end
+  end
+end

data/lib/classifier/storage/file.rb

@@ -0,0 +1,51 @@
+# rbs_inline: enabled
+
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+require_relative 'base'
+
+module Classifier
+  module Storage
+    # File-based storage backend.
+    #
+    # Example:
+    #   bayes = Classifier::Bayes.new('Spam', 'Ham')
+    #   bayes.storage = Classifier::Storage::File.new(path: "/var/models/spam.json")
+    #   bayes.train_spam("Buy now!")
+    #   bayes.save
+    #
+    class File < Base
+      # @rbs @path: String
+
+      attr_reader :path
+
+      # @rbs (path: String) -> void
+      def initialize(path:)
+        super()
+        @path = path
+      end
+
+      # @rbs (String) -> Integer
+      def write(data)
+        ::File.write(@path, data)
+      end
+
+      # @rbs () -> String?
+      def read
+        exists? ? ::File.read(@path) : nil
+      end
+
+      # @rbs () -> void
+      def delete
+        ::File.delete(@path) if exists?
+      end
+
+      # @rbs () -> bool
+      def exists?
+        ::File.exist?(@path)
+      end
+    end
+  end
+end

data/lib/classifier/storage/memory.rb

@@ -0,0 +1,49 @@
+# rbs_inline: enabled
+
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+require_relative 'base'
+
+module Classifier
+  module Storage
+    # In-memory storage for testing and ephemeral use.
+    #
+    # Example:
+    #   bayes = Classifier::Bayes.new('Spam', 'Ham')
+    #   bayes.storage = Classifier::Storage::Memory.new
+    #   bayes.train_spam("Buy now!")
+    #   bayes.save
+    #
+    class Memory < Base
+      # @rbs @data: String?
+
+      # @rbs () -> void
+      def initialize
+        super
+        @data = nil
+      end
+
+      # @rbs (String) -> String
+      def write(data)
+        @data = data
+      end
+
+      # @rbs () -> String?
+      def read
+        @data
+      end
+
+      # @rbs () -> void
+      def delete
+        @data = nil
+      end
+
+      # @rbs () -> bool
+      def exists?
+        !@data.nil?
+      end
+    end
+  end
+end

data/lib/classifier/storage.rb

@@ -0,0 +1,9 @@
+# rbs_inline: enabled
+
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+require_relative 'storage/base'
+require_relative 'storage/memory'
+require_relative 'storage/file'

data/lib/classifier/streaming/line_reader.rb

@@ -0,0 +1,99 @@
+# rbs_inline: enabled
+
+module Classifier
+  module Streaming
+    # Memory-efficient line reader for large files and IO streams.
+    # Reads lines one at a time and can yield in configurable batches.
+    #
+    # @example Reading line by line
+    #   reader = LineReader.new(File.open('large_corpus.txt'))
+    #   reader.each { |line| process(line) }
+    #
+    # @example Reading in batches
+    #   reader = LineReader.new(io, batch_size: 100)
+    #   reader.each_batch { |batch| process_batch(batch) }
+    class LineReader
+      include Enumerable #[String]
+
+      # @rbs @io: IO
+      # @rbs @batch_size: Integer
+
+      attr_reader :batch_size
+
+      # Creates a new LineReader.
+      #
+      # @rbs (IO, ?batch_size: Integer) -> void
+      def initialize(io, batch_size: 100)
+        @io = io
+        @batch_size = batch_size
+      end
+
+      # Iterates over each line in the IO stream.
+      # Lines are chomped (trailing newlines removed).
+      #
+      # @rbs () { (String) -> void } -> void
+      # @rbs () -> Enumerator[String, void]
+      def each
+        return enum_for(:each) unless block_given?
+
+        @io.each_line do |line|
+          yield line.chomp
+        end
+      end
+
+      # Iterates over batches of lines.
+      # Each batch is an array of chomped lines.
+      #
+      # @rbs () { (Array[String]) -> void } -> void
+      # @rbs () -> Enumerator[Array[String], void]
+      def each_batch
+        return enum_for(:each_batch) unless block_given?
+
+        batch = [] #: Array[String]
+        each do |line|
+          batch << line
+          if batch.size >= @batch_size
+            yield batch
+            batch = []
+          end
+        end
+        yield batch unless batch.empty?
+      end
+
+      # Estimates the total number of lines in the IO stream.
+      # This is a rough estimate based on file size and average line length.
+      # Returns nil for non-seekable streams.
+      #
+      # @rbs (?sample_size: Integer) -> Integer?
+      def estimate_line_count(sample_size: 100)
+        return nil unless @io.respond_to?(:size) && @io.respond_to?(:rewind)
+
+        begin
+          original_pos = @io.pos
+          @io.rewind
+
+          sample_bytes = 0
+          sample_lines = 0
+
+          sample_size.times do
+            line = @io.gets
+            break unless line
+
+            sample_bytes += line.bytesize
+            sample_lines += 1
+          end
+
+          @io.seek(original_pos)
+
+          return nil if sample_lines.zero?
+
+          avg_line_size = sample_bytes.to_f / sample_lines
+          io_size = @io.__send__(:size) #: Integer
+          (io_size / avg_line_size).round
+        rescue IOError, Errno::ESPIPE
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/classifier/streaming/progress.rb

@@ -0,0 +1,96 @@
+# rbs_inline: enabled
+
+module Classifier
+  module Streaming
+    # Progress tracking object yielded to blocks during batch/stream operations.
+    # Provides information about training progress including completion percentage,
+    # elapsed time, processing rate, and estimated time remaining.
+    #
+    # @example Basic usage with train_batch
+    #   classifier.train_batch(:spam, documents, batch_size: 100) do |progress|
+    #     puts "#{progress.completed}/#{progress.total} (#{progress.percent}%)"
+    #     puts "Rate: #{progress.rate.round(1)} docs/sec"
+    #     puts "ETA: #{progress.eta&.round}s" if progress.eta
+    #   end
+    class Progress
+      # @rbs @completed: Integer
+      # @rbs @total: Integer?
+      # @rbs @start_time: Time
+      # @rbs @current_batch: Integer
+
+      attr_reader :start_time, :total
+      attr_accessor :completed, :current_batch
+
+      # @rbs (?total: Integer?, ?completed: Integer) -> void
+      def initialize(total: nil, completed: 0)
+        @completed = completed
+        @total = total
+        @start_time = Time.now
+        @current_batch = 0
+      end
+
+      # Returns the completion percentage (0-100).
+      # Returns nil if total is unknown.
+      #
+      # @rbs () -> Float?
+      def percent
+        return nil unless @total&.positive?
+
+        (@completed.to_f / @total * 100).round(2)
+      end
+
+      # Returns the elapsed time in seconds since the operation started.
+      #
+      # @rbs () -> Float
+      def elapsed
+        Time.now - @start_time
+      end
+
+      # Returns the processing rate in items per second.
+      # Returns 0 if no time has elapsed.
+      #
+      # @rbs () -> Float
+      def rate
+        e = elapsed
+        return 0.0 if e.zero?
+
+        @completed / e
+      end
+
+      # Returns the estimated time remaining in seconds.
+      # Returns nil if total is unknown or rate is zero.
+      #
+      # @rbs () -> Float?
+      def eta
+        return nil unless @total
+        return nil if rate.zero?
+        return 0.0 if @completed >= @total
+
+        (@total - @completed) / rate
+      end
+
+      # Returns true if the operation is complete.
+      #
+      # @rbs () -> bool
+      def complete?
+        return false unless @total
+
+        @completed >= @total
+      end
+
+      # Returns a hash representation of the progress state.
+      #
+      # @rbs () -> Hash[Symbol, untyped]
+      def to_h
+        {
+          completed: @completed,
+          total: @total,
+          percent: percent,
+          elapsed: elapsed.round(2),
+          rate: rate.round(2),
+          eta: eta&.round(2)
+        }
+      end
+    end
+  end
+end

data/lib/classifier/streaming.rb

@@ -0,0 +1,122 @@
+# rbs_inline: enabled
+
+require_relative 'streaming/progress'
+require_relative 'streaming/line_reader'
+
+module Classifier
+  # Streaming module provides memory-efficient training capabilities for classifiers.
+  # Include this module in a classifier to add streaming and batch training methods.
+  #
+  # @example Including in a classifier
+  #   class MyClassifier
+  #     include Classifier::Streaming
+  #   end
+  #
+  # @example Streaming training
+  #   classifier.train_from_stream(:category, File.open('corpus.txt'))
+  #
+  # @example Batch training with progress
+  #   classifier.train_batch(:category, documents, batch_size: 100) do |progress|
+  #     puts "#{progress.percent}% complete"
+  #   end
+  module Streaming
+    # Default batch size for streaming operations
+    DEFAULT_BATCH_SIZE = 100
+
+    # Trains the classifier from an IO stream.
+    # Each line in the stream is treated as a separate document.
+    #
+    # @rbs (Symbol | String, IO, ?batch_size: Integer) { (Progress) -> void } -> void
+    def train_from_stream(category, io, batch_size: DEFAULT_BATCH_SIZE, &block)
+      raise NotImplementedError, "#{self.class} must implement train_from_stream"
+    end
+
+    # Trains the classifier with an array of documents in batches.
+    # Supports both positional and keyword argument styles.
+    #
+    # @example Positional style
+    #   classifier.train_batch(:spam, documents, batch_size: 100)
+    #
+    # @example Keyword style
+    #   classifier.train_batch(spam: documents, ham: other_docs, batch_size: 100)
+    #
+    # @rbs (?(Symbol | String)?, ?Array[String]?, ?batch_size: Integer, **Array[String]) { (Progress) -> void } -> void
+    def train_batch(category = nil, documents = nil, batch_size: DEFAULT_BATCH_SIZE, **categories, &block)
+      raise NotImplementedError, "#{self.class} must implement train_batch"
+    end
+
+    # Saves a checkpoint of the current training state.
+    # Requires a storage backend to be configured.
+    #
+    # @rbs (String) -> void
+    def save_checkpoint(checkpoint_id)
+      raise ArgumentError, 'No storage configured' unless respond_to?(:storage) && storage
+
+      original_storage = storage
+
+      begin
+        self.storage = checkpoint_storage_for(checkpoint_id)
+        save
+      ensure
+        self.storage = original_storage
+      end
+    end
+
+    # Lists available checkpoints.
+    # Requires a storage backend to be configured.
+    #
+    # @rbs () -> Array[String]
+    def list_checkpoints
+      raise ArgumentError, 'No storage configured' unless respond_to?(:storage) && storage
+
+      case storage
+      when Storage::File
+        file_storage = storage #: Storage::File
+        dir = File.dirname(file_storage.path)
+        base = File.basename(file_storage.path, '.*')
+        ext = File.extname(file_storage.path)
+
+        pattern = File.join(dir, "#{base}_checkpoint_*#{ext}")
+        Dir.glob(pattern).map do |path|
+          File.basename(path, ext).sub(/^#{Regexp.escape(base)}_checkpoint_/, '')
+        end.sort
+      else
+        []
+      end
+    end
+
+    # Deletes a checkpoint.
+    #
+    # @rbs (String) -> void
+    def delete_checkpoint(checkpoint_id)
+      raise ArgumentError, 'No storage configured' unless respond_to?(:storage) && storage
+
+      checkpoint_storage = checkpoint_storage_for(checkpoint_id)
+      checkpoint_storage.delete if checkpoint_storage.exists?
+    end
+
+    private
+
+    # @rbs (String) -> String
+    def checkpoint_path_for(checkpoint_id)
+      raise ArgumentError, 'Storage must be File storage for checkpoints' unless storage.is_a?(Storage::File)
+
+      file_storage = storage #: Storage::File
+      dir = File.dirname(file_storage.path)
+      base = File.basename(file_storage.path, '.*')
+      ext = File.extname(file_storage.path)
+
+      File.join(dir, "#{base}_checkpoint_#{checkpoint_id}#{ext}")
+    end
+
+    # @rbs (String) -> Storage::Base
+    def checkpoint_storage_for(checkpoint_id)
+      case storage
+      when Storage::File
+        Storage::File.new(path: checkpoint_path_for(checkpoint_id))
+      else
+        raise ArgumentError, "Checkpoints not supported for #{storage.class}"
+      end
+    end
+  end
+end