classifier 2.0.0 → 2.2.0

data/lib/classifier/bayes.rb

@@ -4,68 +4,68 @@
 # Copyright:: Copyright (c) 2005 Lucas Carlson
 # License::   LGPL
 
+require 'json'
+require 'mutex_m'
+
 module Classifier
-  class Bayes
+  class Bayes # rubocop:disable Metrics/ClassLength
+    include Mutex_m
+    include Streaming
+
     # @rbs @categories: Hash[Symbol, Hash[Symbol, Integer]]
     # @rbs @total_words: Integer
     # @rbs @category_counts: Hash[Symbol, Integer]
     # @rbs @category_word_count: Hash[Symbol, Integer]
+    # @rbs @cached_training_count: Float?
+    # @rbs @cached_vocab_size: Integer?
+    # @rbs @dirty: bool
+    # @rbs @storage: Storage::Base?
+
+    attr_accessor :storage
 
     # The class can be created with one or more categories, each of which will be
     # initialized and given a training method. E.g.,
     #      b = Classifier::Bayes.new 'Interesting', 'Uninteresting', 'Spam'
-    # @rbs (*String | Symbol) -> void
+    #      b = Classifier::Bayes.new ['Interesting', 'Uninteresting', 'Spam']
+    # @rbs (*String | Symbol | Array[String | Symbol]) -> void
     def initialize(*categories)
+      super()
       @categories = {}
-      categories.each { |category| @categories[category.prepare_category_name] = {} }
+      categories.flatten.each { |category| @categories[category.prepare_category_name] = {} }
       @total_words = 0
       @category_counts = Hash.new(0)
       @category_word_count = Hash.new(0)
+      @cached_training_count = nil
+      @cached_vocab_size = nil
+      @dirty = false
+      @storage = nil
     end
 
-    # Provides a general training method for all categories specified in Bayes#new
-    # For example:
-    #     b = Classifier::Bayes.new 'This', 'That', 'the_other'
-    #     b.train :this, "This text"
-    #     b.train "that", "That text"
-    #     b.train "The other", "The other text"
+    # Trains the classifier with text for a category.
     #
-    # @rbs (String | Symbol, String) -> void
-    def train(category, text)
-      category = category.prepare_category_name
-      @category_counts[category] += 1
-      text.word_hash.each do |word, count|
-        @categories[category][word] ||= 0
-        @categories[category][word] += count
-        @total_words += count
-        @category_word_count[category] += count
+    #     b.train(spam: "Buy now!", ham: ["Hello", "Meeting tomorrow"])
+    #     b.train(:spam, "legacy positional API")
+    #
+    # @rbs (?(String | Symbol)?, ?String?, **(String | Array[String])) -> void
+    def train(category = nil, text = nil, **categories)
+      return train_single(category, text) if category && text
+
+      categories.each do |cat, texts|
+        (texts.is_a?(Array) ? texts : [texts]).each { |t| train_single(cat, t) }
       end
     end
 
-    # Provides a untraining method for all categories specified in Bayes#new
-    # Be very careful with this method.
+    # Removes training data. Be careful with this method.
     #
-    # For example:
-    #     b = Classifier::Bayes.new 'This', 'That', 'the_other'
-    #     b.train :this, "This text"
-    #     b.untrain :this, "This text"
+    #     b.untrain(spam: "Buy now!")
+    #     b.untrain(:spam, "legacy positional API")
     #
-    # @rbs (String | Symbol, String) -> void
-    def untrain(category, text)
-      category = category.prepare_category_name
-      @category_counts[category] -= 1
-      text.word_hash.each do |word, count|
-        next unless @total_words >= 0
-
-        orig = @categories[category][word] || 0
-        @categories[category][word] ||= 0
-        @categories[category][word] -= count
-        if @categories[category][word] <= 0
-          @categories[category].delete(word)
-          count = orig
-        end
-        @category_word_count[category] -= count if @category_word_count[category] >= count
-        @total_words -= count
+    # @rbs (?(String | Symbol)?, ?String?, **(String | Array[String])) -> void
+    def untrain(category = nil, text = nil, **categories)
+      return untrain_single(category, text) if category && text
+
+      categories.each do |cat, texts|
+        (texts.is_a?(Array) ? texts : [texts]).each { |t| untrain_single(cat, t) }
       end
     end
 
@@ -77,17 +77,19 @@ module Classifier
     # @rbs (String) -> Hash[String, Float]
     def classifications(text)
       words = text.word_hash.keys
-      training_count = @category_counts.values.sum.to_f
-      vocab_size = [@categories.values.flat_map(&:keys).uniq.size, 1].max
+      synchronize do
+        training_count = cached_training_count
+        vocab_size = cached_vocab_size
 
-      @categories.to_h do |category, category_words|
-        smoothed_total = ((@category_word_count[category] || 0) + vocab_size).to_f
+        @categories.to_h do |category, category_words|
+          smoothed_total = ((@category_word_count[category] || 0) + vocab_size).to_f
 
-        # Laplace smoothing: P(word|category) = (count + α) / (total + α * V)
-        word_score = words.sum { |w| Math.log(((category_words[w] || 0) + 1) / smoothed_total) }
-        prior_score = Math.log((@category_counts[category] || 0.1) / training_count)
+          # Laplace smoothing: P(word|category) = (count + α) / (total + α * V)
+          word_score = words.sum { |w| Math.log(((category_words[w] || 0) + 1) / smoothed_total) }
+          prior_score = Math.log((@category_counts[category] || 0.1) / training_count)
 
-        [category.to_s, word_score + prior_score]
+          [category.to_s, word_score + prior_score]
+        end
       end
     end
 
@@ -104,6 +106,119 @@ module Classifier
       best.first.to_s
     end
 
+    # Returns a hash representation of the classifier state.
+    # This can be converted to JSON or used directly.
+    #
+    # @rbs (?untyped) -> untyped
+    def as_json(_options = nil)
+      {
+        version: 1,
+        type: 'bayes',
+        categories: @categories.transform_keys(&:to_s).transform_values { |v| v.transform_keys(&:to_s) },
+        total_words: @total_words,
+        category_counts: @category_counts.transform_keys(&:to_s),
+        category_word_count: @category_word_count.transform_keys(&:to_s)
+      }
+    end
+
+    # Serializes the classifier state to a JSON string.
+    # This can be saved to a file and later loaded with Bayes.from_json.
+    #
+    # @rbs (?untyped) -> String
+    def to_json(_options = nil)
+      as_json.to_json
+    end
+
+    # Loads a classifier from a JSON string or a Hash created by #to_json or #as_json.
+    #
+    # @rbs (String | Hash[String, untyped]) -> Bayes
+    def self.from_json(json)
+      data = json.is_a?(String) ? JSON.parse(json) : json
+      raise ArgumentError, "Invalid classifier type: #{data['type']}" unless data['type'] == 'bayes'
+
+      instance = allocate
+      instance.send(:restore_state, data)
+      instance
+    end
+
+    # Saves the classifier to the configured storage.
+    # Raises ArgumentError if no storage is configured.
+    #
+    # @rbs () -> void
+    def save
+      raise ArgumentError, 'No storage configured. Use save_to_file(path) or set storage=' unless storage
+
+      storage.write(to_json)
+      @dirty = false
+    end
+
+    # Saves the classifier state to a file (legacy API).
+    #
+    # @rbs (String) -> Integer
+    def save_to_file(path)
+      result = File.write(path, to_json)
+      @dirty = false
+      result
+    end
+
+    # Reloads the classifier from the configured storage.
+    # Raises UnsavedChangesError if there are unsaved changes.
+    # Use reload! to force reload and discard changes.
+    #
+    # @rbs () -> self
+    def reload
+      raise ArgumentError, 'No storage configured' unless storage
+      raise UnsavedChangesError, 'Unsaved changes would be lost. Call save first or use reload!' if @dirty
+
+      data = storage.read
+      raise StorageError, 'No saved state found' unless data
+
+      restore_from_json(data)
+      @dirty = false
+      self
+    end
+
+    # Force reloads the classifier from storage, discarding any unsaved changes.
+    #
+    # @rbs () -> self
+    def reload!
+      raise ArgumentError, 'No storage configured' unless storage
+
+      data = storage.read
+      raise StorageError, 'No saved state found' unless data
+
+      restore_from_json(data)
+      @dirty = false
+      self
+    end
+
+    # Returns true if there are unsaved changes.
+    #
+    # @rbs () -> bool
+    def dirty?
+      @dirty
+    end
+
+    # Loads a classifier from the configured storage.
+    # The storage is set on the returned instance.
+    #
+    # @rbs (storage: Storage::Base) -> Bayes
+    def self.load(storage:)
+      data = storage.read
+      raise StorageError, 'No saved state found' unless data
+
+      instance = from_json(data)
+      instance.storage = storage
+      instance
+    end
+
+    # Loads a classifier from a file (legacy API).
+    #
+    # @rbs (String) -> Bayes
+    def self.load_from_file(path)
+      from_json(File.read(path))
+    end
+
     #
     # Provides training and untraining methods for the categories specified in Bayes#new
     # For example:
@@ -134,7 +249,7 @@ module Classifier
     #
     # @rbs () -> Array[String]
     def categories
-      @categories.keys.collect(&:to_s)
+      synchronize { @categories.keys.collect(&:to_s) }
     end
 
     # Allows you to add categories to the classifier.
@@ -148,11 +263,31 @@ module Classifier
     #
     # @rbs (String | Symbol) -> Hash[Symbol, Integer]
     def add_category(category)
-      @categories[category.prepare_category_name] = {}
+      synchronize do
+        invalidate_caches
+        @dirty = true
+        @categories[category.prepare_category_name] = {}
+      end
     end
 
     alias append_category add_category
 
+    # Custom marshal serialization to exclude mutex state
+    # @rbs () -> Array[untyped]
+    def marshal_dump
+      [@categories, @total_words, @category_counts, @category_word_count, @dirty]
+    end
+
+    # Custom marshal deserialization to recreate mutex
+    # @rbs (Array[untyped]) -> void
+    def marshal_load(data)
+      mu_initialize
+      @categories, @total_words, @category_counts, @category_word_count, @dirty = data
+      @cached_training_count = nil
+      @cached_vocab_size = nil
+      @storage = nil
+    end
+
     # Allows you to remove categories from the classifier.
     # For example:
     #     b.remove_category "Spam"
@@ -163,14 +298,223 @@ module Classifier
     #
     # @rbs (String | Symbol) -> void
     def remove_category(category)
+      category = category.prepare_category_name
+      synchronize do
+        raise StandardError, "No such category: #{category}" unless @categories.key?(category)
+
+        invalidate_caches
+        @dirty = true
+        @total_words -= @category_word_count[category].to_i
+
+        @categories.delete(category)
+        @category_counts.delete(category)
+        @category_word_count.delete(category)
+      end
+    end
+
+    # Trains the classifier from an IO stream.
+    # Each line in the stream is treated as a separate document.
+    # This is memory-efficient for large corpora.
+    #
+    # @example Train from a file
+    #   classifier.train_from_stream(:spam, File.open('spam_corpus.txt'))
+    #
+    # @example With progress tracking
+    #   classifier.train_from_stream(:spam, io, batch_size: 500) do |progress|
+    #     puts "#{progress.completed} documents processed"
+    #   end
+    #
+    # @rbs (String | Symbol, IO, ?batch_size: Integer) { (Streaming::Progress) -> void } -> void
+    def train_from_stream(category, io, batch_size: Streaming::DEFAULT_BATCH_SIZE)
       category = category.prepare_category_name
       raise StandardError, "No such category: #{category}" unless @categories.key?(category)
 
-      @total_words -= @category_word_count[category].to_i
+      reader = Streaming::LineReader.new(io, batch_size: batch_size)
+      total = reader.estimate_line_count
+      progress = Streaming::Progress.new(total: total)
+
+      reader.each_batch do |batch|
+        train_batch_internal(category, batch)
+        progress.completed += batch.size
+        progress.current_batch += 1
+        yield progress if block_given?
+      end
+    end
+
+    # Trains the classifier with an array of documents in batches.
+    # Reduces lock contention by processing multiple documents per synchronize call.
+    #
+    # @example Positional style
+    #   classifier.train_batch(:spam, documents, batch_size: 100)
+    #
+    # @example Keyword style
+    #   classifier.train_batch(spam: documents, ham: other_docs, batch_size: 100)
+    #
+    # @example With progress tracking
+    #   classifier.train_batch(:spam, documents, batch_size: 100) do |progress|
+    #     puts "#{progress.percent}% complete"
+    #   end
+    #
+    # @rbs (?(String | Symbol)?, ?Array[String]?, ?batch_size: Integer, **Array[String]) { (Streaming::Progress) -> void } -> void
+    def train_batch(category = nil, documents = nil, batch_size: Streaming::DEFAULT_BATCH_SIZE, **categories, &block)
+      if category && documents
+        train_batch_for_category(category, documents, batch_size: batch_size, &block)
+      else
+        categories.each do |cat, docs|
+          train_batch_for_category(cat, Array(docs), batch_size: batch_size, &block)
+        end
+      end
+    end
+
+    # Loads a classifier from a checkpoint.
+    #
+    # @rbs (storage: Storage::Base, checkpoint_id: String) -> Bayes
+    def self.load_checkpoint(storage:, checkpoint_id:)
+      raise ArgumentError, 'Storage must be File storage for checkpoints' unless storage.is_a?(Storage::File)
+
+      dir = File.dirname(storage.path)
+      base = File.basename(storage.path, '.*')
+      ext = File.extname(storage.path)
+      checkpoint_path = File.join(dir, "#{base}_checkpoint_#{checkpoint_id}#{ext}")
+
+      checkpoint_storage = Storage::File.new(path: checkpoint_path)
+      instance = load(storage: checkpoint_storage)
+      instance.storage = storage
+      instance
+    end
+
+    private
+
+    # Trains a batch of documents for a single category.
+    # @rbs (String | Symbol, Array[String], ?batch_size: Integer) { (Streaming::Progress) -> void } -> void
+    def train_batch_for_category(category, documents, batch_size: Streaming::DEFAULT_BATCH_SIZE)
+      category = category.prepare_category_name
+      raise StandardError, "No such category: #{category}" unless @categories.key?(category)
+
+      progress = Streaming::Progress.new(total: documents.size)
+
+      documents.each_slice(batch_size) do |batch|
+        train_batch_internal(category, batch)
+        progress.completed += batch.size
+        progress.current_batch += 1
+        yield progress if block_given?
+      end
+    end
+
+    # Internal method to train a batch of documents.
+    # Uses a single synchronize block for the entire batch.
+    # @rbs (Symbol, Array[String]) -> void
+    def train_batch_internal(category, batch)
+      synchronize do
+        invalidate_caches
+        @dirty = true
+        batch.each do |text|
+          word_hash = text.word_hash
+          @category_counts[category] += 1
+          word_hash.each do |word, count|
+            @categories[category][word] ||= 0
+            @categories[category][word] += count
+            @total_words += count
+            @category_word_count[category] += count
+          end
+        end
+      end
+    end
+
+    # Core training logic for a single category and text.
+    # @rbs (String | Symbol, String) -> void
+    def train_single(category, text)
+      category = category.prepare_category_name
+      word_hash = text.word_hash
+      synchronize do
+        invalidate_caches
+        @dirty = true
+        @category_counts[category] += 1
+        word_hash.each do |word, count|
+          @categories[category][word] ||= 0
+          @categories[category][word] += count
+          @total_words += count
+          @category_word_count[category] += count
+        end
+      end
+    end
+
+    # Core untraining logic for a single category and text.
+    # @rbs (String | Symbol, String) -> void
+    def untrain_single(category, text)
+      category = category.prepare_category_name
+      word_hash = text.word_hash
+      synchronize do
+        invalidate_caches
+        @dirty = true
+        @category_counts[category] -= 1
+        word_hash.each do |word, count|
+          next unless @total_words >= 0
+
+          orig = @categories[category][word] || 0
+          @categories[category][word] ||= 0
+          @categories[category][word] -= count
+          if @categories[category][word] <= 0
+            @categories[category].delete(word)
+            count = orig
+          end
+          @category_word_count[category] -= count if @category_word_count[category] >= count
+          @total_words -= count
+        end
+      end
+    end
+
+    # Restores classifier state from a JSON string (used by reload)
+    # @rbs (String) -> void
+    def restore_from_json(json)
+      data = JSON.parse(json)
+      raise ArgumentError, "Invalid classifier type: #{data['type']}" unless data['type'] == 'bayes'
+
+      synchronize do
+        restore_state(data)
+      end
+    end
+
+    # Restores classifier state from a hash (used by from_json)
+    # @rbs (Hash[String, untyped]) -> void
+    def restore_state(data)
+      mu_initialize
+      @categories = {} #: Hash[Symbol, Hash[Symbol, Integer]]
+      @total_words = data['total_words']
+      @category_counts = Hash.new(0) #: Hash[Symbol, Integer]
+      @category_word_count = Hash.new(0) #: Hash[Symbol, Integer]
+      @cached_training_count = nil
+      @cached_vocab_size = nil
+      @dirty = false
+      @storage = nil
+
+      data['categories'].each do |cat_name, words|
+        @categories[cat_name.to_sym] = words.transform_keys(&:to_sym)
+      end
+
+      data['category_counts'].each do |cat_name, count|
+        @category_counts[cat_name.to_sym] = count
+      end
+
+      data['category_word_count'].each do |cat_name, count|
+        @category_word_count[cat_name.to_sym] = count
+      end
+    end
+
+    # @rbs () -> void
+    def invalidate_caches
+      @cached_training_count = nil
+      @cached_vocab_size = nil
+    end
+
+    # @rbs () -> Float
+    def cached_training_count
+      @cached_training_count ||= @category_counts.values.sum.to_f
+    end
 
-      @categories.delete(category)
-      @category_counts.delete(category)
-      @category_word_count.delete(category)
+    # @rbs () -> Integer
+    def cached_vocab_size
+      @cached_vocab_size ||= [@categories.values.flat_map(&:keys).uniq.size, 1].max
     end
   end
 end

data/lib/classifier/errors.rb

@@ -0,0 +1,19 @@
+# rbs_inline: enabled
+
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+module Classifier
+  # Base error class for all Classifier errors
+  class Error < StandardError; end
+
+  # Raised when reload would discard unsaved changes
+  class UnsavedChangesError < Error; end
+
+  # Raised when a storage operation fails
+  class StorageError < Error; end
+
+  # Raised when using an unfitted model
+  class NotFittedError < Error; end
+end

data/lib/classifier/extensions/vector.rb

@@ -21,12 +21,20 @@ end
 class Vector
   EPSILON = 1e-10
 
+  # Cache magnitude since Vector is immutable after creation
+  # Note: We undefine the matrix gem's normalize method first, then redefine it
+  # to provide a more robust implementation that handles zero vectors
+  undef_method :normalize if method_defined?(:normalize)
+
   def magnitude
-    sum_of_squares = 0.to_r
-    size.times do |i|
-      sum_of_squares += self[i]**2.to_r
+    # Cache magnitude since Vector is immutable after creation
+    @magnitude ||= begin
+      sum_of_squares = 0.to_r
+      size.times do |i|
+        sum_of_squares += self[i]**2.to_r
+      end
+      Math.sqrt(sum_of_squares.to_f)
     end
-    Math.sqrt(sum_of_squares.to_f)
   end
 
   def normalize