classifier 2.6.0 → 2.7.0

data/lib/classifier/keywords/cli.rb

@@ -0,0 +1,299 @@
+# rbs_inline: enabled
+
+require 'optparse'
+require 'fileutils'
+require 'stringio'
+require 'classifier'
+
+module Classifier
+  module Keywords
+    class CLI
+      class UsageError < StandardError; end
+
+      # @rbs @args: Array[String]
+      # @rbs @stdin: String?
+      # @rbs @options: Hash[Symbol, untyped]
+      # @rbs @output: Array[String]
+      # @rbs @error: Array[String]
+      # @rbs @exit_code: Integer
+      # @rbs @parser: OptionParser
+
+      def initialize(args, stdin: nil)
+        @args = args.dup
+        @stdin = stdin
+        @options = {
+          model: File.expand_path('./keywords.json'),
+          top: nil,
+          quiet: false,
+          min_df: 1,
+          max_df: 1.0,
+          ngram_range: [1, 1]
+        }
+        @output = []
+        @error = []
+        @exit_code = 0
+      end
+
+      def run
+        parse_options
+        execute_command
+        { output: @output.join("\n"), error: @error.join("\n"), exit_code: @exit_code }
+      rescue OptionParser::InvalidOption, OptionParser::MissingArgument,
+             OptionParser::InvalidArgument, UsageError => e
+        @error << "Error: #{e.message}"
+        @exit_code = 2
+        { output: @output.join("\n"), error: @error.join("\n"), exit_code: @exit_code }
+      rescue StandardError => e
+        @error << "Error: #{e.message}"
+        @exit_code = 1
+        { output: @output.join("\n"), error: @error.join("\n"), exit_code: @exit_code }
+      end
+
+      private
+
+      def parse_options
+        @parser = OptionParser.new do |opts|
+          opts.banner = 'Usage: keywords [text] [options] [command] [arguments]'
+          opts.separator ''
+          opts.separator 'Commands:'
+          opts.separator '  fit <files...>     Fit the model from files or stdin (each line is treated as a separate document)'
+          opts.separator '  extract <file>     Extract keywords from a file'
+          opts.separator '  info               Show model information'
+          opts.separator '  <text>             Get weighted terms from text'
+          opts.separator ''
+          opts.separator 'Options:'
+
+          opts.on('-m', '--model FILE', 'Model file (default: ./keywords.json)') do |file|
+            @options[:model] = File.expand_path(file)
+          end
+
+          opts.on('-n', '--top N', Integer, 'Show top N terms only') do |n|
+            raise OptionParser::InvalidArgument, 'must be positive' unless n.positive?
+
+            @options[:top] = n
+          end
+
+          opts.on('--min-df N', Integer, 'Minimum document frequency (default: 1)') do |n|
+            raise OptionParser::InvalidArgument, 'must be non-negative' if n.negative?
+
+            @options[:min_df] = n
+          end
+
+          opts.on('--max-df N', Float, 'Maximum document frequency ratio (default: 1.0)') do |n|
+            raise OptionParser::InvalidArgument, 'must be between 0.0 and 1.0' unless n.between?(0.0, 1.0)
+
+            @options[:max_df] = n
+          end
+
+          opts.on('--ngram MIN,MAX', Array, 'N-gram range (default: 1,1)') do |range|
+            raise OptionParser::InvalidArgument, 'requires exactly two values' if range.count != 2
+
+            raise OptionParser::InvalidArgument, 'must be integers' unless range.all? { |n| n =~ /\A\d+\z/ }
+
+            min, max = range.map(&:to_i)
+
+            raise OptionParser::InvalidArgument, 'bounds must be >= 1 and min <= max' if min < 1 || max < 1 || min > max
+
+            @options[:ngram_range] = [min, max]
+          end
+
+          opts.on('-q', 'Quiet mode') do
+            @options[:quiet] = true
+          end
+
+          opts.on('-v', '--version', 'Show version') do
+            @output << Classifier::VERSION
+            @exit_code = 0
+            throw :done
+          end
+
+          opts.on('-h', '--help', 'Show help') do
+            @output << opts.to_s
+            @exit_code = 0
+            throw :done
+          end
+        end
+
+        catch(:done) do
+          @parser.parse!(@args)
+        end
+      end
+
+      def execute_command
+        return if @exit_code != 0 || @output.any?
+
+        command = @args.first
+
+        case command
+        when 'fit'
+          command_fit
+        when 'extract'
+          command_extract
+        when 'info'
+          command_info
+        else
+          command_keywords
+        end
+      end
+
+      def command_fit
+        @args.shift
+
+        # @type var sources: Array[IO | StringIO | String]
+        sources =
+          if @args.empty?
+            [@stdin ? StringIO.new(@stdin.to_s) : $stdin]
+          else
+            collect_files
+          end
+
+        tfidf = TFIDF.new(
+          min_df: @options[:min_df],
+          max_df: @options[:max_df],
+          ngram_range: @options[:ngram_range]
+        )
+        tfidf.fit_from_stream(Streaming::MultiIO.new(sources))
+
+        raise UsageError, 'No documents found to save the model' if tfidf.num_documents.zero?
+
+        tfidf.save_to_file(@options[:model])
+        @output << "Saved to #{@options[:model].inspect}" unless @options[:quiet]
+      end
+
+      def collect_files
+        files = @args.flat_map do |arg|
+          matches = Dir.glob(arg)
+          raise UsageError, "No files matched #{arg.inspect}" if matches.empty?
+
+          matches.select { |f| File.file?(f) }.map { |f| File.expand_path(f) }
+        end.uniq
+
+        raise UsageError, 'No files to fit' if files.empty?
+
+        files
+      end
+
+      def command_extract
+        @args.shift
+
+        document =
+          if @args.empty?
+            @stdin ? @stdin.to_s : $stdin.read
+          else
+            file = File.expand_path(@args.first)
+            raise UsageError, "File #{file.inspect} does not exist" unless File.exist?(file)
+            raise UsageError, "#{file.inspect} is a directory, not a file" if File.directory?(file)
+
+            File.read(file)
+          end
+
+        transform(document)
+      end
+
+      def command_info
+        ensure_model_exists!
+
+        @args.shift
+
+        tfidf = TFIDF.load_from_file(@options[:model])
+        documents = number_with_delimiter(tfidf.num_documents)
+        vocabulary = number_with_delimiter(tfidf.vocabulary.count)
+        min_df = tfidf.min_df
+        max_df = tfidf.max_df
+        @output << format(
+          "Documents: %<documents>s\nVocabulary: %<vocabulary>s\n" \
+          "Min DF: %<min_df>d\nMax DF: %<max_df>.1f",
+          documents: documents, vocabulary: vocabulary, min_df: min_df, max_df: max_df
+        )
+      end
+
+      def command_keywords
+        if @args.empty? && @stdin.nil? && $stdin.tty?
+          show_getting_started
+          return
+        end
+
+        document =
+          if @args.empty?
+            @stdin ? @stdin.to_s : $stdin.read
+          else
+            @args.join(' ')
+          end
+
+        transform(document)
+      end
+
+      def show_getting_started
+        @output << 'Keywords - Keyword extraction and term analysis using TF-IDF'
+        @output << ''
+        @output << 'Get started by building a vocabulary (fitting data):'
+        @output << ''
+        @output << '  # Fit from files'
+        @output << '  keywords fit corpus/*.txt'
+        @output << ''
+        @output << '  # Fit from stdin'
+        @output << '  cat documents.txt | keywords fit'
+        @output << ''
+        @output << '  # Keep in mind that each line is treated as a separate document.'
+        @output << ''
+        @output << 'Then extract weighted terms and analyze text:'
+        @output << ''
+        @output << '  # Extract from string'
+        @output << "  keywords 'Ruby is a programming language'"
+        @output << '  # ruby:0.52 programming:0.41 language:0.38'
+        @output << ''
+        @output << '  # Extract from file (convenience alias)'
+        @output << '  keywords extract article.txt'
+        @output << ''
+        @output << '  # Pipeline with stdin and web data'
+        @output << '  curl -s https://example.com/article | keywords extract'
+        @output << ''
+        @output << 'Check model statistics:'
+        @output << ''
+        @output << '  keywords info'
+        @output << '  # Documents: 1,234'
+        @output << '  # Vocabulary: 5,678'
+        @output << '  # Min DF: 1'
+        @output << '  # Max DF: 1.0'
+        @output << ''
+        @output << 'General Options:'
+        @output << '  -m, --model FILE       Model file (default: ./keywords.json)'
+        @output << '  -n, --top N            Show top N terms only (e.g. keywords -n 5 "text...")'
+        @output << ''
+        @output << 'Fit-specific Options:'
+        @output << '  --min-df N             Minimum document frequency (default: 1)'
+        @output << '  --max-df N             Maximum document frequency ratio (default: 1.0)'
+        @output << '  --ngram MIN,MAX        N-gram range (default: 1,1)'
+        @output << ''
+        @output << 'Run "keywords --help" for full usage.'
+      end
+
+      def transform(document)
+        ensure_model_exists!
+
+        stem_map = document.stem_to_word_hash
+        tfidf = TFIDF.load_from_file(@options[:model])
+        vector = tfidf.transform(document).sort_by { |_, v| v }.reverse
+        vector = vector.first(@options[:top]) if @options[:top]
+        @output << vector.map { |k, v| "#{label_for(k, stem_map)}:#{v.round(2)}" }.join(' ')
+      end
+
+      def ensure_model_exists!
+        return if File.exist?(@options[:model])
+
+        raise UsageError, "No model found; run 'keywords fit' first or " \
+                          "pass correct model using the '-m' option."
+      end
+
+      def label_for(key, stem_map)
+        stem_map.fetch(key.to_sym) do
+          key.to_s.split('_').map { |part| stem_map[part.to_sym] || part }.join(' ')
+        end
+      end
+
+      def number_with_delimiter(number, delimiter: ',')
+        number.to_s.reverse.scan(/\d{1,3}/).join(delimiter).reverse
+      end
+    end
+  end
+end

data/lib/classifier/lsi.rb

@@ -348,7 +348,7 @@ module Classifier
         avg_density = {}
         @items.each_key { |x| avg_density[x] = proximity_array_for_content_unlocked(x).sum { |pair| pair[1] } }
 
-        avg_density.keys.sort_by { |x| avg_density[x] }.reverse[0..(max_chunks - 1)].map
+        avg_density.keys.sort_by { |x| avg_density[x] }.reverse[0..(max_chunks - 1)]
       end
     end
 
@@ -480,8 +480,8 @@ module Classifier
         raise 'Requested stem ranking on non-indexed content!' unless @items[doc]
 
         arr = node_for_content_unlocked(doc).lsi_vector.to_a
-        top_n = arr.sort.reverse[0..(count - 1)]
-        top_n.collect { |x| @word_list.word_for_index(arr.index(x)) }
+        top_indices = arr.each_index.sort_by { |index| -arr[index] }.first(count)
+        top_indices.collect { |index| @word_list.word_for_index(index) }
       end
     end
 

data/lib/classifier/streaming/line_reader.rb

@@ -15,14 +15,14 @@ module Classifier
     class LineReader
       include Enumerable #[String]
 
-      # @rbs @io: IO
+      # @rbs @io: IO | Classifier::Streaming::MultiIO
       # @rbs @batch_size: Integer
 
       attr_reader :batch_size
 
       # Creates a new LineReader.
       #
-      # @rbs (IO, ?batch_size: Integer) -> void
+      # @rbs (IO | Classifier::Streaming::MultiIO, ?batch_size: Integer) -> void
       def initialize(io, batch_size: 100)
         @io = io
         @batch_size = batch_size
@@ -68,27 +68,29 @@ module Classifier
       def estimate_line_count(sample_size: 100)
         return nil unless @io.respond_to?(:size) && @io.respond_to?(:rewind)
 
+        io = @io #: ::IO
+
         begin
-          original_pos = @io.pos
-          @io.rewind
+          original_pos = io.pos
+          io.rewind
 
           sample_bytes = 0
           sample_lines = 0
 
           sample_size.times do
-            line = @io.gets
+            line = io.gets
             break unless line
 
             sample_bytes += line.bytesize
             sample_lines += 1
           end
 
-          @io.seek(original_pos)
+          io.seek(original_pos)
 
           return nil if sample_lines.zero?
 
           avg_line_size = sample_bytes.to_f / sample_lines
-          io_size = @io.__send__(:size) #: Integer
+          io_size = io.__send__(:size) #: Integer
           (io_size / avg_line_size).round
         rescue IOError, Errno::ESPIPE
           nil

data/lib/classifier/streaming/multi_io.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Classifier
+  module Streaming
+    # A utility class that wraps multiple IO-like streams and treats them as a single,
+    # sequential stream.
+    #
+    # `MultiIO` allows you to iterate over lines across multiple input sources
+    # (such as files, standard input, or string buffers) seamlessly in the order
+    # they are provided.
+    #
+    # If a string is passed as an argument, it is automatically treated as a
+    # file path and opened. The file will be safely closed immediately after
+    # the iteration is complete.
+    #
+    # @example Reading from multiple files sequentially
+    #   log1 = File.open("syslog.log")
+    #   log2 = File.open("auth.log")
+    #   path = '/var/log/nginx/access.log'
+    #
+    #   multi = MultiIO.new([log1, log2, path])
+    #   multi.each_line do |line|
+    #     puts line if line.include?("ERROR")
+    #   end
+    #
+    class MultiIO
+      # @rbs @sources: Array[IO | String]
+
+      def initialize(sources)
+        @sources = sources.dup
+      end
+
+      # @rbs () { (String) -> void } -> void
+      # @rbs () -> Enumerator[String, untyped]
+      def each_line
+        # rubocop:disable Style/ExplicitBlockArgument
+        return enum_for(:each_line) unless block_given?
+
+        @sources.each do |source|
+          if source.is_a?(String)
+            File.open(source) { |io| io.each_line { |line| yield line } }
+          else
+            source.each_line { |line| yield line }
+          end
+        end
+        # rubocop:enable Style/ExplicitBlockArgument
+      end
+    end
+  end
+end

data/lib/classifier/streaming.rb

@@ -2,6 +2,7 @@
 
 require_relative 'streaming/progress'
 require_relative 'streaming/line_reader'
+require_relative 'streaming/multi_io'
 
 module Classifier
   # Streaming module provides memory-efficient training capabilities for classifiers.

data/lib/classifier/tfidf.rb

@@ -30,7 +30,7 @@ module Classifier
     # @rbs @storage: Storage::Base?
     # @rbs @min_word_length: Integer
 
-    attr_reader :vocabulary, :idf, :num_documents
+    attr_reader :vocabulary, :idf, :num_documents, :min_df, :max_df
     attr_accessor :storage
 
     # Creates a new TF-IDF vectorizer.
@@ -285,7 +285,8 @@ module Classifier
     #     puts "#{progress.completed} documents loaded"
     #   end
     #
-    # @rbs (IO, ?batch_size: Integer) { (Streaming::Progress) -> void } -> self
+    # @rbs (IO | Classifier::Streaming::MultiIO, ?batch_size: Integer) -> self
+    # @rbs (IO | Classifier::Streaming::MultiIO, ?batch_size: Integer) { (Streaming::Progress) -> void } -> self
     def fit_from_stream(io, batch_size: Streaming::DEFAULT_BATCH_SIZE)
       reader = Streaming::LineReader.new(io, batch_size: batch_size)
       total = reader.estimate_line_count

data/lib/classifier/version.rb

@@ -1,3 +1,3 @@
 module Classifier
-  VERSION = '2.6.0'.freeze
+  VERSION = '2.7.0'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: classifier
 version: !ruby/object:Gem::Version
-  version: 2.6.0
+  version: 2.7.0
 platform: ruby
 authors:
 - Lucas Carlson
@@ -142,16 +142,29 @@ description: A Ruby library for text classification featuring Naive Bayes, LSI (
 email: lucas@rufy.com
 executables:
 - classifier
+- keywords
 extensions:
 - ext/classifier/extconf.rb
 extra_rdoc_files: []
 files:
-- CLAUDE.md
+- CHANGELOG.md
 - LICENSE
 - README.md
 - bin/bayes.rb
 - bin/summarize.rb
+- docs/README.md
+- docs/bayes.md
+- docs/cli.md
+- docs/configuration.md
+- docs/keywords.md
+- docs/knn.md
+- docs/logistic-regression.md
+- docs/lsi.md
+- docs/persistence.md
+- docs/streaming.md
+- docs/tfidf.md
 - exe/classifier
+- exe/keywords
 - ext/classifier/classifier_ext.c
 - ext/classifier/extconf.rb
 - ext/classifier/incremental_svd.c
@@ -167,6 +180,7 @@ files:
 - lib/classifier/extensions/string.rb
 - lib/classifier/extensions/vector.rb
 - lib/classifier/extensions/word_hash.rb
+- lib/classifier/keywords/cli.rb
 - lib/classifier/knn.rb
 - lib/classifier/logistic_regression.rb
 - lib/classifier/lsi.rb
@@ -180,6 +194,7 @@ files:
 - lib/classifier/storage/memory.rb
 - lib/classifier/streaming.rb
 - lib/classifier/streaming/line_reader.rb
+- lib/classifier/streaming/multi_io.rb
 - lib/classifier/streaming/progress.rb
 - lib/classifier/tfidf.rb
 - lib/classifier/version.rb
@@ -199,7 +214,7 @@ metadata:
   documentation_uri: https://rubyclassifier.com/docs
   source_code_uri: https://github.com/cardmagic/classifier
   bug_tracker_uri: https://github.com/cardmagic/classifier/issues
-  changelog_uri: https://github.com/cardmagic/classifier/releases
+  changelog_uri: https://github.com/cardmagic/classifier/blob/master/CHANGELOG.md
 rdoc_options: []
 require_paths:
 - lib
@@ -214,7 +229,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Text classification with Bayesian, LSI, Logistic Regression, kNN, and TF-IDF
   vectorization.

data/CLAUDE.md

@@ -1,77 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## Project Overview
-
-Ruby gem providing text classification via two algorithms:
-- **Bayes** (`Classifier::Bayes`) - Naive Bayesian classification
-- **LSI** (`Classifier::LSI`) - Latent Semantic Indexing for semantic classification, clustering, and search
-
-## Common Commands
-
-```bash
-# Compile native C extension
-bundle exec rake compile
-
-# Run all tests (compiles first)
-bundle exec rake test
-
-# Run a single test file
-ruby -Ilib test/bayes/bayesian_test.rb
-ruby -Ilib test/lsi/lsi_test.rb
-
-# Run tests with pure Ruby (no native extension)
-NATIVE_VECTOR=true bundle exec rake test
-
-# Run benchmarks
-bundle exec rake benchmark
-bundle exec rake benchmark:compare
-
-# Interactive console
-bundle exec rake console
-
-# Generate documentation
-bundle exec rake doc
-```
-
-## Architecture
-
-### Core Components
-
-**Bayesian Classifier** (`lib/classifier/bayes.rb`)
-- Train with `train(category, text)` or dynamic methods like `train_spam(text)`
-- Classify with `classify(text)` returning the best category
-- Uses log probabilities for numerical stability
-
-**LSI Classifier** (`lib/classifier/lsi.rb`)
-- Uses Singular Value Decomposition (SVD) for semantic analysis
-- Native C extension for 5-50x faster matrix operations; falls back to pure Ruby
-- Key operations: `add_item`, `classify`, `find_related`, `search`
-- `auto_rebuild` option controls automatic index rebuilding after changes
-
-**String Extensions** (`lib/classifier/extensions/word_hash.rb`)
-- `word_hash` / `clean_word_hash` - tokenize text to stemmed word frequencies
-- `CORPUS_SKIP_WORDS` - stopwords filtered during tokenization
-- Uses `fast-stemmer` gem for Porter stemming
-
-**Vector Extensions** (`lib/classifier/extensions/vector.rb`)
-- Pure Ruby SVD implementation (`Matrix#SV_decomp`) - used as fallback
-- Vector normalization and magnitude calculations
-
-### Native C Extension (`ext/classifier/`)
-
-LSI uses a native C extension for fast linear algebra operations:
-- `Classifier::Linalg::Vector` - Vector operations (alloc, normalize, dot product)
-- `Classifier::Linalg::Matrix` - Matrix operations (alloc, transpose, multiply)
-- Jacobi SVD implementation for singular value decomposition
-
-Check current backend: `Classifier::LSI.backend` returns `:native` or `:ruby`
-Force pure Ruby: `NATIVE_VECTOR=true bundle exec rake test`
-
-### Content Nodes (`lib/classifier/lsi/content_node.rb`)
-
-Internal data structure storing:
-- `word_hash` - term frequencies
-- `raw_vector` / `raw_norm` - initial vector representation
-- `lsi_vector` / `lsi_norm` - reduced dimensionality representation after SVD