classifier 1.4.4 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f5bbc7714c3f1b5b6bf2b81484e071eb34c5510455d9520f8bd743ffe25a3bb6
-  data.tar.gz: f9236a1e0c086e1bda93645d94ea21f6634b76acbb4c99e62709e1b011311509
+  metadata.gz: e82726ac5c6e619e701be4591c8ed29e8b9f9a88236ad9d8d602fe3f748dcf43
+  data.tar.gz: 3771f2d2fce4992ed0cd5b1ad3849cc3e1c4fa310a144c1056e5f9d0c49579ef
 SHA512:
-  metadata.gz: 79596fac37a6591587859335d4dbb280f038c1504a6fba9a48f7f3d5c83c50bee959887a827588d7418503ff141bff81401125604ddc104af35863dd84842e28
-  data.tar.gz: 5cd28357e92c65e10630700a65097350030e401f16d4aacaaf9407e8bd8dd2d200739965bab52d547054a90bfe25ab53935e6544f06ebe3f61d356cc972d8ead
+  metadata.gz: 913111c43ffd83a1a461023c6d1f4fef0f1efb7f4e591089db7dbf3d37a1cc98cb3b5de16888019ab1caf8db83ad828dbaf82d2cab237b991216bdb27f529d15
+  data.tar.gz: b7142000fa687a58481051921663665edad1ac3b183947e79cf63bb15a79a0ca7bc66cd76104fccea713162cf05e876bd609d5f4866a13ea2c184f8caa089547

data/CLAUDE.md

@@ -0,0 +1,77 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+Ruby gem providing text classification via two algorithms:
+- **Bayes** (`Classifier::Bayes`) - Naive Bayesian classification
+- **LSI** (`Classifier::LSI`) - Latent Semantic Indexing for semantic classification, clustering, and search
+
+## Common Commands
+
+```bash
+# Compile native C extension
+bundle exec rake compile
+
+# Run all tests (compiles first)
+bundle exec rake test
+
+# Run a single test file
+ruby -Ilib test/bayes/bayesian_test.rb
+ruby -Ilib test/lsi/lsi_test.rb
+
+# Run tests with pure Ruby (no native extension)
+NATIVE_VECTOR=true bundle exec rake test
+
+# Run benchmarks
+bundle exec rake benchmark
+bundle exec rake benchmark:compare
+
+# Interactive console
+bundle exec rake console
+
+# Generate documentation
+bundle exec rake doc
+```
+
+## Architecture
+
+### Core Components
+
+**Bayesian Classifier** (`lib/classifier/bayes.rb`)
+- Train with `train(category, text)` or dynamic methods like `train_spam(text)`
+- Classify with `classify(text)` returning the best category
+- Uses log probabilities for numerical stability
+
+**LSI Classifier** (`lib/classifier/lsi.rb`)
+- Uses Singular Value Decomposition (SVD) for semantic analysis
+- Native C extension for 5-50x faster matrix operations; falls back to pure Ruby
+- Key operations: `add_item`, `classify`, `find_related`, `search`
+- `auto_rebuild` option controls automatic index rebuilding after changes
+
+**String Extensions** (`lib/classifier/extensions/word_hash.rb`)
+- `word_hash` / `clean_word_hash` - tokenize text to stemmed word frequencies
+- `CORPUS_SKIP_WORDS` - stopwords filtered during tokenization
+- Uses `fast-stemmer` gem for Porter stemming
+
+**Vector Extensions** (`lib/classifier/extensions/vector.rb`)
+- Pure Ruby SVD implementation (`Matrix#SV_decomp`) - used as fallback
+- Vector normalization and magnitude calculations
+
+### Native C Extension (`ext/classifier/`)
+
+LSI uses a native C extension for fast linear algebra operations:
+- `Classifier::Linalg::Vector` - Vector operations (alloc, normalize, dot product)
+- `Classifier::Linalg::Matrix` - Matrix operations (alloc, transpose, multiply)
+- Jacobi SVD implementation for singular value decomposition
+
+Check current backend: `Classifier::LSI.backend` returns `:native` or `:ruby`
+Force pure Ruby: `NATIVE_VECTOR=true bundle exec rake test`
+
+### Content Nodes (`lib/classifier/lsi/content_node.rb`)
+
+Internal data structure storing:
+- `word_hash` - term frequencies
+- `raw_vector` / `raw_norm` - initial vector representation
+- `lsi_vector` / `lsi_norm` - reduced dimensionality representation after SVD

data/README.md

@@ -0,0 +1,274 @@
+# Classifier
+
+[![Gem Version](https://badge.fury.io/rb/classifier.svg)](https://badge.fury.io/rb/classifier)
+[![CI](https://github.com/cardmagic/classifier/actions/workflows/ruby.yml/badge.svg)](https://github.com/cardmagic/classifier/actions/workflows/ruby.yml)
+[![License: LGPL](https://img.shields.io/badge/License-LGPL_2.1-blue.svg)](https://opensource.org/licenses/LGPL-2.1)
+
+A Ruby library for text classification using Bayesian and Latent Semantic Indexing (LSI) algorithms.
+
+**[Documentation](https://rubyclassifier.com/docs)** · **[Tutorials](https://rubyclassifier.com/docs/tutorials)** · **[Guides](https://rubyclassifier.com/docs/guides)**
+
+## Table of Contents
+
+- [Installation](#installation)
+- [Bayesian Classifier](#bayesian-classifier)
+- [LSI (Latent Semantic Indexing)](#lsi-latent-semantic-indexing)
+- [Persistence](#persistence)
+- [Performance](#performance)
+- [Development](#development)
+- [Contributing](#contributing)
+- [License](#license)
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem 'classifier'
+```
+
+Then run:
+
+```bash
+bundle install
+```
+
+Or install directly:
+
+```bash
+gem install classifier
+```
+
+### Native C Extension
+
+The gem includes a native C extension for fast LSI operations. It compiles automatically during gem installation. No external dependencies are required.
+
+To verify the native extension is active:
+
+```ruby
+require 'classifier'
+puts Classifier::LSI.backend  # => :native
+```
+
+To force pure Ruby mode (for debugging):
+
+```bash
+NATIVE_VECTOR=true ruby your_script.rb
+```
+
+To suppress the warning when native extension isn't available:
+
+```bash
+SUPPRESS_LSI_WARNING=true ruby your_script.rb
+```
+
+### Compatibility
+
+| Ruby Version | Status |
+|--------------|--------|
+| 4.0          | Supported |
+| 3.4          | Supported |
+| 3.3          | Supported |
+| 3.2          | Supported |
+| 3.1          | EOL (unsupported) |
+
+## Bayesian Classifier
+
+Fast, accurate classification with modest memory requirements. Ideal for spam filtering, sentiment analysis, and content categorization.
+
+### Quick Start
+
+```ruby
+require 'classifier'
+
+classifier = Classifier::Bayes.new('Spam', 'Ham')
+
+# Train the classifier
+classifier.train_spam "Buy cheap viagra now! Limited offer!"
+classifier.train_spam "You've won a million dollars! Claim now!"
+classifier.train_ham "Meeting scheduled for tomorrow at 10am"
+classifier.train_ham "Please review the attached document"
+
+# Classify new text
+classifier.classify "Congratulations! You've won a prize!"
+# => "Spam"
+```
+
+### Learn More
+
+- [Bayes Basics Guide](https://rubyclassifier.com/docs/guides/bayes/basics) - In-depth documentation
+- [Build a Spam Filter Tutorial](https://rubyclassifier.com/docs/tutorials/spam-filter) - Step-by-step guide
+- [Paul Graham: A Plan for Spam](http://www.paulgraham.com/spam.html)
+
+## LSI (Latent Semantic Indexing)
+
+Semantic analysis using Singular Value Decomposition (SVD). More flexible than Bayesian classifiers, providing search, clustering, and classification based on meaning rather than just keywords.
+
+### Quick Start
+
+```ruby
+require 'classifier'
+
+lsi = Classifier::LSI.new
+
+# Add documents with categories
+lsi.add_item "Dogs are loyal pets that love to play fetch", :pets
+lsi.add_item "Cats are independent and love to nap", :pets
+lsi.add_item "Ruby is a dynamic programming language", :programming
+lsi.add_item "Python is great for data science", :programming
+
+# Classify new text
+lsi.classify "My puppy loves to run around"
+# => :pets
+
+# Get classification with confidence score
+lsi.classify_with_confidence "Learning to code in Ruby"
+# => [:programming, 0.89]
+```
+
+### Search and Discovery
+
+```ruby
+# Find similar documents
+lsi.find_related "Dogs are great companions", 2
+# => ["Dogs are loyal pets that love to play fetch", "Cats are independent..."]
+
+# Search by keyword
+lsi.search "programming", 3
+# => ["Ruby is a dynamic programming language", "Python is great for..."]
+```
+
+### Learn More
+
+- [LSI Basics Guide](https://rubyclassifier.com/docs/guides/lsi/basics) - In-depth documentation
+- [Wikipedia: Latent Semantic Analysis](http://en.wikipedia.org/wiki/Latent_semantic_analysis)
+
+## Persistence
+
+Save and load trained classifiers with pluggable storage backends. Works with both Bayes and LSI classifiers.
+
+### File Storage
+
+```ruby
+require 'classifier'
+
+classifier = Classifier::Bayes.new('Spam', 'Ham')
+classifier.train_spam "Buy now! Limited offer!"
+classifier.train_ham "Meeting tomorrow at 3pm"
+
+# Configure storage and save
+classifier.storage = Classifier::Storage::File.new(path: "spam_filter.json")
+classifier.save
+
+# Load later
+loaded = Classifier::Bayes.load(storage: classifier.storage)
+loaded.classify "Claim your prize now!"
+# => "Spam"
+```
+
+### Custom Storage Backends
+
+Create backends for Redis, PostgreSQL, S3, or any storage system:
+
+```ruby
+class RedisStorage < Classifier::Storage::Base
+  def initialize(redis:, key:)
+    super()
+    @redis, @key = redis, key
+  end
+
+  def write(data) = @redis.set(@key, data)
+  def read = @redis.get(@key)
+  def delete = @redis.del(@key)
+  def exists? = @redis.exists?(@key)
+end
+
+# Use it
+classifier.storage = RedisStorage.new(redis: Redis.new, key: "classifier:spam")
+classifier.save
+```
+
+### Learn More
+
+- [Persistence Guide](https://rubyclassifier.com/docs/guides/persistence/basics) - Full documentation with examples
+
+## Performance
+
+### Native C Extension vs Pure Ruby
+
+The native C extension provides dramatic speedups for LSI operations, especially `build_index` (SVD computation):
+
+| Documents | build_index | Overall |
+|-----------|-------------|---------|
+| 5         | 7x faster   | 2.6x    |
+| 10        | 25x faster  | 4.6x    |
+| 15        | 112x faster | 14.5x   |
+| 20        | 385x faster | 48.7x   |
+
+<details>
+<summary>Detailed benchmark (20 documents)</summary>
+
+```
+Operation            Pure Ruby     Native C      Speedup
+----------------------------------------------------------
+build_index            0.5540       0.0014       384.5x
+classify               0.0190       0.0060         3.2x
+search                 0.0145       0.0037         3.9x
+find_related           0.0098       0.0011         8.6x
+----------------------------------------------------------
+TOTAL                  0.5973       0.0123        48.7x
+```
+</details>
+
+### Running Benchmarks
+
+```bash
+rake benchmark              # Run with current configuration
+rake benchmark:compare      # Compare native C vs pure Ruby
+```
+
+## Development
+
+### Setup
+
+```bash
+git clone https://github.com/cardmagic/classifier.git
+cd classifier
+bundle install
+rake compile  # Compile native C extension
+```
+
+### Running Tests
+
+```bash
+rake test                        # Run all tests (compiles first)
+ruby -Ilib test/bayes/bayesian_test.rb  # Run specific test file
+
+# Test with pure Ruby (no native extension)
+NATIVE_VECTOR=true rake test
+```
+
+### Console
+
+```bash
+rake console
+```
+
+## Contributing
+
+1. Fork the repository
+2. Create your feature branch (`git checkout -b feature/amazing-feature`)
+3. Commit your changes (`git commit -am 'Add amazing feature'`)
+4. Push to the branch (`git push origin feature/amazing-feature`)
+5. Open a Pull Request
+
+## Authors
+
+- **Lucas Carlson** - *Original author* - lucas@rufy.com
+- **David Fayram II** - *LSI implementation* - dfayram@gmail.com
+- **Cameron McBride** - cameron.mcbride@gmail.com
+- **Ivan Acosta-Rubio** - ivan@softwarecriollo.com
+
+## License
+
+This library is released under the [GNU Lesser General Public License (LGPL) 2.1](LICENSE).

data/ext/classifier/classifier_ext.c

@@ -0,0 +1,25 @@
+/*
+ * classifier_ext.c
+ * Main entry point for the Classifier native linear algebra extension
+ *
+ * This extension provides zero-dependency Vector, Matrix, and SVD
+ * implementations for the Classifier gem's LSI functionality.
+ */
+
+#include "linalg.h"
+
+VALUE mClassifierLinalg;
+VALUE cClassifierVector;
+VALUE cClassifierMatrix;
+
+void Init_classifier_ext(void)
+{
+    /* Define Classifier::Linalg module */
+    VALUE mClassifier = rb_define_module("Classifier");
+    mClassifierLinalg = rb_define_module_under(mClassifier, "Linalg");
+
+    /* Initialize Vector and Matrix classes */
+    Init_vector();
+    Init_matrix();
+    Init_svd();
+}

data/ext/classifier/extconf.rb

@@ -0,0 +1,15 @@
+require 'mkmf'
+
+# rubocop:disable Style/GlobalVars
+if ENV['COVERAGE']
+  # Coverage flags: disable optimization for accurate line coverage
+  $CFLAGS << ' -O0 -g --coverage -Wall'
+  $LDFLAGS << ' --coverage'
+else
+  # Optimization flags for performance
+  $CFLAGS << ' -O3 -ffast-math -Wall'
+end
+# rubocop:enable Style/GlobalVars
+
+# Create the Makefile
+create_makefile('classifier/classifier_ext')

data/ext/classifier/linalg.h

@@ -0,0 +1,64 @@
+#ifndef CLASSIFIER_LINALG_H
+#define CLASSIFIER_LINALG_H
+
+#include <ruby.h>
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Epsilon for numerical comparisons */
+#define CLASSIFIER_EPSILON 1e-10
+
+/* Vector structure */
+typedef struct {
+    size_t size;
+    double *data;
+    int is_col;  /* 0 = row vector, 1 = column vector */
+} CVector;
+
+/* Matrix structure */
+typedef struct {
+    size_t rows;
+    size_t cols;
+    double *data;  /* Row-major storage */
+} CMatrix;
+
+/* Ruby class references */
+extern VALUE cClassifierVector;
+extern VALUE cClassifierMatrix;
+extern VALUE mClassifierLinalg;
+
+/* Vector functions */
+void Init_vector(void);
+CVector *cvector_alloc(size_t size);
+void cvector_free(void *ptr);
+double cvector_magnitude(CVector *v);
+CVector *cvector_normalize(CVector *v);
+double cvector_sum(CVector *v);
+double cvector_dot(CVector *a, CVector *b);
+
+/* Matrix functions */
+void Init_matrix(void);
+CMatrix *cmatrix_alloc(size_t rows, size_t cols);
+void cmatrix_free(void *ptr);
+CMatrix *cmatrix_transpose(CMatrix *m);
+CMatrix *cmatrix_multiply(CMatrix *a, CMatrix *b);
+CVector *cmatrix_multiply_vector(CMatrix *m, CVector *v);
+CMatrix *cmatrix_diagonal(CVector *v);
+
+/* SVD functions */
+void Init_svd(void);
+void jacobi_svd(CMatrix *a, CMatrix **u, CMatrix **v, CVector **s);
+
+/* TypedData definitions */
+extern const rb_data_type_t cvector_type;
+extern const rb_data_type_t cmatrix_type;
+
+/* Helper macros */
+#define GET_CVECTOR(obj, ptr) TypedData_Get_Struct(obj, CVector, &cvector_type, ptr)
+#define GET_CMATRIX(obj, ptr) TypedData_Get_Struct(obj, CMatrix, &cmatrix_type, ptr)
+
+/* Matrix element access (row-major) */
+#define MAT_AT(m, i, j) ((m)->data[(i) * (m)->cols + (j)])
+
+#endif /* CLASSIFIER_LINALG_H */