classifier 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fea14969bc8a61283823b0b0f5bae013af968caf4676c383155e3b8682b948de
-  data.tar.gz: 4d626c85d084ff75eba2ff305673734a6f25b668e773b1b5a3a0630a6b68df96
+  metadata.gz: e82726ac5c6e619e701be4591c8ed29e8b9f9a88236ad9d8d602fe3f748dcf43
+  data.tar.gz: 3771f2d2fce4992ed0cd5b1ad3849cc3e1c4fa310a144c1056e5f9d0c49579ef
 SHA512:
-  metadata.gz: ef53c06db3326b1b6ebc14255b4ba198286c06e291cba3afc67bba360ca766a173f89269405d216751806ca72f885a87ac80ec24a031053f8e6f2987e8e2267e
-  data.tar.gz: 8f120a9b78e802e6fd3e7172fd311b476745e27d5b3d301dc8d140296a451875e5aa33a901514bfdd1bc96c656ad1a43cbb3935a05223cd38548a71ba6a3a1c1
+  metadata.gz: 913111c43ffd83a1a461023c6d1f4fef0f1efb7f4e591089db7dbf3d37a1cc98cb3b5de16888019ab1caf8db83ad828dbaf82d2cab237b991216bdb27f529d15
+  data.tar.gz: b7142000fa687a58481051921663665edad1ac3b183947e79cf63bb15a79a0ca7bc66cd76104fccea713162cf05e876bd609d5f4866a13ea2c184f8caa089547

data/CLAUDE.md

@@ -11,21 +11,28 @@ Ruby gem providing text classification via two algorithms:
 ## Common Commands
 
 ```bash
-# Run all tests
-rake test
+# Compile native C extension
+bundle exec rake compile
+
+# Run all tests (compiles first)
+bundle exec rake test
 
 # Run a single test file
 ruby -Ilib test/bayes/bayesian_test.rb
 ruby -Ilib test/lsi/lsi_test.rb
 
-# Run tests with native Ruby vector (without GSL)
-NATIVE_VECTOR=true rake test
+# Run tests with pure Ruby (no native extension)
+NATIVE_VECTOR=true bundle exec rake test
+
+# Run benchmarks
+bundle exec rake benchmark
+bundle exec rake benchmark:compare
 
 # Interactive console
-rake console
+bundle exec rake console
 
 # Generate documentation
-rake doc
+bundle exec rake doc
 ```
 
 ## Architecture
@@ -39,7 +46,7 @@ rake doc
 
 **LSI Classifier** (`lib/classifier/lsi.rb`)
 - Uses Singular Value Decomposition (SVD) for semantic analysis
-- Optional GSL gem for 10x faster matrix operations; falls back to pure Ruby SVD
+- Native C extension for 5-50x faster matrix operations; falls back to pure Ruby
 - Key operations: `add_item`, `classify`, `find_related`, `search`
 - `auto_rebuild` option controls automatic index rebuilding after changes
 
@@ -49,15 +56,18 @@ rake doc
 - Uses `fast-stemmer` gem for Porter stemming
 
 **Vector Extensions** (`lib/classifier/extensions/vector.rb`)
-- Pure Ruby SVD implementation (`Matrix#SV_decomp`)
+- Pure Ruby SVD implementation (`Matrix#SV_decomp`) - used as fallback
 - Vector normalization and magnitude calculations
 
-### GSL Integration
+### Native C Extension (`ext/classifier/`)
+
+LSI uses a native C extension for fast linear algebra operations:
+- `Classifier::Linalg::Vector` - Vector operations (alloc, normalize, dot product)
+- `Classifier::Linalg::Matrix` - Matrix operations (alloc, transpose, multiply)
+- Jacobi SVD implementation for singular value decomposition
 
-LSI checks for the `gsl` gem at load time. When available:
-- Uses `GSL::Matrix` and `GSL::Vector` for faster operations
-- Serialization handled via `vector_serialize.rb`
-- Test without GSL: `NATIVE_VECTOR=true rake test`
+Check current backend: `Classifier::LSI.backend` returns `:native` or `:ruby`
+Force pure Ruby: `NATIVE_VECTOR=true bundle exec rake test`
 
 ### Content Nodes (`lib/classifier/lsi/content_node.rb`)
 

data/README.md

@@ -6,11 +6,14 @@
 
 A Ruby library for text classification using Bayesian and Latent Semantic Indexing (LSI) algorithms.
 
+**[Documentation](https://rubyclassifier.com/docs)** · **[Tutorials](https://rubyclassifier.com/docs/tutorials)** · **[Guides](https://rubyclassifier.com/docs/guides)**
+
 ## Table of Contents
 
 - [Installation](#installation)
 - [Bayesian Classifier](#bayesian-classifier)
 - [LSI (Latent Semantic Indexing)](#lsi-latent-semantic-indexing)
+- [Persistence](#persistence)
 - [Performance](#performance)
 - [Development](#development)
 - [Contributing](#contributing)
@@ -36,47 +39,27 @@ Or install directly:
 gem install classifier
 ```
 
-### Optional: GSL for Faster LSI
-
-For significantly faster LSI operations, install the [GNU Scientific Library](https://www.gnu.org/software/gsl/).
+### Native C Extension
 
-<details>
-<summary><strong>Ruby 3+</strong></summary>
+The gem includes a native C extension for fast LSI operations. It compiles automatically during gem installation. No external dependencies are required.
 
-The released `gsl` gem doesn't support Ruby 3+. Install from source:
+To verify the native extension is active:
 
-```bash
-# Install GSL library
-brew install gsl        # macOS
-apt-get install libgsl-dev  # Ubuntu/Debian
-
-# Build and install the gem
-git clone https://github.com/cardmagic/rb-gsl.git
-cd rb-gsl
-git checkout fix/ruby-3.4-compatibility
-gem build gsl.gemspec
-gem install gsl-*.gem
+```ruby
+require 'classifier'
+puts Classifier::LSI.backend  # => :native
 ```
-</details>
 
-<details>
-<summary><strong>Ruby 2.x</strong></summary>
+To force pure Ruby mode (for debugging):
 
 ```bash
-# macOS
-brew install gsl
-gem install gsl
-
-# Ubuntu/Debian
-apt-get install libgsl-dev
-gem install gsl
+NATIVE_VECTOR=true ruby your_script.rb
 ```
-</details>
 
-When GSL is installed, Classifier automatically uses it. To suppress the GSL notice:
+To suppress the warning when native extension isn't available:
 
 ```bash
-SUPPRESS_GSL_WARNING=true ruby your_script.rb
+SUPPRESS_LSI_WARNING=true ruby your_script.rb
 ```
 
 ### Compatibility
@@ -111,29 +94,10 @@ classifier.classify "Congratulations! You've won a prize!"
 # => "Spam"
 ```
 
-### Persistence with Madeleine
-
-```ruby
-require 'classifier'
-require 'madeleine'
-
-m = SnapshotMadeleine.new("classifier_data") {
-  Classifier::Bayes.new('Interesting', 'Uninteresting')
-}
-
-m.system.train_interesting "fascinating article about science"
-m.system.train_uninteresting "boring repetitive content"
-m.take_snapshot
-
-# Later, restore and use:
-m.system.classify "new scientific discovery"
-# => "Interesting"
-```
-
 ### Learn More
 
-- [Bayesian Filtering Explained](http://www.process.com/precisemail/bayesian_filtering.htm)
-- [Wikipedia: Bayesian Filtering](http://en.wikipedia.org/wiki/Bayesian_filtering)
+- [Bayes Basics Guide](https://rubyclassifier.com/docs/guides/bayes/basics) - In-depth documentation
+- [Build a Spam Filter Tutorial](https://rubyclassifier.com/docs/tutorials/spam-filter) - Step-by-step guide
 - [Paul Graham: A Plan for Spam](http://www.paulgraham.com/spam.html)
 
 ## LSI (Latent Semantic Indexing)
@@ -176,33 +140,83 @@ lsi.search "programming", 3
 
 ### Learn More
 
+- [LSI Basics Guide](https://rubyclassifier.com/docs/guides/lsi/basics) - In-depth documentation
 - [Wikipedia: Latent Semantic Analysis](http://en.wikipedia.org/wiki/Latent_semantic_analysis)
-- [C2 Wiki: Latent Semantic Indexing](http://www.c2.com/cgi/wiki?LatentSemanticIndexing)
+
+## Persistence
+
+Save and load trained classifiers with pluggable storage backends. Works with both Bayes and LSI classifiers.
+
+### File Storage
+
+```ruby
+require 'classifier'
+
+classifier = Classifier::Bayes.new('Spam', 'Ham')
+classifier.train_spam "Buy now! Limited offer!"
+classifier.train_ham "Meeting tomorrow at 3pm"
+
+# Configure storage and save
+classifier.storage = Classifier::Storage::File.new(path: "spam_filter.json")
+classifier.save
+
+# Load later
+loaded = Classifier::Bayes.load(storage: classifier.storage)
+loaded.classify "Claim your prize now!"
+# => "Spam"
+```
+
+### Custom Storage Backends
+
+Create backends for Redis, PostgreSQL, S3, or any storage system:
+
+```ruby
+class RedisStorage < Classifier::Storage::Base
+  def initialize(redis:, key:)
+    super()
+    @redis, @key = redis, key
+  end
+
+  def write(data) = @redis.set(@key, data)
+  def read = @redis.get(@key)
+  def delete = @redis.del(@key)
+  def exists? = @redis.exists?(@key)
+end
+
+# Use it
+classifier.storage = RedisStorage.new(redis: Redis.new, key: "classifier:spam")
+classifier.save
+```
+
+### Learn More
+
+- [Persistence Guide](https://rubyclassifier.com/docs/guides/persistence/basics) - Full documentation with examples
 
 ## Performance
 
-### GSL vs Native Ruby
+### Native C Extension vs Pure Ruby
 
-GSL provides dramatic speedups for LSI operations, especially `build_index` (SVD computation):
+The native C extension provides dramatic speedups for LSI operations, especially `build_index` (SVD computation):
 
 | Documents | build_index | Overall |
 |-----------|-------------|---------|
-| 5         | 4x faster   | 2.5x    |
-| 10        | 24x faster  | 5.5x    |
-| 15        | 116x faster | 17x     |
+| 5         | 7x faster   | 2.6x    |
+| 10        | 25x faster  | 4.6x    |
+| 15        | 112x faster | 14.5x   |
+| 20        | 385x faster | 48.7x   |
 
 <details>
-<summary>Detailed benchmark (15 documents)</summary>
+<summary>Detailed benchmark (20 documents)</summary>
 
 ```
-Operation              Native          GSL      Speedup
+Operation            Pure Ruby     Native C      Speedup
 ----------------------------------------------------------
-build_index            0.1412       0.0012       116.2x
-classify               0.0142       0.0049         2.9x
-search                 0.0102       0.0026         3.9x
-find_related           0.0069       0.0016         4.2x
+build_index            0.5540       0.0014       384.5x
+classify               0.0190       0.0060         3.2x
+search                 0.0145       0.0037         3.9x
+find_related           0.0098       0.0011         8.6x
 ----------------------------------------------------------
-TOTAL                  0.1725       0.0104        16.6x
+TOTAL                  0.5973       0.0123        48.7x
 ```
 </details>
 
@@ -210,7 +224,7 @@ TOTAL                  0.1725       0.0104        16.6x
 
 ```bash
 rake benchmark              # Run with current configuration
-rake benchmark:compare      # Compare GSL vs native Ruby
+rake benchmark:compare      # Compare native C vs pure Ruby
 ```
 
 ## Development
@@ -221,15 +235,16 @@ rake benchmark:compare      # Compare GSL vs native Ruby
 git clone https://github.com/cardmagic/classifier.git
 cd classifier
 bundle install
+rake compile  # Compile native C extension
 ```
 
 ### Running Tests
 
 ```bash
-rake test                        # Run all tests
+rake test                        # Run all tests (compiles first)
 ruby -Ilib test/bayes/bayesian_test.rb  # Run specific test file
 
-# Test without GSL (pure Ruby)
+# Test with pure Ruby (no native extension)
 NATIVE_VECTOR=true rake test
 ```
 

data/ext/classifier/classifier_ext.c

@@ -0,0 +1,25 @@
+/*
+ * classifier_ext.c
+ * Main entry point for the Classifier native linear algebra extension
+ *
+ * This extension provides zero-dependency Vector, Matrix, and SVD
+ * implementations for the Classifier gem's LSI functionality.
+ */
+
+#include "linalg.h"
+
+VALUE mClassifierLinalg;
+VALUE cClassifierVector;
+VALUE cClassifierMatrix;
+
+void Init_classifier_ext(void)
+{
+    /* Define Classifier::Linalg module */
+    VALUE mClassifier = rb_define_module("Classifier");
+    mClassifierLinalg = rb_define_module_under(mClassifier, "Linalg");
+
+    /* Initialize Vector and Matrix classes */
+    Init_vector();
+    Init_matrix();
+    Init_svd();
+}

data/ext/classifier/extconf.rb

@@ -0,0 +1,15 @@
+require 'mkmf'
+
+# rubocop:disable Style/GlobalVars
+if ENV['COVERAGE']
+  # Coverage flags: disable optimization for accurate line coverage
+  $CFLAGS << ' -O0 -g --coverage -Wall'
+  $LDFLAGS << ' --coverage'
+else
+  # Optimization flags for performance
+  $CFLAGS << ' -O3 -ffast-math -Wall'
+end
+# rubocop:enable Style/GlobalVars
+
+# Create the Makefile
+create_makefile('classifier/classifier_ext')

data/ext/classifier/linalg.h

@@ -0,0 +1,64 @@
+#ifndef CLASSIFIER_LINALG_H
+#define CLASSIFIER_LINALG_H
+
+#include <ruby.h>
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Epsilon for numerical comparisons */
+#define CLASSIFIER_EPSILON 1e-10
+
+/* Vector structure */
+typedef struct {
+    size_t size;
+    double *data;
+    int is_col;  /* 0 = row vector, 1 = column vector */
+} CVector;
+
+/* Matrix structure */
+typedef struct {
+    size_t rows;
+    size_t cols;
+    double *data;  /* Row-major storage */
+} CMatrix;
+
+/* Ruby class references */
+extern VALUE cClassifierVector;
+extern VALUE cClassifierMatrix;
+extern VALUE mClassifierLinalg;
+
+/* Vector functions */
+void Init_vector(void);
+CVector *cvector_alloc(size_t size);
+void cvector_free(void *ptr);
+double cvector_magnitude(CVector *v);
+CVector *cvector_normalize(CVector *v);
+double cvector_sum(CVector *v);
+double cvector_dot(CVector *a, CVector *b);
+
+/* Matrix functions */
+void Init_matrix(void);
+CMatrix *cmatrix_alloc(size_t rows, size_t cols);
+void cmatrix_free(void *ptr);
+CMatrix *cmatrix_transpose(CMatrix *m);
+CMatrix *cmatrix_multiply(CMatrix *a, CMatrix *b);
+CVector *cmatrix_multiply_vector(CMatrix *m, CVector *v);
+CMatrix *cmatrix_diagonal(CVector *v);
+
+/* SVD functions */
+void Init_svd(void);
+void jacobi_svd(CMatrix *a, CMatrix **u, CMatrix **v, CVector **s);
+
+/* TypedData definitions */
+extern const rb_data_type_t cvector_type;
+extern const rb_data_type_t cmatrix_type;
+
+/* Helper macros */
+#define GET_CVECTOR(obj, ptr) TypedData_Get_Struct(obj, CVector, &cvector_type, ptr)
+#define GET_CMATRIX(obj, ptr) TypedData_Get_Struct(obj, CMatrix, &cmatrix_type, ptr)
+
+/* Matrix element access (row-major) */
+#define MAT_AT(m, i, j) ((m)->data[(i) * (m)->cols + (j)])
+
+#endif /* CLASSIFIER_LINALG_H */