topical 0.0.1.pre.1 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0559da99a2324dc6a4716e075d1222dfb9049f188e224f0976349aa6cdce5002'
-  data.tar.gz: 03b340c866055159801e476d61866b0b13856966f34b0c0904a49a299fce7ba7
+  metadata.gz: 470b52a96cb237a1c1e159dc5987d58c07f38ee91c721f39ec36f2f607c7cae2
+  data.tar.gz: 7a4e2c14668795facc41b743146ad6f020830aa65290ac9377834c062fc2819d
 SHA512:
-  metadata.gz: 535427f8e78814474292ad924dfd367bebc0bf3b796e010e407e8432d1aade54db55599b613255002db0eebfacf6939785699db344cd7d4d19be8eaeb744e804
-  data.tar.gz: 4390654a5a9c7375e10b0174e2a67b4a2e5df7a69176f0761e9a4c4005cdd5b53736d2639a74a1943dc408b2677e2cdf57e1c51a0fcd1886bce8a3db1c83fa27
+  metadata.gz: 680f27ae0aef8c867e2c6232b5aaddf3ddb3564d8a6e32d0e03bec6796002e33e899d8bd81277fc293c1ce6bfa4208e9739feff7e410a99c5e1560f7d2aa1219
+  data.tar.gz: 045bc5d1bf65f0ef8d583ee78abb4af33dc822de61639c26f71e6da5691e31b28f53e733c53fffc61bd9c56b0fbf02acc5e45100ff78c27299eace93a680c74e

data/README.md

@@ -1,104 +1,77 @@
-# Topical
+<img src="/docs/assets/topical-wide.png" alt="ragnar" height="80px">
 
 Topic modeling for Ruby using modern clustering algorithms. Extract meaningful topics from document embeddings using HDBSCAN clustering and c-TF-IDF term extraction.
 
-## Quick Start
+## Quick Start (requires red-candle)
 
 ```bash
 # Install the gem
 gem install topical
 
+# Install red-candle so we can generate embeddings
+gem install red-candle
+
 # Try it out immediately in IRB
 irb
 ```
 
-```ruby
-require 'topical'
-
-# Create some sample documents
-documents = [
-  "Ruby is a dynamic programming language with elegant syntax",
-  "Rails is a web framework written in Ruby for building web applications",
-  "Python is great for machine learning and data science applications",
-  "TensorFlow and PyTorch are popular machine learning frameworks in Python",
-  "JavaScript runs in browsers and Node.js for full-stack development",
-  "React and Vue are modern JavaScript frameworks for building UIs",
-  "Machine learning models need training data and validation sets",
-  "Deep learning uses neural networks with multiple layers",
-  "Web development involves HTML, CSS, and JavaScript",
-  "Backend development often uses databases and APIs"
-]
-
-# Create simple mock embeddings (in practice, use real embeddings from red-candle or other embedding models)
-# Here we create 3 distinct clusters based on keywords
-embeddings = documents.map do |doc|
-  text = doc.downcase
-  [
-    text.include?("ruby") || text.include?("rails") ? 1.0 : 0.0,  # Ruby cluster
-    text.include?("python") || text.include?("machine") || text.include?("learning") ? 1.0 : 0.0,  # ML cluster  
-    text.include?("javascript") || text.include?("web") || text.include?("css") ? 1.0 : 0.0,  # Web cluster
-    rand(-0.1..0.1)  # Small random noise
-  ]
-end
-
-# Extract topics
-topics = Topical.extract(
-  embeddings: embeddings,
-  documents: documents,
-  clustering_method: :kmeans,
-  k: 3
-)
-
-# Display results
-topics.each do |topic|
-  puts "\n📌 #{topic.label}"
-  puts "   Documents: #{topic.size}"
-  puts "   Key terms: #{topic.terms.first(5).join(', ')}"
-  puts "   Sample: \"#{topic.documents.first[0..80]}...\""
-end
-```
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'topical'
-
-# Optional but recommended: for generating real embeddings
-gem 'red-candle'  
-```
-
-And then execute:
-
-    $ bundle install
-
-Or install it yourself as:
-
-    $ gem install topical
-
-## Real-World Usage with Embeddings
-
-### Using with red-candle (recommended)
-
 ```ruby
 require 'topical'
 require 'red-candle'
 
 # Initialize embedding model
-embedder = RedCandle::Embedding.new("sentence-transformers/all-MiniLM-L6-v2")
+embedder = Candle::EmbeddingModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
 
 # Your documents
 documents = [
-  "The Federal Reserve raised interest rates to combat inflation",
-  "Stock markets rallied on positive earnings reports",
-  "New AI breakthrough in natural language processing",
-  "Machine learning transforms healthcare diagnostics",
-  # ... more documents
+  # Finance/Economics Topic
+  "The Federal Reserve raised interest rates to combat inflation pressures",
+  "Stock markets rallied on positive earnings reports from tech companies",
+  "Global supply chain disruptions continue to affect consumer prices",
+  "Cryptocurrency markets experienced significant volatility this quarter",
+  "Central banks coordinate policy to address economic uncertainty",
+  "Corporate bond yields rise as investors seek safer assets",
+  "Emerging markets face capital outflows amid dollar strength",
+
+  # Technology/AI Topic
+  "New AI breakthrough in natural language processing announced by researchers",
+  "Machine learning transforms healthcare diagnostics and treatment planning",
+  "Quantum computing reaches new milestone in error correction",
+  "Open source community releases major updates to popular frameworks",
+  "Cloud computing adoption accelerates across enterprise sectors",
+  "Cybersecurity threats evolve with sophisticated ransomware attacks",
+  "Artificial intelligence ethics guidelines proposed by tech consortium",
+
+  # Healthcare/Medical Topic
+  "Clinical trials show promising results for new cancer immunotherapy",
+  "Telemedicine adoption continues to reshape patient care delivery",
+  "Gene editing techniques advance treatment for rare diseases",
+  "Mental health awareness campaigns gain momentum globally",
+  "Vaccine development accelerates using mRNA technology platforms",
+  "Healthcare systems invest in digital transformation initiatives",
+  "Personalized medicine approaches show improved patient outcomes",
+
+  # Climate/Environment Topic
+  "Renewable energy investments surpass fossil fuel spending globally",
+  "Climate scientists warn of accelerating Arctic ice melt",
+  "Carbon capture technology receives significant government funding",
+  "Sustainable agriculture practices reduce environmental impact",
+  "Electric vehicle adoption reaches record levels worldwide",
+  "Ocean conservation efforts expand marine protected areas",
+  "Green hydrogen emerges as key solution for industrial decarbonization",
+
+  # Sports Topic
+  "Championship team breaks decades-old winning streak record",
+  "Olympic athletes prepare for upcoming international competition",
+  "Sports analytics revolutionize player performance evaluation",
+  "Major league implements new rules to improve game pace",
+  "Youth sports participation increases following pandemic recovery",
+  "Stadium technology enhances fan experience with augmented reality",
+  "Professional athletes advocate for mental health support"
 ]
 
 # Generate embeddings
-embeddings = documents.map { |doc| embedder.embed(doc) }
+embeddings = documents.map { |doc| embedder.embedding(doc).first.to_a }
 
 # Extract topics with HDBSCAN clustering
 engine = Topical::Engine.new(
@@ -107,7 +80,7 @@ engine = Topical::Engine.new(
   verbose: true
 )
 
-topics = engine.fit(embeddings, documents)
+topics = engine.fit(embeddings: embeddings, documents: documents)
 
 # Analyze results
 topics.each do |topic|
@@ -131,36 +104,123 @@ puts "\nOutliers: #{outliers.length} documents"
 engine = Topical::Engine.new(
   # Clustering options
   clustering_method: :hdbscan,    # :hdbscan or :kmeans
-  min_cluster_size: 10,            # Minimum documents per topic (HDBSCAN)
+  min_cluster_size: 3,            # Minimum documents per topic (HDBSCAN)
   min_samples: 5,                  # Core points needed (HDBSCAN)
   k: 20,                           # Number of topics (K-means only)
-  
+
   # Dimensionality reduction
   reduce_dimensions: true,          # Auto-reduce high-dim embeddings with UMAP
   n_components: 50,                 # Target dimensions for reduction
-  
+
   # Labeling options
-  labeling_method: :hybrid,         # :term_based, :llm_based, or :hybrid
-  llm_provider: nil,               # Optional: custom LLM provider
-  
+  labeling_method: :term_based,     # Fast, reliable term-based labeling
+
   # Other options
   verbose: true                     # Show progress
 )
 
 # Fit the model
-topics = engine.fit(embeddings, documents, metadata: metadata)
+topics = engine.fit(embeddings: embeddings, documents: documents)
 
 # Save and load models
 engine.save("topic_model.json")
 loaded = Topical::Engine.load("topic_model.json")
 
-# Transform new documents
-new_topics = engine.transform(new_embeddings)
+# Transform: Assign new documents to existing topics
+# Note: transform does NOT create new topics - it assigns documents to the closest existing topic
+new_documents = [
+  # These will be assigned to existing topics based on similarity
+  "Stock market reaches all-time high amid economic recovery",  # Should go to Finance
+  "New smartphone features AI-powered camera system",           # Should go to Technology
+  "Clinical study reveals breakthrough in diabetes treatment",  # Should go to Healthcare
+  "Record heat wave highlights climate change urgency"          # Should go to Climate
+]
+new_embeddings = new_documents.map { |doc| embedder.embedding(doc).first.to_a }
+
+# Returns array of topic IDs that each document was assigned to
+assigned_topic_ids = engine.transform(embeddings: new_embeddings, documents: new_documents)
+
+# See which topics the new documents were assigned to
+assigned_topic_ids.each_with_index do |topic_id, idx|
+  topic = engine.get_topic(topic_id)
+  if topic
+    puts "Document #{idx}: Assigned to Topic '#{topic.label}'"
+    puts "  Document: #{new_documents[idx]}"
+  else
+    puts "Document #{idx}: Marked as outlier (no matching topic)"
+  end
+end
 
 # Get specific topic
 topic = engine.get_topic(0)
 ```
 
+### Understanding Transform vs Fit
+
+- **`fit`**: Discovers topics from your training documents. Creates new topic clusters.
+- **`transform`**: Assigns new documents to existing topics discovered during `fit`. Does NOT create new topics.
+
+If you have documents that represent a completely new topic not seen during training:
+1. They may be assigned to the closest existing topic (even if not very similar)
+2. They may be marked as outliers if using HDBSCAN (returned as topic_id -1)
+3. To discover new topics, you need to analyze them separately or re-fit
+
+### Detecting New Topics
+
+Yes, you can run `fit` on just the new documents to discover their topics independently! This is useful for:
+- Detecting topic drift over time
+- Identifying emerging themes
+- Validating if new content fits your existing model
+
+See [examples/detect_new_topics.rb](examples/detect_new_topics.rb) for a complete example.
+
+```ruby
+# To discover new topics, you have several options:
+
+# Option 1: Fit only on new documents to discover their topics
+new_engine = Topical::Engine.new(
+  clustering_method: :hdbscan,
+  min_cluster_size: 3  # May need to adjust for small batches
+)
+new_topics = new_engine.fit(embeddings: new_embeddings, documents: new_documents)
+puts "Found #{new_topics.size} topics in new documents"
+
+# Option 2: Check if new documents are outliers (potential new topic)
+assigned_ids = engine.transform(embeddings: new_embeddings)
+outlier_indices = assigned_ids.each_index.select { |i| assigned_ids[i] == -1 }
+if outlier_indices.size > 3  # If many outliers, might be new topic
+  puts "#{outlier_indices.size} documents don't fit existing topics - potential new topic!"
+  outlier_docs = outlier_indices.map { |i| new_documents[i] }
+  outlier_embeds = outlier_indices.map { |i| new_embeddings[i] }
+
+  # Cluster just the outliers
+  outlier_engine = Topical::Engine.new(min_cluster_size: 3)
+  outlier_topics = outlier_engine.fit(embeddings: outlier_embeds, documents: outlier_docs)
+end
+
+# Option 3: Incremental topic discovery (combine old + new and re-fit)
+all_documents = original_documents + new_documents
+all_embeddings = original_embeddings + new_embeddings
+updated_topics = engine.fit(embeddings: all_embeddings, documents: all_documents)
+
+# Option 4: Compare similarity scores to detect poor fits
+assigned_ids = engine.transform(embeddings: new_embeddings)
+similarities = new_embeddings.map.with_index do |embed, idx|
+  topic_id = assigned_ids[idx]
+  next nil if topic_id == -1
+
+  topic = engine.get_topic(topic_id)
+  # Calculate distance to topic centroid (simplified)
+  # In practice, you'd compute actual distance to topic center
+  { document: new_documents[idx], topic: topic.label, similarity: rand(0.3..1.0) }
+end
+
+low_similarity = similarities.compact.select { |s| s[:similarity] < 0.5 }
+if low_similarity.size > 3
+  puts "#{low_similarity.size} documents have low similarity - might be new topic"
+end
+```
+
 ### Topic Analysis
 
 ```ruby
@@ -181,7 +241,7 @@ topic.to_h
 
 # Compute metrics across all topics
 diversity = Topical::Metrics.compute_diversity(topics)
-coverage = Topical::Metrics.compute_coverage(topics, total_docs)
+coverage = Topical::Metrics.compute_coverage(topics, documents.count + new_documents.count)
 ```
 
 ## Clustering Methods
@@ -203,24 +263,18 @@ Topical uses **c-TF-IDF** (class-based TF-IDF) to find distinctive terms for eac
 - Automatically filters stop words
 - Configurable minimum/maximum word lengths
 
-## Topic Labeling Methods
+## Topic Labeling
+
+Topical uses **term-based labeling** - fast, reliable labels generated from the most distinctive terms in each topic cluster. Labels are created by combining the top 2-3 terms that best characterize each topic.
+
+### Advanced: LLM-Powered Summaries
 
-1. **Term-based** (`:term_based`)
-   - Fast, uses top distinctive terms
-   - No external dependencies
-   
-2. **LLM-based** (`:llm_based`)
-   - High quality, contextual labels
-   - Requires red-candle or API provider
-   
-3. **Hybrid** (`:hybrid`)
-   - Best of both: fast with LLM enhancement
-   - Falls back to term-based if LLM unavailable
+For richer topic analysis, you can combine Topical's clustering with red-candle's LLM capabilities. See `examples/topic_summaries_with_llm.rb` for a complete example of generating detailed topic summaries using your choice of LLM.
 
 ## Dependencies
 
 - **Required**: `clusterkit` - For HDBSCAN clustering and UMAP dimensionality reduction
-- **Optional**: `red-candle` - For generating embeddings and LLM-powered topic labeling
+- **Optional**: `red-candle` - For generating embeddings in examples and advanced LLM summaries
 
 ## Performance Tips
 
@@ -233,9 +287,7 @@ Topical uses **c-TF-IDF** (class-based TF-IDF) to find distinctive terms for eac
 
 Check out the `examples/` directory for complete examples:
 - `quick_demo.rb` - Simple demonstration with mock data
-- `news_clustering.rb` - Clustering news articles
-- `customer_feedback.rb` - Analyzing customer feedback topics
-- `research_papers.rb` - Organizing research papers by topic
+- `topic_summaries_with_llm.rb` - Advanced example showing how to generate detailed topic summaries using red-candle LLM
 
 ## Development
 
@@ -245,8 +297,8 @@ To install this gem onto your local machine, run `bundle exec rake install`.
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/cpetersen/topical.
+Bug reports and pull requests are welcome on GitHub at https://github.com/scientist-labs/topical.
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/examples/detect_new_topics.rb

@@ -0,0 +1,190 @@
+#!/usr/bin/env ruby
+# Example: Detecting emergence of new topics in document streams
+
+require 'topical'
+require 'candle'
+require 'json'
+
+puts "Loading embedding model..."
+embedder = Candle::EmbeddingModel.from_pretrained("jinaai/jina-embeddings-v2-base-en")  # Default ragnar model
+puts "Model loaded!"
+
+# Initial documents (3 clear topics)
+initial_documents = [
+  # Finance
+  "Stock market reaches record highs amid economic recovery",
+  "Federal Reserve considers interest rate adjustments",
+  "Cryptocurrency adoption grows among institutional investors",
+  "Banking sector reports strong quarterly earnings",
+  "Global trade agreements impact currency markets",
+  
+  # Technology
+  "Artificial intelligence breakthrough in natural language processing",
+  "Cloud computing services expand globally",
+  "Quantum computing achieves new milestone",
+  "Cybersecurity threats evolve with new techniques",
+  "Open source community releases major updates",
+  
+  # Healthcare
+  "Clinical trials show promise for new cancer treatment",
+  "Telemedicine adoption continues post-pandemic growth",
+  "Gene therapy advances for rare diseases",
+  "Mental health awareness campaigns expand",
+  "Vaccine development using mRNA technology"
+]
+
+puts "=" * 60
+puts "INITIAL TOPIC MODELING"
+puts "=" * 60
+
+# Train initial model
+puts "\nGenerating embeddings for #{initial_documents.size} documents..."
+initial_embeddings = initial_documents.map.with_index do |doc, i|
+  print "." if i % 5 == 0
+  embedder.embedding(doc).first.to_a
+end
+puts " done!"
+
+puts "Creating topic model..."
+engine = Topical::Engine.new(
+  clustering_method: :kmeans,  # Use k-means for small dataset
+  k: 3,  # We know we have 3 distinct topics
+  reduce_dimensions: false,  # Don't reduce dimensions for small dataset
+  verbose: true     # Show progress
+)
+
+puts "Fitting model..."
+initial_topics = engine.fit(embeddings: initial_embeddings, documents: initial_documents)
+puts "\nFound #{initial_topics.size} initial topics:"
+initial_topics.each do |topic|
+  puts "  Topic #{topic.id}: #{topic.terms.take(5).join(', ')}"
+  puts "    Size: #{topic.size} documents"
+end
+
+# New documents arrive - including a new topic (Education)
+new_documents = [
+  # More finance
+  "Market volatility increases ahead of earnings season",
+  
+  # More tech
+  "Machine learning models improve prediction accuracy",
+  
+  # NEW TOPIC: Education (not in original training)
+  "Online learning platforms transform education delivery",
+  "Universities adopt hybrid teaching models globally",
+  "STEM education initiatives target underserved communities",
+  "Educational technology startups receive record funding",
+  "Student debt crisis prompts policy discussions",
+  "Coding bootcamps address skills gap in workforce"
+]
+
+puts "\nGenerating embeddings for new documents..."
+new_embeddings = new_documents.map.with_index do |doc, i|
+  print "."
+  embedder.embedding(doc).first.to_a
+end
+puts " done!"
+
+puts "\n" + "=" * 60
+puts "DETECTING NEW TOPICS IN INCOMING DOCUMENTS"
+puts "=" * 60
+
+# Method 1: Try to assign to existing topics
+puts "\n1. Checking fit with existing topics..."
+assigned_ids = engine.transform(embeddings: new_embeddings)
+
+assigned_ids.each_with_index do |topic_id, idx|
+  doc_preview = new_documents[idx][0..50] + "..."
+  if topic_id == -1
+    puts "  ❌ Outlier: #{doc_preview}"
+  else
+    topic = engine.get_topic(topic_id)
+    puts "  ✓ Topic #{topic_id}: #{doc_preview}"
+  end
+end
+
+outlier_count = assigned_ids.count(-1)
+puts "\nFound #{outlier_count} outliers (documents that don't fit existing topics)"
+
+# Method 2: Cluster new documents independently
+puts "\n2. Clustering new documents independently..."
+if new_documents.size >= 5  # Need minimum documents for clustering
+  new_engine = Topical::Engine.new(
+    clustering_method: :kmeans,  # Use kmeans for small datasets
+    k: 2,  # Expect ~2 topics in new docs
+    verbose: false
+  )
+
+  begin
+    new_topics_only = new_engine.fit(embeddings: new_embeddings, documents: new_documents)
+    puts "Found #{new_topics_only.size} topics in new documents:"
+    
+    new_topics_only.each do |topic|
+      puts "  New Topic: #{topic.terms.take(5).join(', ')}"
+      puts "    Documents: #{topic.size}"
+      puts "    Sample: #{topic.documents.first[0..60]}..." if topic.documents.any?
+    end
+  rescue => e
+    puts "Could not cluster new documents alone: #{e.message}"
+  end
+else
+  puts "Too few new documents (#{new_documents.size}) for independent clustering"
+end
+
+# Method 3: Identify potential new topic from outliers
+if outlier_count >= 3
+  puts "\n3. Analyzing outliers for potential new topic..."
+  outlier_indices = assigned_ids.each_index.select { |i| assigned_ids[i] == -1 }
+  outlier_docs = outlier_indices.map { |i| new_documents[i] }
+  outlier_embeds = outlier_indices.map { |i| new_embeddings[i] }
+  
+  # Check if outliers are similar to each other (potential new topic)
+  puts "Outlier documents:"
+  outlier_docs.each { |doc| puts "  - #{doc[0..60]}..." }
+  
+  # Try clustering just the outliers
+  if outlier_docs.size >= 3
+    # Check similarity among outliers to see if they form coherent group
+    puts "\n🎯 Multiple outliers detected - potential new topic!"
+    puts "Consider re-training model to discover new topics"
+  end
+end
+
+# Method 4: Re-fit with all documents to discover new structure
+puts "\n4. Re-fitting with all documents..."
+all_docs = initial_documents + new_documents
+all_embeds = initial_embeddings + new_embeddings
+
+updated_engine = Topical::Engine.new(
+  clustering_method: :kmeans,
+  k: 4,  # Try 4 topics since we might have a new one
+  reduce_dimensions: false,  # Don't reduce dimensions for small dataset
+  verbose: false
+)
+
+updated_topics = updated_engine.fit(embeddings: all_embeds, documents: all_docs)
+puts "After re-fitting with all documents: #{updated_topics.size} topics"
+
+if updated_topics.size > initial_topics.size
+  puts "✨ NEW TOPICS EMERGED! (#{updated_topics.size - initial_topics.size} new)"
+  updated_topics.each do |topic|
+    puts "  Topic: #{topic.terms.take(5).join(', ')} (#{topic.size} docs)"
+  end
+else
+  puts "No new topics detected after re-fitting"
+end
+
+puts "\n" + "=" * 60
+puts "SUMMARY"
+puts "=" * 60
+puts "Initial topics: #{initial_topics.size}"
+puts "Outliers in new batch: #{outlier_count}/#{new_documents.size}"
+puts "Topics after re-fit: #{updated_topics.size}"
+puts "\nRecommendation:"
+if outlier_count > new_documents.size * 0.3
+  puts "⚠️  High outlier rate suggests emerging new topic(s)"
+  puts "Consider re-training your topic model with recent documents"
+else
+  puts "✓ New documents mostly fit existing topics"
+  puts "Current model appears adequate"
+end

data/examples/quick_demo.rb

@@ -115,4 +115,4 @@ puts "🎉 Demo complete! Try it with your own documents and real embeddings."
 puts
 puts "💡 Tip: Install red-candle to generate real embeddings:"
 puts "   gem install red-candle"
-puts "   Then use: RedCandle::Embedding.new('model-name').embed(text)"
+puts "   Then use: RedCandle::Embedding.new('model-name').embed(text)"