topical 0.0.1.pre.1 → 0.1.1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: topical
 version: !ruby/object:Gem::Version
-  version: 0.0.1.pre.1
+  version: 0.1.1
 platform: ruby
 authors:
 - Chris Petersen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-08-30 00:00:00.000000000 Z
+date: 2025-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: clusterkit
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: red-candle
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
 description: Extract topics from document embeddings using HDBSCAN clustering and
   c-TF-IDF term extraction. Provides automatic topic labeling, quality metrics, and
   support for various clustering algorithms.
@@ -95,30 +109,32 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- docs/assets/topical-wide.png
+- examples/detect_new_topics.rb
 - examples/quick_demo.rb
+- examples/topic_summaries_with_llm.rb
 - examples/verify_migration.rb
 - lib/topical.rb
 - lib/topical/clustering/adapter.rb
 - lib/topical/clustering/hdbscan_adapter.rb
 - lib/topical/clustering/kmeans_adapter.rb
+- lib/topical/dimensionality_reducer.rb
 - lib/topical/engine.rb
 - lib/topical/extractors/term_extractor.rb
 - lib/topical/labelers/base.rb
-- lib/topical/labelers/hybrid.rb
-- lib/topical/labelers/llm_adapter.rb
-- lib/topical/labelers/llm_based.rb
 - lib/topical/labelers/term_based.rb
 - lib/topical/metrics.rb
+- lib/topical/model_serializer.rb
 - lib/topical/topic.rb
 - lib/topical/version.rb
 - sig/topical.rbs
-homepage: https://github.com/cpetersen/topical
+homepage: https://github.com/scientist-labs/topical
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/cpetersen/topical
-  source_code_uri: https://github.com/cpetersen/topical
-  changelog_uri: https://github.com/cpetersen/topical/blob/main/CHANGELOG.md
+  homepage_uri: https://github.com/scientist-labs/topical
+  source_code_uri: https://github.com/scientist-labs/topical
+  changelog_uri: https://github.com/scientist-labs/topical/blob/main/CHANGELOG.md
   documentation_uri: https://rubydoc.info/gems/topical
 post_install_message:
 rdoc_options: []

data/lib/topical/labelers/hybrid.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Topical
-  module Labelers
-    # Hybrid labeling that combines term-based and LLM approaches
-    class Hybrid < Base
-      def initialize(provider: nil)
-        @term_labeler = TermBased.new
-        @llm_labeler = LLMBased.new(provider: provider)
-      end
-      
-      def generate_label(topic)
-        # Start with term-based label
-        term_label = @term_labeler.generate_label(topic)
-        
-        # Try to enhance with LLM if available
-        llm_label = @llm_labeler.generate_label(topic)
-        
-        # For now, just return the LLM label if different, otherwise term label
-        llm_label != "LLM Topic #{topic.id}" ? llm_label : term_label
-      end
-    end
-  end
-end

data/lib/topical/labelers/llm_adapter.rb

@@ -1,126 +0,0 @@
-# frozen_string_literal: true
-
-module Topical
-  module Labelers
-    # Adapter to allow different LLM backends (red-candle, remote APIs, etc.)
-    class LLMAdapter
-      # Factory method to create appropriate LLM client
-      def self.create(type: :auto, **options)
-        case type
-        when :red_candle
-          RedCandleAdapter.new(**options)
-        when :openai
-          # Future: OpenAIAdapter.new(**options)
-          raise NotImplementedError, "OpenAI adapter not yet implemented"
-        when :anthropic
-          # Future: AnthropicAdapter.new(**options)
-          raise NotImplementedError, "Anthropic adapter not yet implemented"
-        when :auto
-          # Try red-candle first, then fall back to others
-          begin
-            RedCandleAdapter.new(**options)
-          rescue LoadError
-            nil  # No LLM available
-          end
-        else
-          raise ArgumentError, "Unknown LLM type: #{type}"
-        end
-      end
-    end
-    
-    # Adapter for red-candle (local LLMs)
-    class RedCandleAdapter
-      def initialize(model: nil, **options)
-        require 'red-candle'
-        
-        @model = model || default_model
-        @options = options
-        @llm = load_or_create_llm
-      end
-      
-      def generate(prompt:, max_tokens: 100, temperature: 0.3, response_format: nil)
-        # Red-candle specific generation
-        response = @llm.generate(
-          prompt,
-          max_length: max_tokens,
-          temperature: temperature,
-          do_sample: temperature > 0
-        )
-        
-        # Handle JSON response format if requested
-        if response_format && response_format[:type] == "json_object"
-          ensure_json_response(response)
-        else
-          response
-        end
-      end
-      
-      def available?
-        true
-      end
-      
-      private
-      
-      def default_model
-        # Use a small, fast model by default for topic labeling
-        "TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"
-      end
-      
-      def load_or_create_llm
-        # Create new LLM instance with red-candle
-        RedCandle::Model.new(
-          model_id: @model,
-          model_type: :llama,
-          quantized: true
-        )
-      end
-      
-      def ensure_json_response(response)
-        # Try to extract JSON from response
-        begin
-          require 'json'
-          # Look for JSON-like content
-          json_match = response.match(/\{.*\}/m)
-          if json_match
-            JSON.parse(json_match[0])
-            json_match[0]  # Return the JSON string if valid
-          else
-            # Generate a basic JSON response
-            generate_fallback_json(response)
-          end
-        rescue JSON::ParserError
-          generate_fallback_json(response)
-        end
-      end
-      
-      def generate_fallback_json(text)
-        # Create a simple JSON from text response
-        require 'json'
-        label = text.lines.first&.strip || "Unknown"
-        {
-          label: label,
-          description: text,
-          confidence: 0.5
-        }.to_json
-      end
-    end
-    
-    # Future adapter for remote LLMs
-    class RemoteAdapter
-      def initialize(api_key:, endpoint:, **options)
-        @api_key = api_key
-        @endpoint = endpoint
-        @options = options
-      end
-      
-      def generate(prompt:, max_tokens: 100, temperature: 0.3, response_format: nil)
-        # Make API call
-        raise NotImplementedError, "Remote LLM adapter coming soon"
-      end
-      
-      def available?
-        !@api_key.nil?
-      end
-    end
-  end
-end

data/lib/topical/labelers/llm_based.rb

@@ -1,111 +0,0 @@
-# frozen_string_literal: true
-
-module Topical
-  module Labelers
-    # LLM-powered topic labeling (requires red-candle or other LLM provider)
-    class LLMBased < Base
-      def initialize(provider: nil)
-        @provider = provider
-      end
-      
-      def generate_label(topic)
-        unless llm_available?
-          # Fallback to term-based if LLM not available
-          return TermBased.new.generate_label(topic)
-        end
-        
-        # Select best documents to send to LLM
-        sample_docs = topic.representative_docs(k: 3)
-        
-        # Generate comprehensive analysis
-        response = analyze_with_llm(sample_docs, topic.terms)
-        
-        response[:label]
-      rescue => e
-        # Fallback on error
-        puts "LLM labeling failed: #{e.message}" if ENV['DEBUG']
-        TermBased.new.generate_label(topic)
-      end
-      
-      private
-      
-      def llm_available?
-        return true if @provider
-        
-        # Try to create LLM adapter
-        begin
-          require_relative 'llm_adapter'
-          @provider = LLMAdapter.create(type: :auto)
-          @provider && @provider.available?
-        rescue LoadError, StandardError => e
-          puts "LLM not available: #{e.message}" if ENV['DEBUG']
-          false
-        end
-      end
-      
-      def analyze_with_llm(documents, terms)
-        prompt = build_analysis_prompt(documents, terms)
-        
-        response = @provider.generate(
-          prompt: prompt,
-          max_tokens: 150,
-          temperature: 0.3,
-          response_format: { type: "json_object" }
-        )
-        
-        # Parse JSON response
-        require 'json'
-        result = JSON.parse(response, symbolize_names: true)
-        
-        # Validate and clean
-        {
-          label: clean_label(result[:label]),
-          description: result[:description] || "Topic about #{result[:label]}",
-          themes: result[:themes] || [],
-          confidence: result[:confidence] || 0.8
-        }
-      end
-      
-      def build_analysis_prompt(documents, terms)
-        doc_samples = documents.map.with_index do |doc, i|
-          preview = doc.length > 300 ? "#{doc[0..300]}..." : doc
-          "Document #{i + 1}:\n#{preview}"
-        end.join("\n\n")
-        
-        <<~PROMPT
-          Analyze this cluster of related documents and provide a structured summary.
-          
-          Distinctive terms found: #{terms.first(10).join(', ')}
-          
-          Sample documents:
-          #{doc_samples}
-          
-          Provide a JSON response with:
-          {
-            "label": "A 2-4 word topic label",
-            "description": "One sentence describing what connects these documents",
-            "themes": ["theme1", "theme2", "theme3"],
-            "confidence": 0.0-1.0 score of how coherent this topic is
-          }
-          
-          Focus on what meaningfully connects these documents, not just common words.
-        PROMPT
-      end
-      
-      def clean_label(label)
-        return "Unknown Topic" unless label
-        
-        # Remove quotes, trim, limit length
-        cleaned = label.to_s.strip.gsub(/^["']|["']$/, '')
-        cleaned = cleaned.split("\n").first if cleaned.include?("\n")
-        
-        # Limit to reasonable length
-        if cleaned.length > 50
-          cleaned[0..47] + "..."
-        else
-          cleaned
-        end
-      end
-    end
-  end
-end