classifier 1.3.5 → 1.4.0

data/lib/classifier/lsi.rb

@@ -3,15 +3,16 @@
 # License::   LGPL
 
 begin
-   raise LoadError if ENV['NATIVE_VECTOR'] == "true" # to test the native vector class, try `rake test NATIVE_VECTOR=true`
-
-   require 'gsl' # requires http://rb-gsl.rubyforge.org/
-   require 'classifier/extensions/vector_serialize'
-   $GSL = true
+  # to test the native vector class, try `rake test NATIVE_VECTOR=true`
+  raise LoadError if ENV['NATIVE_VECTOR'] == 'true'
 
+  require 'gsl' # requires https://github.com/SciRuby/rb-gsl/
+  require 'classifier/extensions/vector_serialize'
+  $GSL = true
 rescue LoadError
-	warn "Notice: for 10x faster LSI support, please install http://rb-gsl.rubyforge.org/"
-	require 'classifier/extensions/vector'
+  warn 'Notice: for 10x faster LSI support, please install https://github.com/SciRuby/rb-gsl/'
+  $GSL = false
+  require 'classifier/extensions/vector'
 end
 
 require 'classifier/lsi/word_list'
@@ -19,12 +20,10 @@ require 'classifier/lsi/content_node'
 require 'classifier/lsi/summary'
 
 module Classifier
-
   # This class implements a Latent Semantic Indexer, which can search, classify and cluster
   # data based on underlying semantic relations. For more information on the algorithms used,
   # please consult Wikipedia[http://en.wikipedia.org/wiki/Latent_Semantic_Indexing].
   class LSI
-
     attr_reader :word_list
     attr_accessor :auto_rebuild
 
@@ -34,8 +33,10 @@ module Classifier
     #
     def initialize(options = {})
       @auto_rebuild = true unless options[:auto_rebuild] == false
-      @word_list, @items = WordList.new, {}
-      @version, @built_at_version = 0, -1
+      @word_list = WordList.new
+      @items = {}
+      @version = 0
+      @built_at_version = -1
     end
 
     # Returns true if the index needs to be rebuilt.  The index needs
@@ -58,7 +59,7 @@ module Classifier
     #   ar = ActiveRecordObject.find( :all )
     #   lsi.add_item ar, *ar.categories { |x| ar.content }
     #
-    def add_item( item, *categories, &block )
+    def add_item(item, *categories, &block)
       clean_word_hash = block ? block.call(item).clean_word_hash : item.to_s.clean_word_hash
       @items[item] = ContentNode.new(clean_word_hash, *categories)
       @version += 1
@@ -69,24 +70,25 @@ module Classifier
     # you are passing in a string with no categorries. item
     # will be duck typed via to_s .
     #
-    def <<( item )
-      add_item item
+    def <<(item)
+      add_item(item)
     end
 
     # Returns the categories for a given indexed items. You are free to add and remove
     # items from this as you see fit. It does not invalide an index to change its categories.
     def categories_for(item)
       return [] unless @items[item]
-      return @items[item].categories
+
+      @items[item].categories
     end
 
     # Removes an item from the database, if it is indexed.
     #
-    def remove_item( item )
-      if @items.keys.contain? item
-        @items.remove item
-        @version += 1
-      end
+    def remove_item(item)
+      return unless @items.key?(item)
+
+      @items.delete(item)
+      @version += 1
     end
 
     # Returns an array of items that are indexed.
@@ -94,13 +96,6 @@ module Classifier
       @items.keys
     end
 
-    # Returns the categories for a given indexed items. You are free to add and remove
-    # items from this as you see fit. It does not invalide an index to change its categories.
-    def categories_for(item)
-      return [] unless @items[item]
-      return @items[item].categories
-    end
-
     # This function rebuilds the index if needs_rebuild? returns true.
     # For very large document spaces, this indexing operation may take some
     # time to complete, so it may be wise to place the operation in another
@@ -115,30 +110,31 @@ module Classifier
     # cutoff parameter tells the indexer how many of these values to keep.
     # A value of 1 for cutoff means that no semantic analysis will take place,
     # turning the LSI class into a simple vector search engine.
-    def build_index( cutoff=0.75 )
+    def build_index(cutoff = 0.75)
       return unless needs_rebuild?
+
       make_word_list
 
       doc_list = @items.values
-      tda = doc_list.collect { |node| node.raw_vector_with( @word_list ) }
+      tda = doc_list.collect { |node| node.raw_vector_with(@word_list) }
 
       if $GSL
-         tdm = GSL::Matrix.alloc(*tda).trans
-         ntdm = build_reduced_matrix(tdm, cutoff)
-
-         ntdm.size[1].times do |col|
-           vec = GSL::Vector.alloc( ntdm.column(col) ).row
-           doc_list[col].lsi_vector = vec
-           doc_list[col].lsi_norm = vec.normalize
-         end
+        tdm = GSL::Matrix.alloc(*tda).trans
+        ntdm = build_reduced_matrix(tdm, cutoff)
+
+        ntdm.size[1].times do |col|
+          vec = GSL::Vector.alloc(ntdm.column(col)).row
+          doc_list[col].lsi_vector = vec
+          doc_list[col].lsi_norm = vec.normalize
+        end
       else
-         tdm = Matrix.rows(tda).trans
-         ntdm = build_reduced_matrix(tdm, cutoff)
+        tdm = Matrix.rows(tda).trans
+        ntdm = build_reduced_matrix(tdm, cutoff)
 
-         ntdm.row_size.times do |col|
-           doc_list[col].lsi_vector = ntdm.column(col) if doc_list[col]
-           doc_list[col].lsi_norm = ntdm.column(col).normalize  if doc_list[col]
-         end
+        ntdm.row_size.times do |col|
+          doc_list[col].lsi_vector = ntdm.column(col) if doc_list[col]
+          doc_list[col].lsi_norm = ntdm.column(col).normalize if doc_list[col]
+        end
       end
 
       @built_at_version = @version
@@ -152,13 +148,13 @@ module Classifier
     # your dataset's general content. For example, if you were to use categorize on the
     # results of this data, you could gather information on what your dataset is generally
     # about.
-    def highest_relative_content( max_chunks=10 )
-       return [] if needs_rebuild?
+    def highest_relative_content(max_chunks = 10)
+      return [] if needs_rebuild?
 
-       avg_density = Hash.new
-       @items.each_key { |x| avg_density[x] = proximity_array_for_content(x).inject(0.0) { |x,y| x + y[1]} }
+      avg_density = {}
+      @items.each_key { |x| avg_density[x] = proximity_array_for_content(x).inject(0.0) { |x, y| x + y[1] } }
 
-       avg_density.keys.sort_by { |x| avg_density[x] }.reverse[0..max_chunks-1].map
+      avg_density.keys.sort_by { |x| avg_density[x] }.reverse[0..max_chunks - 1].map
     end
 
     # This function is the primitive that find_related and classify
@@ -173,17 +169,17 @@ module Classifier
     # The parameter doc is the content to compare. If that content is not
     # indexed, you can pass an optional block to define how to create the
     # text data. See add_item for examples of how this works.
-    def proximity_array_for_content( doc, &block )
+    def proximity_array_for_content(doc, &block)
       return [] if needs_rebuild?
 
-      content_node = node_for_content( doc, &block )
+      content_node = node_for_content(doc, &block)
       result =
         @items.keys.collect do |item|
-          if $GSL
-             val = content_node.search_vector * @items[item].search_vector.col
-          else
-             val = (Matrix[content_node.search_vector] * @items[item].search_vector)[0]
-          end
+          val = if $GSL
+                  content_node.search_vector * @items[item].search_vector.col
+                else
+                  (Matrix[content_node.search_vector] * @items[item].search_vector)[0]
+                end
           [item, val]
         end
       result.sort_by { |x| x[1] }.reverse
@@ -194,17 +190,17 @@ module Classifier
     # calculated vectors instead of their full versions. This is useful when
     # you're trying to perform operations on content that is much smaller than
     # the text you're working with. search uses this primitive.
-    def proximity_norms_for_content( doc, &block )
+    def proximity_norms_for_content(doc, &block)
       return [] if needs_rebuild?
 
-      content_node = node_for_content( doc, &block )
+      content_node = node_for_content(doc, &block)
       result =
         @items.keys.collect do |item|
-          if $GSL
-            val = content_node.search_norm * @items[item].search_norm.col
-          else
-            val = (Matrix[content_node.search_norm] * @items[item].search_norm)[0]
-          end
+          val = if $GSL
+                  content_node.search_norm * @items[item].search_norm.col
+                else
+                  (Matrix[content_node.search_norm] * @items[item].search_norm)[0]
+                end
           [item, val]
         end
       result.sort_by { |x| x[1] }.reverse
@@ -217,11 +213,12 @@ module Classifier
     #
     # While this may seem backwards compared to the other functions that LSI supports,
     # it is actually the same algorithm, just applied on a smaller document.
-    def search( string, max_nearest=3 )
+    def search(string, max_nearest = 3)
       return [] if needs_rebuild?
-      carry = proximity_norms_for_content( string )
+
+      carry = proximity_norms_for_content(string)
       result = carry.collect { |x| x[0] }
-      return result[0..max_nearest-1]
+      result[0..max_nearest - 1]
     end
 
     # This function takes content and finds other documents
@@ -233,11 +230,11 @@ module Classifier
     # This is particularly useful for identifing clusters in your document space.
     # For example you may want to identify several "What's Related" items for weblog
     # articles, or find paragraphs that relate to each other in an essay.
-    def find_related( doc, max_nearest=3, &block )
+    def find_related(doc, max_nearest = 3, &block)
       carry =
-        proximity_array_for_content( doc, &block ).reject { |pair| pair[0] == doc }
+        proximity_array_for_content(doc, &block).reject { |pair| pair[0] == doc }
       result = carry.collect { |x| x[0] }
-      return result[0..max_nearest-1]
+      result[0..max_nearest - 1]
     end
 
     # This function uses a voting system to categorize documents, based on
@@ -249,10 +246,17 @@ module Classifier
     # text. A cutoff of 1 means that every document in the index votes on
     # what category the document is in. This may not always make sense.
     #
-    def classify( doc, cutoff=0.30, &block )
+    def classify(doc, cutoff = 0.30, &block)
+      votes = vote(doc, cutoff, &block)
+
+      ranking = votes.keys.sort_by { |x| votes[x] }
+      ranking[-1]
+    end
+
+    def vote(doc, cutoff = 0.30, &block)
       icutoff = (@items.size * cutoff).round
-      carry = proximity_array_for_content( doc, &block )
-      carry = carry[0..icutoff-1]
+      carry = proximity_array_for_content(doc, &block)
+      carry = carry[0..icutoff - 1]
       votes = {}
       carry.each do |pair|
         categories = @items[pair[0]].categories
@@ -261,23 +265,46 @@ module Classifier
           votes[category] += pair[1]
         end
       end
+      votes
+    end
+
+    # Returns the same category as classify() but also returns
+    # a confidence value derived from the vote share that the
+    # winning category got.
+    #
+    # e.g.
+    # category,confidence = classify_with_confidence(doc)
+    # if confidence < 0.3
+    #   category = nil
+    # end
+    #
+    #
+    # See classify() for argument docs
+    def classify_with_confidence(doc, cutoff = 0.30, &block)
+      votes = vote(doc, cutoff, &block)
+      votes_sum = votes.values.inject(0.0) { |sum, v| sum + v }
+      return [nil, nil] if votes_sum.zero?
 
       ranking = votes.keys.sort_by { |x| votes[x] }
-      return ranking[-1]
+      winner = ranking[-1]
+      vote_share = votes[winner] / votes_sum.to_f
+      [winner, vote_share]
     end
 
     # Prototype, only works on indexed documents.
     # I have no clue if this is going to work, but in theory
     # it's supposed to.
-    def highest_ranked_stems( doc, count=3 )
-      raise "Requested stem ranking on non-indexed content!" unless @items[doc]
+    def highest_ranked_stems(doc, count = 3)
+      raise 'Requested stem ranking on non-indexed content!' unless @items[doc]
+
       arr = node_for_content(doc).lsi_vector.to_a
-      top_n = arr.sort.reverse[0..count-1]
-      return top_n.collect { |x| @word_list.word_for_index(arr.index(x))}
+      top_n = arr.sort.reverse[0..count - 1]
+      top_n.collect { |x| @word_list.word_for_index(arr.index(x)) }
     end
 
     private
-    def build_reduced_matrix( matrix, cutoff=0.75 )
+
+    def build_reduced_matrix(matrix, cutoff = 0.75)
       # TODO: Check that M>=N on these dimensions! Transpose helps assure this
       u, v, s = matrix.SV_decomp
 
@@ -287,23 +314,21 @@ module Classifier
         s[ord] = 0.0 if s[ord] < s_cutoff
       end
       # Reconstruct the term document matrix, only with reduced rank
-      u * ($GSL ? GSL::Matrix : ::Matrix).diag( s ) * v.trans
+      u * ($GSL ? GSL::Matrix : ::Matrix).diag(s) * v.trans
     end
 
     def node_for_content(item, &block)
-      if @items[item]
-        return @items[item]
-      else
-        clean_word_hash = block ? block.call(item).clean_word_hash : item.to_s.clean_word_hash
+      return @items[item] if @items[item]
 
-        cn = ContentNode.new(clean_word_hash, &block) # make the node and extract the data
+      clean_word_hash = block ? block.call(item).clean_word_hash : item.to_s.clean_word_hash
 
-        unless needs_rebuild?
-          cn.raw_vector_with( @word_list ) # make the lsi raw and norm vectors
-        end
+      cn = ContentNode.new(clean_word_hash, &block) # make the node and extract the data
+
+      unless needs_rebuild?
+        cn.raw_vector_with(@word_list) # make the lsi raw and norm vectors
       end
 
-      return cn
+      cn
     end
 
     def make_word_list
@@ -312,7 +337,5 @@ module Classifier
         node.word_hash.each_key { |key| @word_list.add_word key }
       end
     end
-
   end
 end
-

data/lib/classifier.rb

@@ -26,5 +26,6 @@
 
 require 'rubygems'
 require 'classifier/extensions/string'
+require 'classifier/extensions/vector'
 require 'classifier/bayes'
 require 'classifier/lsi'

data/test/test_helper.rb

@@ -0,0 +1,5 @@
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require 'minitest'
+require 'minitest/autorun'
+require 'classifier'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: classifier
 version: !ruby/object:Gem::Version
-  version: 1.3.5
+  version: 1.4.0
 platform: ruby
 authors:
 - Lucas Carlson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-17 00:00:00.000000000 Z
+date: 2024-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast-stemmer
@@ -24,20 +24,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: mathn
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -99,11 +85,12 @@ files:
 - lib/classifier/lsi/content_node.rb
 - lib/classifier/lsi/summary.rb
 - lib/classifier/lsi/word_list.rb
+- test/test_helper.rb
 homepage: https://github.com/cardmagic/classifier
 licenses:
 - LGPL
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -118,9 +105,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.5.9
+signing_key:
 specification_version: 4
 summary: A general classifier module to allow Bayesian and other types of classifications.
 test_files: []