clusterer 0.1.0 → 0.1.9

data/lib/clusterer/lsi/document_vector.rb

@@ -0,0 +1,54 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Clusterer
+  module DocumentVector
+    module InstanceMethods
+      attr_accessor :position
+      
+      def cosine_similarity(doc)
+        return 1.0 unless doc# && doc.centroid
+        self.dot((doc.class == DocumentsCentroidVector ? doc.centroid : doc)) #.transpose
+      end
+    end
+
+    module ClassMethods
+      def centroid_class
+        DocumentsCentroidVector
+      end
+    end
+  end
+end
+
+if $LINALG == true
+  module Linalg
+    class DMatrix
+      include Clusterer::DocumentVector::InstanceMethods
+      extend Clusterer::DocumentVector::ClassMethods
+    end
+  end
+else
+  class Vector
+    include Clusterer::DocumentVector::InstanceMethods
+    extend Clusterer::DocumentVector::ClassMethods
+  end
+end

data/lib/clusterer/lsi/documents_centroid_vector.rb

@@ -0,0 +1,51 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Clusterer
+  class DocumentsCentroidVector
+    attr_reader :no_of_documents
+    attr_reader :centroid
+    
+    def initialize(docs = [])
+      @no_of_documents = docs.size
+      return if @no_of_documents == 0
+      @centroid = docs[0]
+      docs.slice(1..docs.length).each {|d| @centroid = @centroid + d}
+      @centroid = @centroid / @no_of_documents
+    end
+
+    def to_dmatrix
+      @centroid
+    end
+    
+    def merge!(centroid)
+      return unless centroid.centroid
+      unless @centroid
+        @centroid = centroid.centroid
+        @no_of_documents = centroid.no_of_documents
+      else
+        @centroid = (@centroid * @no_of_documents) + (centroid.centroid * centroid.no_of_documents)
+        @centroid = @centroid / (@no_of_documents += centroid.no_of_documents)
+      end
+    end
+  end
+end

data/lib/clusterer/lsi/lsi.rb

@@ -0,0 +1,95 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+begin
+  require 'linalg'
+  $LINALG = true
+rescue LoadError
+  warn 'For faster LSI support, please install Linalg: '
+  require 'clusterer/lsi/dmatrix'
+end
+
+require 'clusterer/lsi/document_vector'
+require 'clusterer/lsi/documents_centroid_vector'
+
+module Clusterer
+  class Lsi
+    include Linalg if $LINALG
+
+    attr_reader :documents
+    def initialize(docs)
+      @documents = docs
+    end
+
+    def rebuild_if_needed
+      perform_svd unless @t && @d && @s
+    end
+
+    def clear_cached_results
+      @t= @s= @d= @s_inv= @sd= nil
+    end
+    
+    def perform_svd (cutoff = 0.80)
+      matrix = DMatrix[*@documents].transpose
+      @t, @s, @d =  matrix.svd
+      val = @s.trace * cutoff
+      cnt = -1
+      (0..([@s.nrow, @s.ncol].min - 1)).inject(0) {|n,v| cnt += 1; (n > val) ? break : n + @s[v,v] }
+      @t = DMatrix.join_columns((0..cnt).collect {|i|@t.column(i) })
+      @d = DMatrix.join_rows((0..cnt).collect {|i| @d.row(i) })
+      @s = DMatrix.join_columns((0..cnt).collect {|i|@s.column(i) })
+      @s = DMatrix.join_rows((0..cnt).collect {|i|@s.row(i) }) unless @s.ncol == cnt
+    end
+
+    def cluster_documents(k, options = { })
+      rebuild_if_needed
+      cnt = -1
+      clusters = Algorithms.send(options[:algorithm] || :kmeans, 
+                                 sd.columns.collect{|c| c.position = (cnt += 1); c}, k, options)
+      clusters.collect {|clus| clus.documents.collect {|d| @documents[d.position]}}
+    end
+    
+    def search(document, threshold = 0.5)
+      rebuild_if_needed
+      vec = $LINALG ? DMatrix[document] : DMatrix[document] #DMatrix[document] #transform_to_vector(document)
+      vec = (vec * @t) * s_inv
+      results = []
+      vec = (vec * @s).transpose # * @s
+      vec = vec.column(0) unless $LINALG
+      sd.columns.each_with_index {|d,i| results << documents[i] if d.cosine_similarity(vec) >= threshold}
+      results
+    end
+
+    def <<(doc)
+      @documents << doc
+    end
+    
+    private
+    def sd
+      @sd ||= @s*@d
+    end
+    
+    def s_inv
+      @s_inv ||=  @s.inverse
+    end
+  end
+end

data/lib/clusterer/similarity.rb

@@ -0,0 +1,34 @@
+#The MIT License
+
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+
+module Clusterer
+  module DocumentSimilarity
+    #find similarity between two documents, or cluster centroids
+    def cosine_similarity(document)
+      return 1.0 if self.empty? || document.nil? || document.empty?
+      similarity = 0
+      self.each do |w,value|
+        similarity += (value * (document[w] || 0))
+      end
+      similarity /= (self.vector_length * document.vector_length)
+    end
+  end
+
+  module ClusterSimilarity
+   #the algorithms given below find similarity between two clusters
+    def intra_cluster_similarity(y)
+      (self+y).intra_cluster_cosine_similarity - self.intra_cluster_cosine_similarity - y.intra_cluster_cosine_similarity
+    end
+
+    def centroid_similarity(y)
+      self.centroid.cosine_similarity(y.centroid)
+    end
+
+    def upgma(y)
+      self.documents.inject(0) do |n,d|
+        n + y.documents.inject(0) {|s,e| s + d.cosine_similarity(e) }
+      end / (self.documents.size * y.documents.size)
+    end
+  end
+end

data/lib/{word_hash.rb → clusterer/stop_words.rb}

@@ -1,28 +1,26 @@
-#The MIT License
-
+#--
 ###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-begin
-  require 'stemmer'
-rescue LoadError
-  puts "Please install stemmer from http://rubyforge.org/projects/stemmer or 'gem install stemmer'"
-  exit(-1)
-end
-
-class String
-  def clean_word_hash
-    word_hash gsub(/[^\w\s]/,"").split
-  end
-  private
-  def word_hash(words)
-    h = Hash.new
-    words.each do |w|
-      w = w.downcase.stem
-      h[w] = (h[w] || 0) + 1 if w.size > 2 and !STOP_WORDS.include?(w)
-    end
-    h
-  end
-
+module Clusterer
   STOP_WORDS = ["and",
                 "but",
                 "came",

data/lib/clusterer/tokenizer.rb

@@ -0,0 +1,70 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+begin
+  require 'stemmer'
+rescue LoadError
+  puts "If you want to use stemming for better performance, then please install stemmer from http://rubyforge.org/projects/stemmer or 'gem install stemmer'"
+  class String
+    def stem
+      self
+    end
+  end
+end
+
+module Clusterer
+  #the tokenizer algorithms take a block, to which the string tokens are passed
+  
+  module Tokenizer
+    def simple_tokenizer (text, options = {})
+      text.gsub(/[^\w\s]/,"").split.each do |word|
+        word.downcase!
+        word = word.stem unless options[:no_stem]
+        yield(word) if word.size > 2 and !STOP_WORDS.include?(word)
+      end
+    end
+
+    def simple_ngram_tokenizer (text, options = {})
+      ngram = options[:ngram] || 3
+      
+      ngram_list = (0..ngram).collect { []}
+      text.split(/[\.\?\!]/).each do |sentence|
+        #split the text into sentences, Ngrams cannot straddle sentences
+        
+        sentence.gsub(/[^\w\s]/,"").split.each do |word|
+          word.downcase!
+          word = word.stem unless options[:no_stem]
+          if word.size > 2 and !STOP_WORDS.include?(word)
+            yield(word)
+            2.upto(ngram) do |i|
+              ngram_list[i].delete_if {|j| j << word; j.size == i ? (yield(j.join(" ")); true) : false}
+              ngram_list[i] << [word]
+            end
+          else
+            #the ngrams cannot have a stop word at beginning and end
+            2.upto(ngram) {|i| ngram_list[i].delete_if {|j| (j.size == i - 1) ? true : (j << word; false)}}
+          end
+        end
+      end
+    end
+  end
+end

data/tests/algorithms_test.rb

@@ -0,0 +1,48 @@
+#The MIT License
+
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'clusterer'
+
+class TestAlgorithms < Test::Unit::TestCase
+  include Clusterer
+
+  def setup
+    @idf = InverseDocumentFrequency.new()
+    @d = Document.new("hello world, mea culpa, goodbye world.", :idf => @idf).normalize!(@idf)
+    @e = Document.new("the world is not a bad place to live.", :idf => @idf).normalize!(@idf)
+    @f = Document.new("the world is a crazy place to live.", :idf => @idf).normalize!(@idf)
+    @g = Document.new("unique document.")
+  end
+
+  def test_kmeans
+    3.times do
+      assert_equal 2, Algorithms.kmeans([@d, @e, @f, @g], 2).size
+      assert_equal 2, Algorithms.kmeans([@d, @e, @f, @g], 2, :maximum_iterations => 5).size
+      assert_equal 2, Algorithms.kmeans([@d, @e, @f, @g], 2, :maximum_iterations => 5,
+                                        :seeds => [Cluster.new([@d]), Cluster.new([@d])]).size
+      assert_equal 3, Algorithms.kmeans([@d, @e, @f, @g],3).size
+    end
+  end
+
+  def test_hierarchical_clustering
+    assert_equal 2, Algorithms.hierarchical([@d, @e, @f, @g], 2, :similarity_function => :intra_cluster_similarity).size
+    assert_equal 1, Algorithms.hierarchical([@d, @e, @f, @g], 1, :similarity_function => :centroid_similarity).size
+    assert_equal 2, Algorithms.hierarchical([@d, @e, @f, @g], 2, :similarity_function => :upgma).size
+    assert_equal 2, Algorithms.hierarchical([@d, @e, @f, @g], 2, :refined => true).size
+    assert_equal 3, Algorithms.hierarchical([@d, @e, @f, @g], 3, :similarity_function => :centroid_similarity,
+                                            :refined => true).size
+  end
+
+  def test_bisecting_kmeans
+    assert_equal 2, Algorithms.bisecting_kmeans([@d, @e, @f, @g], 2, :maximum_iterations => 5).size
+    assert_equal 1, Algorithms.bisecting_kmeans([@d, @e, @f, @g], 1).size
+    assert_equal 2, Algorithms.bisecting_kmeans([@d, @e, @f, @g], 2).size
+    assert_equal 2, Algorithms.bisecting_kmeans([@d, @e, @f, @g], 2, :maximum_iterations => 5, :refined => true).size
+    assert_equal 3, Algorithms.bisecting_kmeans([@d, @e, @f, @g], 3, :refined => true).size
+  end
+
+end

data/tests/bayes_test.rb

@@ -0,0 +1,68 @@
+#The MIT License
+
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'clusterer'
+
+class TestBayes < Test::Unit::TestCase
+  include Clusterer
+
+  def setup
+    @idf = InverseDocumentFrequency.new()
+    @d = Document.new("hello world, mea culpa, goodbye world.",:idf => @idf).normalize!(@idf)
+    @e = Document.new("the world is not a bad place to live.",:idf => @idf).normalize!(@idf)
+    @f = Document.new("the world is a crazy place to live.",:idf => @idf).normalize!(@idf)
+    @g = Document.new("unique document.")
+  end
+
+  def test_multinomial
+    b = MultinomialBayes.new(["good", "evil"])
+    b.train(@d, "good")
+    b.train(@e, :good)
+    b.train_evil @g
+    assert_raise(ArgumentError) { b.train(@e, :funny) }
+    assert_equal :good, b.classify(@f)
+    assert !b.distribution(@f).empty?
+  end
+
+  def test_complement
+    b = ComplementBayes.new(["good", "evil"])
+    b.train(@d, "good")
+    b.train(@e, :good)
+    b.train_evil @g
+    assert_raise(ArgumentError) { b.train(@e, :funny) }
+    assert_equal :good, b.classify(@f)
+    assert !b.distribution(@f).empty?
+  end
+
+  def test_weight_normalized_complement
+    b = WeightNormalizedComplementBayes.new(["good", "evil"])
+    b.train(@d, "good")
+    b.train(@e, :good)
+    b.train(@g, "evil")
+    assert_raise(ArgumentError) { b.train(@e, :funny) }
+    assert_equal :good, b.classify(@f)
+    assert !b.distribution(@f).empty?
+    b.train(@f, "good")
+    assert b.instance_variable_get("@weighted_likelihood").empty?
+    assert_equal :good, b.classify(@f)
+    assert !b.instance_variable_get("@weighted_likelihood").empty?
+  end
+
+  def test_weight_normalized_multinomial
+    b = WeightNormalizedMultinomialBayes.new(["good", "evil"])
+    b.train(@d, "good")
+    b.train(@e, :good)
+    b.train(@g, "evil")
+    assert_raise(ArgumentError) { b.train(@e, :funny) }
+    assert_equal :good, b.classify(@f)
+    assert !b.distribution(@f).empty?
+    b.train(@f, "good")
+    assert b.instance_variable_get("@weighted_likelihood").empty?
+    assert_equal :good, b.classify(@f)
+    assert !b.instance_variable_get("@weighted_likelihood").empty?
+  end
+end