clusterer 0.1.0 → 0.1.9

data/lib/clusterer/cluster.rb

@@ -0,0 +1,56 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Clusterer
+  class Cluster
+    attr_reader :centroid, :documents
+    include ClusterSimilarity
+
+    def initialize(docs = [])
+      @documents = docs
+    end
+
+    def centroid
+      @centroid ||= (@documents.empty? ? nil : @documents[0].class.centroid_class.new(documents))
+    end
+
+    def merge!(cluster)
+      documents.concat(cluster.documents)
+      @centroid ? centroid.merge!(cluster.centroid) : @centroid = cluster.centroid
+      @intra_cluster_similarity = nil
+    end
+
+    def + (cluster)
+      c = Cluster.new(self.documents.clone)
+      c.merge!(cluster)
+      return c
+    end
+
+    def ==(cluster)
+      cluster && self.documents == cluster.documents
+    end
+
+    def intra_cluster_cosine_similarity
+      @intra_cluster_similarity ||= documents.inject(0) {|n,d| n + d.cosine_similarity(centroid) }
+    end
+  end
+end

data/lib/clusterer/clustering.rb

@@ -0,0 +1,35 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Clusterer
+  class Clustering
+    class << self
+      def cluster(algorithm, objects, options = { })
+        options[:no_of_clusters] ||= Math.sqrt(objects.size).to_i
+        idf = InverseDocumentFrequency.new
+        docs = objects.collect {|o|
+          (defined? yield) == "yield" ? Document.new(o, options.merge(:idf => idf)) {|o| yield(o)} : Document.new(o, options.merge(:idf => idf))}
+        Algorithms.send(algorithm, docs.collect {|d| d.normalize!(idf) }, options[:no_of_clusters])
+      end
+    end
+  end
+end

data/lib/clusterer/document.rb

@@ -0,0 +1,71 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Clusterer
+  #Document tokenizes the text and stores the count of each token in the document.
+  class Document < DocumentBase
+    #stores the text using hash
+
+    #Reference to the original text or the object from which the text is derived.
+    attr_reader :object
+    include(Tokenizer)
+
+    #Reference to the centroid class which is used by Kmeans algorithm
+    def self.centroid_class
+      DocumentsCentroid
+    end
+
+    def initialize (object, options = { })
+      @object = object
+      send(options[:tokenizer] || :simple_tokenizer,
+           ((defined? yield) == "yield" ? yield(object) : object.to_s),
+           options[:tokenizer_options] || {}) {|term| self << term }
+      
+      if (idf = options[:idf])
+        idf.increment_documents_count
+        self.each_key {|term| idf << term}
+      end
+    end
+
+    def << (term)
+      self[term] = (self[term] || 0) + 1
+    end
+
+    def normalize!(idf = nil, add_term = false)
+      normalizing_factor = 0.0
+      idf.increment_documents_count if add_term
+      
+      self.each do |term,frequency|
+        idf << term if add_term
+        f =  idf ? idf[term] : 1.0
+        self[term] = Math.log(1 + frequency) * f
+        normalizing_factor += self[term] ** 2
+      end
+
+      normalizing_factor = Math.sqrt(normalizing_factor)
+      normalizing_factor = 1 if normalizing_factor.zero?
+      self.each {|term,frequency| self[term] = frequency/normalizing_factor}
+      @vector_length = 1
+      self.freeze
+    end
+  end
+end

data/lib/clusterer/document_array.rb

@@ -0,0 +1,79 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Clusterer
+  class DocumentArray < Array
+    #stores the text in an array format, used with LSI or SVD
+    attr_reader :object
+    
+    @@term_array_position_mapper = {}
+    include(Tokenizer)
+
+    def initialize(object = "",options = { })
+      @object = object
+      super(@@term_array_position_mapper.size,0.0)
+      send(options[:tokenizer] || :simple_tokenizer,
+           ((defined? yield) == "yield" ? yield(object) : object.to_s),
+           options[:tokenizer_options] || {}) {|term| self << term }
+
+      if (idf = options[:idf])
+        idf.increment_documents_count
+        self.each_with_index {|ind,val| idf << @@term_array_position_mapper.index(ind) if val && val > 0.0}
+      end
+    end
+    
+    def << (term)
+      self[term_array_position_mapper(term)] = (self[term_array_position_mapper(term)] || 0) + 1
+    end
+    
+    def normalize!(idf = nil, add_term = false)
+      normalizing_factor = 0.0
+      idf.increment_documents_count if add_term
+
+      self[@@term_array_position_mapper.size - 1] ||= 0.0 
+
+      self.each_with_index do |frequency, ind|
+        f = add_term ? (idf << term) : (idf ? idf[@@term_array_position_mapper.index(ind)] : 1.0)
+        self[ind] = (frequency || 0) * f
+        normalizing_factor += self[ind] ** 2
+      end
+      
+      normalizing_factor = Math.sqrt(normalizing_factor)
+      normalizing_factor = 1 if normalizing_factor.zero?
+      self.each_with_index {|frequency, ind| self[ind] = frequency/normalizing_factor}
+      @vector_length = 1.0
+      self.freeze
+    end
+    
+    def vector_length
+      @vector_length ||= Math.sqrt(self.inject(0) {|n,y| n + y*y})
+    end
+
+    def term_array_position_mapper(term)
+      if (x = @@term_array_position_mapper[term])
+        x
+      else
+        @@term_array_position_mapper[term] = @@term_array_position_mapper.size
+      end
+    end
+  end
+end

data/lib/clusterer/document_base.rb

@@ -0,0 +1,32 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Clusterer
+  class DocumentBase < Hash
+    #base class, not meant for direct use
+    include DocumentSimilarity
+
+    def vector_length
+      @vector_length ||= Math.sqrt(self.inject(0) {|n,y| n + y[1]*y[1]}) 
+    end
+  end
+end

data/lib/clusterer/documents_centroid.rb

@@ -0,0 +1,44 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Clusterer
+  class DocumentsCentroid < DocumentBase
+    attr_reader :no_of_documents
+
+    def initialize(docs = [])
+      @no_of_documents = docs.size
+      docs.each do |d|
+        d.each {|w,f| self[w] = (self[w] || 0.0) + f/@no_of_documents}
+      end
+    end
+
+    def merge!(centroid)
+      @vector_length = nil
+      temp = @no_of_documents/(@no_of_documents + centroid.no_of_documents)
+      self.each {|w,v| self[w] = v*temp}
+      @no_of_documents += centroid.no_of_documents
+
+      temp = centroid.no_of_documents/@no_of_documents
+      centroid.each {|w,v| self[w] = (self[w] || 0) + v*temp }
+    end
+  end
+end

data/lib/clusterer/inverse_document_frequency.rb

@@ -0,0 +1,83 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Clusterer
+  #DocumentsCount is used to store the count of number of documents seen.
+  #This class could have been just replaced by a simple variable, in
+  #InverseDocumentFrequency class but to make the InverseDocumentFrequency
+  #class more flexible and be able to store the count in DB/File store
+  #this class is provided.
+  class DocumentsCount
+    attr_reader :value
+    def initialize 
+      @value = 0
+    end
+
+    def increment
+      @value +=1
+    end
+  end
+
+  #TermsCount is used to store the count of number of documents in which the
+  #term has been seen. This class could have been just replaced by a simple
+  #hash object, in InverseDocumentFrequency class but to make the
+  #InverseDocumentFrequency class more flexible and be able to store the
+  #term count in DB/File store this class is provided.
+  class TermsCount < Hash
+    def increment_count(term)
+      self[term] = (self[term] || 0) + 1
+    end
+  end
+  
+  #InverseDocumentFrequency maintains a count of the total number of documents
+  #and the number of documents where a term has been seen with the help of helper
+  #classes. It also calculates the normalizing factor, the formula for whichis
+  #Math.log(total_number of documents/ number of documents containing the term)
+  class InverseDocumentFrequency < Hash
+    def documents_count
+      @documents_count.value
+    end
+
+    def clean_cached_normalizing_factor
+      @nf.clear
+    end
+    
+    def initialize (options = { })
+      @terms_count = options[:terms_count] || TermsCount.new
+      @nf = Hash.new
+      @documents_count = options[:documents_count] || DocumentsCount.new
+    end
+
+    def increment_documents_count
+      @documents_count.increment
+    end
+    
+    def << (term)
+      @terms_count.increment_count(term) unless term.nil? || term.empty?
+    end
+
+    def [] (term)
+      @nf[term] ||= (@terms_count[term] && @documents_count.value >1) ? Math.log(@documents_count.value/@terms_count[term].to_f) : 1.0
+    end
+  end
+end
+

data/lib/clusterer/lsi/dmatrix.rb

@@ -0,0 +1,132 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Clusterer
+  class DMatrix < Matrix
+    #algorithm description from "Simple Algoritms for the partial singular value decomposition"
+    #by J. C. Nash and S. Shlien
+    #Plane rotation method
+    #there were some typos in the original algorithm in the paper
+    #also see the Pascal code in NashSVD, file alg01.pas; for an idea
+    #the partial algorithm is an adaptation of that algo
+    
+    def svd
+      m, n = self.row_size, self.column_size
+      tol =  0.001
+      slimit = [n/4.to_i, 6].max
+      u, z, v = DMatrix[*(1..m).to_a.collect {|i| Array.new(n,0) }], Array.new(n), DMatrix.diagonal(*Array.new(n,1))
+
+      nt = n
+      slimit.times do
+        rcount = nt *(nt-1)/2
+        (nt-1).times do |j|
+          (j+1).upto(nt - 1) do |k|
+            p=q=r=0
+            m.times do |i|
+              p += self[i,j]*self[i,k]
+              q += self[i,j]*self[i,j]
+              r += self[i,k]*self[i,k]
+            end
+            z[j], z[k] = q, r
+            if q < r
+              p, q = p/r, q/r - 1
+              vt = Math.sqrt(4*p*p + q*q)
+              s = Math.sqrt(0.5*(1 - q/vt))
+              s = -s if p < 0
+              c = p / (vt*s)
+            elsif  (q * r <= tol * tol) || (p/q)*(p/r) <= tol
+              rcount -= 1
+              next
+            else
+              p, r = p/q, 1 - r/q
+              vt = Math.sqrt(4*p*p + r*r)
+              c = Math.sqrt(0.5*(1 + r/vt))
+              s = p/(vt * c)
+            end
+            m.times do |i|
+              r = self[i,j]
+              self[i,j] = c * r + s * self[i,k]
+              self[i,k] = -s*r + c * self[i,k]
+            end
+            n.times do |i|
+              r = v[i,j]
+              v[i,j] = c * r + s * v[i,k] #typo in paper replace r by s 
+              v[i,k] = -s*r + c * v[i,k]  #typo in paper replace A(i,k) by v(i,k)
+            end
+          end
+        end
+        until nt < 3 || z[nt-1]/(z[0] + tol) > tol
+          nt -= 1
+        end
+        break unless rcount > 0
+      end
+      nt.times do |j|
+        z[j] = Math.sqrt(z[j])
+        m.times {|i| u[i,j] = self[i,j]/z[j] }
+      end
+      z = DMatrix.diagonal(*z)
+      return u, z, v.transpose
+    end
+
+    def []=(i,j,val)
+      @rows[i][j] = val
+    end
+
+    def self.join_rows(rows)
+      DMatrix[*rows.collect {|r| [*r] }]
+    end
+    
+    def transpose
+      x = super
+      y = DMatrix[]
+      y.instance_variable_set("@rows",x.instance_variable_get("@rows"))
+      y
+    end
+    
+    def self.join_columns(columns)
+      DMatrix[*columns.collect {|c| [*c] }].transpose
+    end
+  end
+end
+
+class Matrix
+  alias :nrow :row_size
+  alias :ncol :column_size
+  alias :rows :row_vectors
+  alias :columns :column_vectors
+end
+
+class Vector
+  alias :dot :inner_product
+  
+  def transpose
+    self
+  end
+  
+  def dimensions
+    [size]
+  end
+
+  def / (x)
+    self * (1/x)
+  end
+end