clusterer 0.1.0 → 0.1.9

data/tests/cluster_test.rb

@@ -0,0 +1,54 @@
+#The MIT License
+
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'clusterer'
+
+class TestCluster < Test::Unit::TestCase
+  include Clusterer
+
+  def setup
+    @idf = InverseDocumentFrequency.new()
+    @d = Document.new("hello world, mea culpa, goodbye world.", :idf => @idf).normalize!(@idf)
+    @e = Document.new("the world is not a bad place to live.", :idf => @idf).normalize!(@idf)
+    @f = Document.new("the world is a crazy place to live.", :idf => @idf).normalize!(@idf)
+    @g = Document.new("unique document.")
+    @c1 = Cluster.new([@d, @e, @f])
+    @c2 = Cluster.new([@d, @g])
+  end
+
+  def test_centroid
+    assert @c1.centroid
+    assert_nil Cluster.new.centroid
+  end
+
+  def test_merge!
+    @c1.merge!(@c2)
+    assert_nil @c1.instance_variable_get("@intra_cluster_similarity")
+    c = Cluster.new
+    c.merge!(@c2)
+    assert_equal c.centroid, @c2.centroid
+  end
+
+  def test_add
+    c= @c1 + @c2
+    assert_not_equal c, @c1
+    assert_not_equal c, @c2
+    assert_equal (Cluster.new + @c1), @c1
+  end
+
+  def test_equal
+    assert_not_equal @c1, @c2
+    assert_not_equal @c1, nil
+    assert_equal @c1, Cluster.new([@d, @e, @f])
+  end
+
+  def test_intra_cluster_cosine_similarity
+    assert_equal Cluster.new.intra_cluster_cosine_similarity,Cluster.new.intra_cluster_cosine_similarity
+    assert @c1.intra_cluster_cosine_similarity > 0
+    assert_not_equal @c1.intra_cluster_cosine_similarity, @c2.intra_cluster_cosine_similarity
+  end
+end

data/tests/document_array_test.rb

@@ -0,0 +1,64 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'clusterer'
+
+class DocumentArrayTest < Test::Unit::TestCase
+  include Clusterer
+
+  def setup
+    @idf = InverseDocumentFrequency.new()
+    @d = DocumentArray.new("hello world, mea culpa, goodbye world.",:idf => @idf)
+    @e = DocumentArray.new("the world is not a bad place to live.",:idf => @idf).normalize!(@idf)
+    @f = DocumentArray.new("the world is a crazy place to live.",:idf => @idf).normalize!(@idf)
+    @g = DocumentArray.new("unique document.")
+  end
+  
+  def test_insert
+    t = @d.term_array_position_mapper('weird')
+    assert_nil @d[t]
+    @d << "weird"
+    assert @d[t]
+    assert_equal @d[t] + 1, (@d << "weird"; @d[t])
+  end
+
+  def test_term_array_position_mapper
+    @d.term_array_position_mapper("world")
+  end
+  
+  def test_vector_length
+    assert_not_nil @f.vector_length
+    assert_in_delta 1.0, @f.vector_length, 0.01
+  end
+
+  def test_object
+    assert_equal "unique document.", @g.object
+  end
+
+  def test_normalize!
+    @d.normalize!
+    assert_in_delta 1.0, @d.vector_length, 0.01
+  end
+end

data/tests/document_centroid_test.rb

@@ -0,0 +1,64 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'clusterer'
+
+class DocumentCentroidTest < Test::Unit::TestCase
+  include Clusterer
+
+  def setup
+    @idf = InverseDocumentFrequency.new()
+    @d = Document.new("hello world, mea culpa, goodbye world.", :idf => @idf).normalize!(@idf)
+    @e = Document.new("the world is not a bad place to live.", :idf => @idf).normalize!(@idf)
+    @f = Document.new("the world is a crazy place to live.", :idf => @idf).normalize!(@idf)
+    @g = Document.new("unique document.")
+    @c1 = Clusterer::Cluster.new([@d, @e, @f])
+    @c2 = Clusterer::Cluster.new([@d, @g])
+  end
+
+  def test_initialization
+    c = DocumentsCentroid.new([@d, @e, @f])
+    assert 3, c.no_of_documents
+    assert c.vector_length
+  end
+
+  def test_merge!
+    c1 = DocumentsCentroid.new([@d, @e, @f])
+    c2 = DocumentsCentroid.new([@d, @g])
+    c3 = c2.clone
+    t = c1.vector_length
+    c1.merge!(c2)
+    assert 5, c1.no_of_documents
+    assert_equal c2, c3
+    assert_not_equal t, c1.vector_length
+
+    c4 = DocumentsCentroid.new()
+    t = c1.vector_length
+    c1.merge!(c4)
+    assert 5, c1.no_of_documents
+    assert_equal t, c1.vector_length
+  end
+end

data/tests/document_test.rb

@@ -0,0 +1,71 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'clusterer'
+
+class DocumentTest < Test::Unit::TestCase
+  include Clusterer
+
+  def setup
+    @idf = InverseDocumentFrequency.new()
+    @d = Document.new("hello world, mea culpa, goodbye world.",:idf => @idf) {|x| x.to_s }
+    @e = Document.new("the world is not a bad place to live.",:idf => @idf).normalize!(@idf)
+    @f = Document.new("the world is a crazy place to live.").normalize!(@idf, true)
+    @g = Document.new(["unique ","document."])
+  end
+  
+  def test_cosine_similarity
+    assert @d.cosine_similarity(@d) > 0.99 #almost 1
+    assert @f.cosine_similarity(@g) < 0.0001  # almost 0
+  end
+
+  def test_centroid_class
+    assert_equal DocumentsCentroid, @d.class.centroid_class
+  end
+  
+  def test_add
+    assert_nil @d['weird']
+    t = @d.length + 1
+    @d << "weird"
+    assert_equal t, @d.length 
+    assert @d['weird']
+    assert_equal @d['weird'] + 1, (@d << "weird"; @d['weird'])
+  end
+
+  def test_vector_length
+    assert_not_nil @f.vector_length
+    assert_in_delta 1.0, @f.vector_length, 0.01
+  end
+
+  def test_object
+    assert_equal ["unique ","document."], @g.object
+    assert_equal "the world is a crazy place to live.", @f.object
+  end
+
+  def test_normalize!
+    @d.normalize!
+    assert_in_delta 1.0, @d.vector_length, 0.01
+  end
+end

data/tests/inverse_document_frequency_test.rb

@@ -0,0 +1,76 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'clusterer'
+
+class InverseDocumentFrequencyTest < Test::Unit::TestCase
+  include Clusterer
+
+  def test_insertion
+    idf = InverseDocumentFrequency.new()
+    idf << "hello"
+    assert_equal 1, idf.instance_variable_get("@terms_count").size
+    assert_in_delta 1.0, idf.instance_variable_get("@terms_count")["hello"], 0.01
+    idf << "hello"
+    assert_equal 2.0, idf.instance_variable_get("@terms_count")["hello"]
+    assert_equal 1, idf.instance_variable_get("@terms_count").size
+    
+    idf << "world"
+    assert_equal 2, idf.instance_variable_get("@terms_count").size
+  end
+
+  def test_documents_count
+    idf = InverseDocumentFrequency.new()
+    Document.new("the world is not a bad place to live.", :idf => idf)
+    Document.new("the world is a crazy place to live.", :idf => idf)
+    assert_equal 2, idf.documents_count
+  end
+
+  def test_clean_cached_normalizing_factor
+    idf = InverseDocumentFrequency.new()
+    Document.new("the world is not a bad place to live.", :idf => idf)
+    Document.new("hello, the world is a crazy place to live.", :idf => idf)
+    t ="crazy".stem
+    f = idf[t]
+    assert_in_delta Math.log(2/1), f, 0.1
+    Document.new("the world is a weird place to live.", :idf => idf)
+    assert_equal f, idf[t]
+    idf.clean_cached_normalizing_factor
+    assert_not_equal f, idf[t]
+  end
+
+  def test_array_index
+    idf = InverseDocumentFrequency.new()
+    Document.new("the world is not a bad place to live.", :idf => idf)
+    assert_in_delta 1.0, idf["world"], 0.001
+    assert_in_delta 1.0, idf["hello"], 0.001
+    
+    Document.new("hello, the world is a crazy place to live.", :idf => idf)
+    idf.clean_cached_normalizing_factor
+    idf << "hello"
+    assert idf["hello"] < 0.99
+  end
+end

data/tests/lsi_test.rb

@@ -0,0 +1,77 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'clusterer'
+
+class LsiTest < Test::Unit::TestCase
+  include Clusterer
+
+  def setup
+    @idf = InverseDocumentFrequency.new()
+    @d = DocumentArray.new("hello world, mea culpa, goodbye world.",:idf => @idf)
+    @e = DocumentArray.new("the world is not a bad place to live.",:idf => @idf)
+    @f = DocumentArray.new("the world is a crazy place to live.",:idf => @idf)
+    @g = DocumentArray.new("unique document.")
+    [@d, @e, @f, @g].each {|d| d.normalize! }
+  end
+  
+  def test_initialize
+    l = Lsi.new([@d, @e, @f, @g])
+  end
+
+  def test_perform_svd
+    l = Lsi.new([@d, @e, @f, @g])
+    l.perform_svd(1.0)
+    assert l.instance_variable_get("@t")
+    assert l.instance_variable_get("@d")
+    assert l.instance_variable_get("@s")
+    assert l.instance_variable_get("@t") * l.instance_variable_get("@s") * l.instance_variable_get("@d")
+    l = Lsi.new([@d, @e, @f])
+    l.perform_svd(0.1)
+    assert l.instance_variable_get("@t")
+    assert l.instance_variable_get("@d")
+    assert l.instance_variable_get("@s")
+    assert l.instance_variable_get("@t") * l.instance_variable_get("@s") * l.instance_variable_get("@d")
+  end
+  
+  def test_add_document
+    l = Lsi.new([@d, @e, @f, @g])
+    l << @d
+    l.perform_svd(0.75)
+  end
+
+  def test_cluster_documents
+    l = Lsi.new([@d, @e, @f, @g])
+    puts l.cluster_documents(2).collect {|c| c.collect {|d| d.object } }.inspect
+    assert_equal 2, l.cluster_documents(2).size
+    assert_equal 2, l.cluster_documents(2,:algorithm => :hierarchical).size
+    assert_equal 2, l.cluster_documents(2, :algorithm => :bisecting_kmeans).size
+  end
+
+  def test_search
+    l = Lsi.new([@d, @e, @f, @g])
+    assert l.search(@f).size >= 1
+  end
+end

data/tests/similarity_test.rb

@@ -0,0 +1,62 @@
+#--
+###Copyright (c) 2006 Surendra K Singhi <ssinghi AT kreeti DOT com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require 'test/unit'
+require 'clusterer'
+
+class TestSimilarity < Test::Unit::TestCase
+  include Clusterer
+
+  def setup
+    @d = Document.new("hello world, mea culpa, goodbye world.")
+    @e = Document.new("the world is not a bad place to live.").normalize!
+    @f = Document.new("the world is a crazy place to live.").normalize!
+    @g = Document.new("unique document.")
+
+    @c1 = Clusterer::Cluster.new([@d, @e, @f])
+    @c2 = Clusterer::Cluster.new([@g, @g])
+  end
+
+  def test_cosine_similarity
+    assert_in_delta 1.0, @d.cosine_similarity(@d), 0.01
+    assert_in_delta 1.0, @e.cosine_similarity(@e), 0.01
+    assert_in_delta 0.0, @f.cosine_similarity(@g), 0.01
+    assert @e.cosine_similarity(@f) > 0.5  # very similar
+  end
+
+  def test_intra_cluster_similarity
+    assert @c1.intra_cluster_similarity(@c2) < 0
+    assert_in_delta 0.0, @c1.intra_cluster_similarity(@c1), 0.01
+  end
+
+  def test_centroid_similarity
+    assert_in_delta 0.0, @c1.centroid_similarity(@c2), 0.01
+    assert_in_delta 1.0, @c1.centroid_similarity(@c1), 0.01
+  end
+
+  def test_upgma
+    assert_in_delta 0, @c1.upgma(@c2), 0.01
+    assert @c1.upgma(@c1) > 0.5
+  end
+end