sclust 1.0.0 → 2.1.0

data/lib/sclust/util/weightedmovingaverage.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/ruby
+
+
+module SClust
+    module Util
+        class WeightedMovingAverage
+            
+            attr_reader :weight, :value
+            attr_writer :weight, :value
+            
+            def initialize(weight, initial_value = 0.0)
+                
+                raise Exception.new("Weight was #{weight} but must be between 0.0 and 1.0.") if ( weight > 1 or weight < 0)
+                
+                @weight = weight
+                @weight_compliment = 1.0-weight
+                @value  = initial_value
+            end
+            
+            def adjust(value)
+                @value = ( @weight_compliment*@value ) + ( @weight * value )
+            end
+        end
+    end
+end

data/lib/sclust/util/word.rb

@@ -0,0 +1,53 @@
+# 
+# The MIT License
+# 
+# Copyright (c) 2010 Samuel R. Baskinger
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# 
+
+
+module SClust
+    module Util
+        class Word
+        
+            attr_reader :word, :weight, :data
+            attr_writer :word, :weight, :data
+            
+            def initialize(word="", weight=0.0, other_data={})
+                @word = word
+                @weight = weight
+                @data = other_data
+            end
+            
+            # Return @word.
+            def to_s
+                @word
+            end
+            
+            def hash
+                @word.hash
+            end
+            
+            def eql?(w)
+                @word.eql?(w)
+            end
+        end
+    end
+end

data/tests/clustertest.rb

@@ -1,50 +1,77 @@
+# 
+# The MIT License
+# 
+# Copyright (c) 2010 Samuel R. Baskinger
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# 
+
 require 'test/unit'
 
-require 'sclust/doccluster'
+require 'sclust/util/doccol'
+require 'sclust/kmean/doccluster'
+require 'sclust/util/filters'
+
+Log4r::StderrOutputter.new('default')
+Log4r::Outputter['default'].formatter = Log4r::PatternFormatter.new( :pattern => '%d %C: %m' , :date_pattern => '[%Y-%m-%d-%H:%M:%S %Z]')
+Log4r::Logger.root.level = Log4r::DEBUG
+Log4r::Logger.root.add( 'default' )
+
+require 'sclust/util/doc'
+
+
+#$logger = Log4r::Logger.new($0)
+#$logger.add('default')
+#$logger.info("Starting")
+
 
 class ClusterTest < Test::Unit::TestCase
     
     def setup()
-        @dc = SClust::DocumentCollection.new()
-        filter = SClust::NullFilter.new()
-        d1 = SClust::Document.new("a b c d d e a q a b", :filter=>filter, :ngrams=>[1]) 
-        d2 = SClust::Document.new("a b d e a", :filter=>filter, :ngrams=>[1])
-        d3 = SClust::Document.new("bob", :filter=>filter, :ngrams=>[1])
-        d4 = SClust::Document.new("frank a", :filter=>filter, :ngrams=>[1])
-    
-        @dc + d1
-        @dc + d2
-        @dc + d3
-        @dc + d4
     end
     
     def teardown()
     end
     
     def test_makecluster()
-        c = SClust::DocumentClusterer.new(@dc)
+        filter = SClust::Util::NullFilter.new()
+        d1 = SClust::Util::Document.new("a b c d d e a q a b", :filter=>filter, :ngrams=>[1]) 
+        d2 = SClust::Util::Document.new("a b d e a", :filter=>filter, :ngrams=>[1])
+        d3 = SClust::Util::Document.new("bob", :filter=>filter, :ngrams=>[1])
+        d4 = SClust::Util::Document.new("frank a", :filter=>filter, :ngrams=>[1])
+
+        c = SClust::KMean::DocumentClusterer.new()
         
-        c.cluster
+        c << d1
+        c << d2
+        c << d3
+        c << d4
         
+        c.topics = 3
+
+        c.cluster
+
         c.each_cluster do |cl|
-            
-            max = 0
-            
-            0.upto(cl.center.terms.length - 1) do |i|
-                
-                term  = cl.center.terms[i]
-                value = cl.center.values[i]
-                
-                max = i if ( cl.center.values[i] > cl.center.values[max] )    
-            end
-            
-            puts("Cluster: #{cl.center.terms[max]} #{cl.center.values[max]}")
-                        
+            puts('===================================')
             cl.center.get_max_terms(3).each do |t|
                 puts("Got Term: #{t} with value #{cl.center.get_term_value(t)}")
             end
-            
-            assert(cl.center.values[max] == cl.center.get_term_value(cl.center.get_max_terms(1)[0]), "Max value was not found.")
         end
     end
 

data/tests/filters_test.rb

@@ -0,0 +1,48 @@
+# 
+# The MIT License
+# 
+# Copyright (c) 2010 Samuel R. Baskinger
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# 
+
+require 'test/unit'
+
+require 'sclust/util/filters'
+
+class DocTests < Test::Unit::TestCase
+
+  def setup() end
+  def teardown() end
+
+  def test_docfilter()
+    f = SClust::Util::DocumentTermFilter.new()
+
+    assert( f.apply("aba").original_word == "aba", "did not filter out a.")
+  end
+  
+  def test_tokenizer()
+      
+      f = SClust::Util::TokenizerFilter.new()
+      
+      assert(f.apply("hi bye") == [ "hi", "bye" ])
+      assert(f.apply("hi \r\n\n\rbye") == [ "hi", "bye" ])
+  end
+end
+

data/tests/ldatest.rb

@@ -0,0 +1,75 @@
+# 
+# The MIT License
+# 
+# Copyright (c) 2010 Samuel R. Baskinger
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# 
+
+require 'test/unit'
+
+require 'sclust/lda/lda2'
+require 'sclust/util/doc'
+require 'log4r'
+
+Log4r::StderrOutputter.new('default')
+Log4r::Outputter['default'].formatter = Log4r::PatternFormatter.new( :pattern => '%d %C: %m' , :date_pattern => '[%Y-%m-%d-%H:%M:%S %Z]')
+Log4r::Logger.root.level = Log4r::DEBUG
+Log4r::Logger.root.add( 'default' )
+
+
+class DocTests < Test::Unit::TestCase
+
+    def setup() 
+        @null_filter = SClust::Util::NullFilter.new()
+    end
+  
+    def teardown() 
+    end
+
+    def test_lda_001()
+        
+        
+        lda = SClust::LDA2::LDA2.new()
+    
+        lda.topics=4
+    
+        lda << SClust::Util::Document.new("a b 1 z ", :filter => @null_filter)
+        lda << SClust::Util::Document.new("a b 2 5 ", :filter => @null_filter)
+        lda << SClust::Util::Document.new("a b 3 4 ", :filter => @null_filter)
+        lda << SClust::Util::Document.new("a b c d e f g", :filter => @null_filter)
+        lda << SClust::Util::Document.new("d e f z", :filter => @null_filter)
+        lda << SClust::Util::Document.new("g h z", :filter => @null_filter)
+        lda << SClust::Util::Document.new("h i z", :filter => @null_filter)
+        lda << SClust::Util::Document.new("x y 6", :filter => @null_filter)
+        lda << SClust::Util::Document.new("x y 7", :filter => @null_filter)
+        lda << SClust::Util::Document.new("x y 8", :filter => @null_filter)
+
+        lda.lda(:iterations=>100)
+
+        lda.get_max_terms(100).each do |topic|
+            puts("---------- Topic ---------- ")
+          
+            topic.each do |words|
+                puts("\t#{words.weight} - #{words.to_s}")
+            end
+        end
+    end
+end
+

data/tests/sparse_vector_test.rb

@@ -0,0 +1,61 @@
+# 
+# The MIT License
+# 
+# Copyright (c) 2010 Samuel R. Baskinger
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# 
+
+require 'test/unit'
+
+require 'sclust/sparse_vector'
+
+class ClusterTest < Test::Unit::TestCase
+    
+    def setup()
+    end
+    
+    def teardown()
+    end
+    
+    def test_spvec01()
+        sp = SClust::SparseLabeledVector.new(0)
+        
+        sp[5] = 0
+        sp.store(0, 1, "bye")
+        sp.store(2, 0, "hi")
+        
+        assert(sp[0] == 1, "Could not define value.")
+        
+        assert(sp[1] == 0, "Default value not returned for unknown keys.")
+        
+        assert(sp.length == 1, "Data size was #{sp.length} instead of 1. Assigning default value may have accidentally stored the default value.")
+
+        assert(sp.key_map[0] == "bye", "Could not find map from key 0 to label \"bye\"")        
+
+        assert(sp.label_map["bye"] == 0, "Could not find map from label \"bye\" to key 0")
+
+        sp.delete(0)
+        sp.delete(1)
+        
+        assert(sp[0] == 0, "Default value not returned for deleted key.")
+        
+    end
+
+end

data/tests/test001.rb

@@ -1,5 +1,30 @@
-require 'sclust/doc'
-require 'sclust/doccol'
+# 
+# The MIT License
+# 
+# Copyright (c) 2010 Samuel R. Baskinger
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# 
+
+require 'sclust/util/doc'
+require 'sclust/util/doccol'
+require 'sclust/util/filters'
 require 'test/unit'
 
 
@@ -12,11 +37,11 @@ class DocTests < Test::Unit::TestCase
   #end
 
   def test_builddoc
-    d = SClust::Document.new("hi, this is a nice doc! Yup. Oh? A very nice doc, indeed.")
+    d = SClust::Util::Document.new("hi, this is a nice doc! Yup. Oh? A very nice doc, indeed.")
 
     d.terms.each do |k,v| 
-      assert(k != ".", "Period found")
-      assert(k != "", "Empty term found")
+      assert(k.original_word != ".", "Period found")
+      assert(k.original_word != "", "Empty term found")
       #puts("#{k}=#{v}")
     end 
 
@@ -27,23 +52,28 @@ end
 class DocCollectionTests < Test::Unit::TestCase
 
   def test_collectionadd()
-    dc = SClust::DocumentCollection.new()
-    d1 = SClust::Document.new("a b c d d e a q a b") 
-    d2 = SClust::Document.new("a b d e a")
-    d3 = SClust::Document.new("bob")
-    d4 = SClust::Document.new("frank a")
+    filter = SClust::Util::NullFilter.new()
+    dc = SClust::KMean::DocumentCollection.new()
+    d1 = SClust::Util::Document.new("a b c d d e a q a b", :filter=>filter, :ngrams => [1]) 
+    d2 = SClust::Util::Document.new("a b d e a", :filter=>filter, :ngrams => [1])
+    d3 = SClust::Util::Document.new("bob", :filter=>filter, :ngrams => [1])
+    d4 = SClust::Util::Document.new("frank a", :filter=>filter, :ngrams => [1])
 
-    dc + d1
-    dc + d2
-    dc + d3
-    dc + d4
+    dc << d1
+    dc << d2
+    dc << d3
+    dc << d4
 
     dc.terms.each do |k,v|
-      if k == "a"
-        assert(v == 3, "A appers in 3 documents out of 4.")
-        assert(dc.idf("a") > 2.2, "Known value for a")
-        assert(dc.idf("a") < 2.3, "Known value for a")
-      end
+	if k == "a"
+	    assert(v == 6, "A appers in #{v} documents out of 4.")
+	    assert(dc.idf("a") > 0.2, "Known value for a")
+	    assert(dc.idf("a") < 0.3, "Known value for a")
+	end
     end
+
+    print("TERMS: ")
+    d1.words.each { |w| print "#{w}, " }
+    assert(d1.tf('a') * d1.words.size == 3)
   end
 end