sclust 1.0.0 → 2.1.0

metadata

@@ -1,38 +1,58 @@
 --- !ruby/object:Gem::Specification 
 name: sclust
 version: !ruby/object:Gem::Version 
-  version: 1.0.0
+  version: 2.1.0
 platform: ruby
 authors: 
-  - Sam Baskinger
+- Sam Baskinger
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2009-12-01 00:00:00 -06:00
+date: 2010-10-02 00:00:00 -05:00
 default_executable: 
 dependencies: 
-  - !ruby/object:Gem::Dependency 
-    name: log4r
-    type: :runtime
-    version_requirement: 
-    version_requirements: !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            version: 1.0.5
-      version: 
-  - !ruby/object:Gem::Dependency 
-    name: mechanize
-    type: :runtime
-    version_requirement: 
-    version_requirements: !ruby/object:Gem::Requirement 
-      requirements: 
-        - - ">="
-          - !ruby/object:Gem::Version 
-            version: 0.9.3
-      version: 
-description: A k-mean text clustering library for ruby.
+- !ruby/object:Gem::Dependency 
+  name: log4r
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.5
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: mechanize
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: stemmer
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.4.1
+    version: 
+description: A k-mean and LDA text clustering library for ruby.
 email: basking2@rubyforge.org.com
 executables: []
 
@@ -41,10 +61,18 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
-  - lib/sclust/cluster.rb
-  - lib/sclust/doc.rb
-  - lib/sclust/doccluster.rb
-  - lib/sclust/doccol.rb
+- lib/sclust/kmean/doccluster.rb
+- lib/sclust/kmean/cluster.rb
+- lib/sclust/lda/lda2.rb
+- lib/sclust/lda/lda.rb
+- lib/sclust/util/weightedmovingaverage.rb
+- lib/sclust/util/doc.rb
+- lib/sclust/util/sparse_vector.rb
+- lib/sclust/util/rss.rb
+- lib/sclust/util/word.rb
+- lib/sclust/util/filters.rb
+- lib/sclust/util/doccol.rb
+- lib/sclust/util/stopwords.rb
 has_rdoc: true
 homepage: http://sclust.rubyforge.org
 licenses: []
@@ -53,18 +81,18 @@ post_install_message:
 rdoc_options: []
 
 require_paths: 
-  - lib
+- lib
 required_ruby_version: !ruby/object:Gem::Requirement 
   requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.6.8
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.6.8
   version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
   version: 
 requirements: []
 
@@ -72,9 +100,15 @@ rubyforge_project: http://sclust.rubyforge.org/
 rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
-summary: k-mean clustering.
+summary: k-mean/lda clustering.
 test_files: 
-  - tests/clustertest.rb
-  - tests/test001.rb
-  - tests/clustertest.rb
-  - tests/test001.rb
+- tests/filters_test.rb
+- tests/test001.rb
+- tests/ldatest.rb
+- tests/clustertest.rb
+- tests/sparse_vector_test.rb
+- tests/filters_test.rb
+- tests/test001.rb
+- tests/ldatest.rb
+- tests/clustertest.rb
+- tests/sparse_vector_test.rb

data/lib/sclust/cluster.rb

@@ -1,197 +0,0 @@
-require 'rubygems'
-require 'log4r'
-
-module SClust
-
-class CosineDistance
-    
-    # Given two vectors, compute the distance
-    def self.distance(a,b)
-
-        acc1 = 0.0
-        acc2 = 0.0
-        acc3 = 0.0
-        
-        0.upto(a.length-1) do |i| 
-            acc1 += a[i]*b[i] 
-            acc2 *= a[i]*a[i]
-            acc3 *= b[i]*b[i]
-        end
-        
-        1 - ( acc1 / (Math.sqrt(acc2) * Math.sqrt(acc3)) )
-    end
-end
-
-class ClusterPoint
-
-    attr_reader :terms, :values, :cluster, :source_object
-    attr_writer :cluster, :source_object
-
-    # Initialize the ClusterPoint with a list of terms (labels, objects, whatever) and numeric values.
-    def initialize(terms, values, source_object = nil)
-      @terms   = terms
-      @values  = values
-      @cluster = nil
-      @source_object = source_object
-    end
-
-    def distance(clusterPoint)
-        CosineDistance.distance(@values, clusterPoint.values)
-    end
-
-    # Add each item in the cluster point to this cluster point adjusting the values per the given weight.
-    # Weght is a value from 0.0 - 1.0, inclusive. A value of 1 means that this clusterPoint is 100% assigned to
-    # this cluster point while a weight value of 0 will have no effect.
-    def add(clusterPoint, weight)
-        0.upto(@values.length-1) { |i| @values[i] = ( @values[i] * (1-weight)) + (clusterPoint.values[i] * weight) }
-        
-        # Validation code
-        #0.upto(@values.length-1) do |i|
-        #    if ( @values[i].nan? || ! @values[i].finite? ) 
-        #        throw Exception.new("Cluster has invalid number #{@values[i]}")
-        #    end
-        #end
-    end
-
-  
-    # Similar to add, but subtract.
-    def sub(clusterPoint, weight)
-        0.upto(@values.length-1) { |i| @values[i] = ( @values[i] - (clusterPoint.values[i] * weight) ) / (1 - weight) }
-
-        # Validation code
-        #0.upto(@values.length-1) do |i|
-        #    if ( @values[i].nan? || ! @values[i].finite? ) 
-        #        throw Exception.new("Cluster has invalid number #{@values[i]} w:#{weight} and #{clusterPoint.values[i]}")
-        #    end
-        #end
-    end
-    
-    def get_max_terms(n=3)
-        
-        values = {}
-        
-        @terms.length.times do |i|
-            t = @terms[i]
-            v = @values[i]
-            values[v] = [] unless values.has_key?(v)
-            values[v] << t
-        end
-        
-        vlist = values.keys.sort { |x,y|  ( x > y ) ? -1 : 1 }
-        
-        result = []
-        
-        n = vlist.length if ( n > vlist.length )
-        
-        n.times { |i| result += values[vlist[i]] }
-        
-        result.slice(0,n)
-        
-    end
-    
-    def get_term_value(term)
-        i=0
-        
-        catch(:found) do
-            @terms.each do |t|
-                throw :found if ( t == term )
-                i+=1
-            end
-        end
-        
-        @values[i]
-    end
-        
-end
-
-class Cluster
-
-    attr_reader :center, :size
-
-    def initialize(centerPoint)
-        @fixed      = false
-        @center     = centerPoint.clone
-        @size       = 1
-    end
-  
-    def +(point)
-        point.cluster = self
-        
-        @size+=1
-        
-        @center.add(point, 1.0/@size.to_f)
-    end
-
-    def -(point)
-        point.cluster = nil
-    
-        @center.sub(point, 1.0/@size.to_f)
-
-        @size-=1
-    end
-    
-    def get_max_terms(n=3)
-        @center.get_max_terms(n)
-    end
-    
-end
-
-class Clusterer
-    
-    attr_reader :clusters, :points, :cluster_count, :iterations, :logger
-    attr_writer :clusters, :points, :cluster_count, :iterations
-
-    # Optionally takes a notifier.
-    def initialize(points)
-        @iterations    = 2
-        @cluster_count = 10
-        @points        = points
-        @clusters      = []
-        @logger        = Log4r::Logger.new('Clusterer')
-    
-        # Randomly select a few starting documents.
-        @cluster_count.times { @clusters << Cluster.new(@points[rand(points.length)]) }
-    end
-
-    def +(point)
-        @points << point
-    end
-  
-    def each_cluster(&c)
-        @clusters.each { |cluster| yield cluster }
-    end
-        
-    def assign_all_points
-  
-        @points.each do |pt|
-            
-            @logger.debug("Assigning point #{pt}.")
-      
-            min_cluster = @clusters[0]
-            min_dst = min_cluster.center.distance(pt)
-    
-            @clusters.each do |cluster|
-        
-                tmp_distance = cluster.center.distance(pt)
-        
-                if ( tmp_distance < min_dst )
-                    min_cluster = cluster
-                    min_dst = tmp_distance
-                end
-            end
-
-            pt.cluster - pt if pt.cluster
-        
-            min_cluster + pt
-        end
-    end
-  
-  def cluster
-      iterations.times do |i|
-          @logger.info("Starting iteration #{i+1} of #{iterations}.")
-          assign_all_points
-      end
-  end
-end
-
-end

data/lib/sclust/doc.rb

@@ -1,92 +0,0 @@
-module SClust
-    
-# Filters a document term
-class DocumentTermFilter
-    # Return nil if the term should be excluded. Otherwise the version of the term 
-    # that should be included is returned.
-    def filter(term)
-        if ( term.nil? )
-            nil
-        elsif (term.size < 2)
-            nil
-        elsif ( term =~ /^[\d\.]+$/ )
-            nil
-        else
-            term.downcase!
-        end
-    end
-end
-
-class NullFilter
-    def filter(term)
-        term
-    end
-end
-
-class Document
-
-    attr_reader :terms, :userDate, :filter
-
-    # Takes { :userData, :ngrams => [1,2,3], :filter }
-    def initialize(text, opts={})
-        
-        @text = text
-        @userData = opts[:userData]
-
-        opts[:ngramrange] ||= [ 1, 2, 3 ]
-        opts[:filter] ||= DocumentTermFilter.new()
-        
-        word_arr = text.split(/[ ,\.\t!\?\(\)\{\}\[\]\t\r\n]+/m)
-    
-        @terms = Hash.new(0)
-        
-        # Array of counts of grams built.
-        builtGramCounts = []
-        
-        # Build a set of n-grams from our requested ngram range.
-        opts[:ngrams].each do |n|
-            
-            builtGramCounts[n] = 0
-
-            # For each word in our list...
-            0.upto(word_arr.length-1) do |j| 
-                
-                if ( n + j < word_arr.length )
-                    
-                    term = word_arr[j]
-
-                    (n-1).times { |ngram| term += " #{word_arr[j+ngram+1]}" }
-                    
-                end
-
-                term = opts[:filter].filter(term)
-                
-                @terms[term] += 1.0 if term
-                
-                builtGramCounts[n] += 1
-
-            end
-            
-        end
-
-        @terms.each { |k,v| @terms[k] /= @terms.length }
-
-    end
-  
-    def term_frequency(term)
-        @terms[term]
-    end
-  
-    alias tf term_frequency
-
-    def each_term(&call) 
-        terms.each_key { |k| yield k }
-    end
-
-    def has_term?(term)
-        @terms.has_key?(term)
-    end
-
-end
-
-end

data/lib/sclust/doccluster.rb

@@ -1,39 +0,0 @@
-require 'sclust/doc'
-require 'sclust/doccol'
-require 'sclust/cluster'
-
-module SClust
-  
-# A document clusterer that overrides the + operator
-# to allow for adding Document objects.
-class DocumentClusterer < Clusterer
-    
-    def initialize(documentCollection)
-        
-        # List of all terms
-        term_list = documentCollection.terms.keys.sort
-        point_list = []
-        
-        documentCollection.doclist.each do |doc|
-            
-            doc_terms       = [] # Sorted list of terms.
-            doc_term_values = [] # Corosponding values.
-        
-            
-            # Buid a BIG term vector list for this document.
-            term_list.each do |term|
-                doc_terms << term
-                doc_term_values << doc.tf(term) - documentCollection.idf(term)
-            end
-            
-            # def initialize(terms, values, source_object = nil)
-            point_list << ClusterPoint.new(doc_terms, doc_term_values, doc)
-        end
-        
-        super(point_list)
-        
-    end
-
-end
-
-end