sclust 1.0.0 → 2.1.0

data/lib/sclust/util/doccol.rb

@@ -0,0 +1,187 @@
+# 
+# The MIT License
+# 
+# Copyright (c) 2010 Samuel R. Baskinger
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# 
+
+require 'rubygems'
+require 'log4r'
+
+require 'sclust/util/sparse_vector'
+
+module SClust
+    module Util
+        
+        
+        class DocumentCollection 
+        
+            # terms - a hash were they keys are the terms in the documents and the values stored are the number of documents contiaining the term.
+            attr_reader :terms
+        
+            # A list of documents
+            attr_reader :doclist
+            
+            # Log4r::Logger for this document collection.
+            attr_reader :logger
+            
+            def initialize()
+                @logger = Log4r::Logger.new(self.class.to_s)
+                @logger.add('default')
+                @terms   = SClust::Util::SparseVector.new(0)
+                @doclist = []
+            end
+        
+            # Add a document to the collection and adjust the @terms attribute to store any new terms in the document.
+            # The document is also added to the @doclist attribute.
+            def <<(d)
+                
+                seen_terms = {}
+                
+                d.each_term { |term, frequency| seen_terms[term] = 1 }
+                
+                if ( seen_terms.size > 0 )
+                
+                    seen_terms.each_key { |term| @terms[term] += 1 }
+                    
+                    @doclist<<d
+                    
+                    #@logger.info("There are #{@doclist.size} documents and #{@terms.size} terms.")
+                end
+                
+                self
+            end
+            
+            # The sum of the terms divided by the documents. If the document only has 1-gram terms, then this
+            # number will always be less than the number of words per document. If, however, you enable
+            # 2-grams, 3-grams, etc in a document, this value will not corrolate perfectly with the word count.
+            def average_terms_per_document()
+                @terms.reduce(0.0) { |count, keyval_pair| count + keyval_pair[1] } / @doclist.size
+            end
+
+            # Number of words that make up a document. Words are no unique like terms are.
+            # Two occurences of the word "the" are a single term "the". Get it? :) Great. One caveate is that
+            # a "term" is typically a 1-gram, that is 1 word is 1 term. It is possible for a term to be constructed
+            # of two or more words (an 2-gram, 3-gram, ... n-gram) in which case this relationship will vary
+            # widely.
+            def average_words_per_document()
+                @doclist.reduce(0.0) { |count, doc| count + doc.words.size } / @doclist.size
+            end
+            
+            # Return the size of the document list.
+            def document_count()
+                @doclist.size
+            end
+            
+            # Sum all words
+            def word_count()
+                @doclist.reduce(0) { |count, doc| count+doc.words.size }
+            end
+            
+            # Return the size of the term vector
+            def term_count()
+                @terms.size
+            end
+
+            
+            def drop_terms(min_frequency=0.10, max_frequency=0.80)
+                
+                min_docs = @doclist.length * min_frequency
+                max_docs = @doclist.length * max_frequency
+                
+                @logger.info("Analyzing #{@terms.length} terms for removal.")
+                @logger.info("Upper/lower boundary are #{max_frequency}/#{min_frequency}% document frequency or #{max_docs}/#{min_docs} documents.")
+                
+                remove_list = []
+                
+                @terms.each do |term, frequency|
+                                
+                    if ( frequency < min_docs or frequency > max_docs )
+                        @logger.info("Removing term #{term} occuring in #{frequency} documents out of #{@doclist.length}")
+                        @terms.delete(term)
+                        remove_list << term
+                    end
+                end
+                
+                @logger.info("Removed #{remove_list.length} of #{@terms.length + remove_list.length} terms. Updating #{doclist.length} documents.")
+                
+                @doclist.each do |doc|
+                    remove_list.each do |term|
+                        doc.terms.delete(term)
+                    end
+                end
+            end
+        
+            def inverse_document_frequency(term)
+                Math.log( @doclist.length / @terms[term] )
+            end
+        
+            alias idf inverse_document_frequency
+        
+            def each_term(&c)
+                @terms.each_key { |k| yield k }
+            end
+            
+            # Filter out documents that are not in the given range 
+            # of document frequency as expressed as a percentage of the total 
+            # number of documents in the collection. If floats are passed, then they are treated as 
+            # percentages. If integers are passed, they are treated like docuent counts.
+            def filter_df(min=1, max=0.20)
+                
+                delete_list = []
+                delete_hash = {}
+                
+                mindocs = ( min.is_a?(Integer) ) ? min : ( min * @doclist.size )
+                maxdocs = ( max.is_a?(Integer) ) ? max : ( max * @doclist.size )
+                
+                @logger.info("Building term to delete list for range #{mindocs} - #{maxdocs}.")
+                
+                @terms.each { |term, freq| delete_list << term if (freq <= mindocs or freq >= maxdocs ) }
+                
+                @logger.info("Identified #{delete_list.size} terms for removal.")
+                
+                # NOTE: We do a two-phase delete so we can delete from backing documents.
+                
+                delete_list.each do |term|
+                    @logger.debug { "Removing term #{term}."}
+                    @terms.delete(term)
+                    delete_hash[term] = 1
+                end
+                
+                @logger.info("Updating documents.")
+                
+                i=0
+                
+                @doclist.each do |doc|
+                    @logger.debug { "Processing document #{i += 1} / #{@doclist.size}" }
+                    
+                    doc.delete_term_if { |term| delete_hash.member?(term) }
+                end
+                
+                @logger.info("Deleting documents that now have no terms left in them. #{@doclist.size} documents.")
+                
+                @doclist.delete_if { |doc| doc.terms.size == 0 }
+                
+                @logger.info("Document count now #{@doclist.size} documents.")
+
+            end
+        end
+    end
+end

data/lib/sclust/util/filters.rb

@@ -0,0 +1,210 @@
+# 
+# The MIT License
+# 
+# Copyright (c) 2010 Samuel R. Baskinger
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# 
+
+require 'rubygems'
+require 'stemmer'
+require 'sclust/util/stopwords'
+require 'nokogiri'
+
+module SClust
+    module Util
+    
+        class Filter
+            class StemmedWord 
+                
+                attr_reader :original_word, :stemmed_word
+                attr_writer :original_word, :stemmed_word
+                
+                def initialize(stemmed_word, original_word)
+                    #super(stemmed_word)
+                    @stemmed_word = stemmed_word
+                    @original_word = String.new(original_word)
+    
+                end
+                
+                def initialize_copy(s)
+                    super(s)
+                    
+                    if ( stemmed_word.class == "SClust::Filter::StemmedWord" )
+                        @original_word = s.original_word
+                    end
+                end
+                
+                def to_s() 
+                    @stemmed_word 
+                end
+                   
+                def < (sw)
+                    @stemmed_word< sw.stemmed_word
+                end
+                
+                def < (sw)
+                    @stemmed_word> sw.stemmed_word
+                end
+                def ==(sw)
+                    @stemmed_word == sw.stemmed_word
+                end
+                
+                def <=>(sw)
+                    @stemmed_word <=> sw.stemmed_word
+                end
+                
+                def +(sw)
+                    if ( sw.nil?)
+                        self
+                    elsif (sw.is_a?(String) )
+                        StemmedWord.new(@stemmed_word + sw, @original_word + sw)
+                    else
+                        StemmedWord.new(@stemmed_word + sw.stemmed_word, @original_word + sw.original_word)
+                    end
+                end
+                
+            end
+            
+            def initialize(prev=nil)
+                @previous_filters = (prev)? [ prev ] : []
+                @succeeding_filters = []
+            end
+            
+            def apply(term)
+                
+                if ( term )
+                    
+                    catch(:filtered_term) do
+                        @previous_filters.each { |f| term = f.filter(term) ; throw :filtered_term if term.nil? }
+                        
+                        term = filter(term) ; throw :filtered_term if term.nil?
+                        
+                        @succeeding_filters.each { |f| term = f.filter(term) ; throw :filtered_term if term.nil? }
+                    end
+                end
+                
+                term
+            end
+            
+            def after(filter)
+                @previous_filters << filter
+                self
+            end
+            
+            def before(filter)
+                @succeeding_filters << filter
+                self
+            end
+            
+            def filter(term)
+                raise Exception.new("Method \"filter\" must be overridden by child classes to implement the specific filter.")
+            end
+        end
+        
+        # Similar to StemFilter, but this will wrap the word in a Filter::StemmedWord object.
+        class StemmedWordFilter < Filter
+            def filter(term)
+                Filter::StemmedWord.new(term.stem, term)
+            end
+        end
+        
+        class StemFilter < Filter
+            def filter(term)
+                term.stem
+            end
+        end
+        
+        class LowercaseFilter < Filter
+            def filter(term)
+                term.downcase
+            end
+        end
+        
+        class StopwordFilter < Filter
+            
+            include SClust::Util::StopwordList
+    
+            filter = LowercaseFilter.new()
+            
+            @@stopwords = {}
+            
+            @@stopword_list.each { |term| @@stopwords[filter.apply(term)] = true }
+    
+            def filter(term)
+                ( @@stopwords[term] ) ? nil : term
+            end
+        end
+        
+        class TrimWhitespace < Filter
+            def filter(term)
+                term.chomp.sub(/^\s*/, '').sub(/\s*$/, '')
+            end
+        end
+        
+        
+        class TokenizerFilter < Filter
+            def filter(document)
+                document.split(/[\s,\.\t!\?\(\)\{\}\[\]\t\r\n";':]+/m)
+            end
+        end
+        
+        class HTMLFilter < Filter
+            def filter(doc)
+                Nokogiri::HTML::DocumentFragment.parse(doc).text
+            end
+        end
+        
+        # A tokenizer that applies a few overall document filters.
+        class DocumentTokenizer < TokenizerFilter
+            def initialize()
+                super()
+                after(HTMLFilter.new())
+            end
+        end
+        
+        # Filters a document term
+        class DocumentTermFilter < Filter
+    
+            def initialize()
+                super()
+                after(LowercaseFilter.new())
+                after(StopwordFilter.new())
+                after(TrimWhitespace.new())
+                #after(StemFilter.new())
+            end
+            
+            # Return nil if the term should be excluded. Otherwise the version of the term 
+            # that should be included is returned.
+            def filter(term)
+                if ( term =~ /^[\d\.]+$/ )
+                    nil
+                else
+                    term
+                end
+            end
+        end
+    
+        class NullFilter < Filter
+            def filter(term)
+                term
+            end
+        end
+    end
+end

data/lib/sclust/util/rss.rb

@@ -0,0 +1,96 @@
+# 
+# The MIT License
+# 
+# Copyright (c) 2010 Samuel R. Baskinger
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# 
+
+require 'rubygems'
+require 'rss'
+
+module SClust
+    
+    # NOTE: RSS collides with the module ::RSS, so we use the :: prefix when accessing the ::RSS module 
+    # that ships with Ruby. :)
+    module RSS
+        def self.rss_to_documents(rss, &addNewDoc)
+            
+            $logger.debug("Operating on #{rss} of type #{rss.class}")
+            
+            # This block builds an RSS::Element (document).
+            unless (rss.instance_of?(::RSS::Element))
+                
+                # Check if we have a URI string...
+                if ( rss.instance_of?(String) )
+                    begin
+                        rss = URI.parse(rss)
+                    rescue URI::InvalidURIError => e
+                        $logger.warning("Exception parsing URI: #{e.message}")
+                    end
+                end
+                
+                $logger.debug("Rss is now of type #{rss.class}.")
+        
+                # Parse it...
+                if (rss.instance_of?(URI::HTTP))
+                    begin
+                        #rss = RSS::Parser::parse(Net::HTTP::get(rss), false)
+                        rss = ::RSS::Parser::parse($wwwagent.get_file(rss), false)
+                    rescue Exception => e
+                        $logger.error("Failed to retrieve URL #{rss}: #{e.message}")
+                        throw e
+                    end
+                elsif(rss.instance_of?(String))
+                    rss = ::RSS::Parser::parse(rss, false)
+                elsif(rss.is_a?(File))
+                    rss = ::RSS::Parser::parse(rss, false);
+                else
+                    rss = nil
+                end
+                
+                throw Exception.new("RSS was not a URI string, a URI object, an RSS document, or an RSS document string: #{rss}") unless rss
+            end
+            
+            unless ( rss.nil? || rss.items.nil? )
+            
+                $logger.debug("Adding #{rss.items.size} to document collection.")
+            
+                # Add this documents of this item to the document collection.
+                rss.items.each do |item|
+                    
+                    if ( item.instance_of?(::RSS::Rss::Channel::Item))
+                        
+                        addNewDoc.call(item.title, item.description, item) if ( item.description )
+                        
+                    elsif ( item.instance_of?(::RSS::RDF::Item) )
+                        
+                        addNewDoc.call(item.title, item.content_encoded, item)
+        
+                    else
+                        
+                        addNewDoc.call(item.title.content, item.content.content, item)
+        
+                    end
+                end
+            end
+        end
+    end
+    
+end