classifier 1.1.1 → 1.2.0

data/lib/classifier/extensions/vector_serialize.rb

@@ -0,0 +1,14 @@
+module GSL
+  
+  class Vector
+    def _dump(v)
+      Marshal.dump( self.to_a )
+    end
+    
+    def self._load(arr)
+      arry = Marshal.load(arr)
+      return GSL::Vector.new(arry)
+    end
+    
+  end
+end

data/lib/classifier/extensions/word_hash.rb

@@ -0,0 +1,125 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+# These are extensions to the String class to provide convenience 
+# methods for the Classifier package.
+class String
+  
+  # Removes common punctuation symbols, returning a new string. 
+  # E.g.,
+  #   "Hello (greeting's), with {braces} < >...?".without_punctuation
+  #   => "Hello  greetings   with  braces         "
+  def without_punctuation
+    tr( ',?.!;:"@#$%^&*()_=+[]{}\|<>/`~', " " ) .tr( "'\-", "")
+  end
+  
+  # Return a Hash of strings => ints. Each word in the string is stemmed,
+  # interned, and indexes to its frequency in the document.  
+	def word_hash
+		word_hash_for_words(gsub(/[^\w\s]/,"").split + gsub(/[\w]/," ").split)
+	end
+
+	# Return a word hash without extra punctuation or short symbols, just stemmed words
+	def clean_word_hash
+		word_hash_for_words gsub(/[^\w\s]/,"").split
+	end
+	
+	private
+	
+	def word_hash_for_words(words)
+		d = Hash.new
+		words.each do |word|
+			word.downcase! if word =~ /[\w]+/
+			key = word.stem.intern
+			if word =~ /[^\w]/ || ! CORPUS_SKIP_WORDS.include?(word) && word.length > 2
+				d[key] ||= 0
+				d[key] += 1
+			end
+		end
+		return d
+	end
+	
+	CORPUS_SKIP_WORDS = [
+      "a",
+      "again",
+      "all",
+      "along",
+      "are",
+      "also",
+      "an",
+      "and",
+      "as",
+      "at",
+      "but",
+      "by",
+      "came",
+      "can",
+      "cant",
+      "couldnt",
+      "did",
+      "didn",
+      "didnt",
+      "do",
+      "doesnt",
+      "dont",
+      "ever",
+      "first",
+      "from",
+      "have",
+      "her",
+      "here",
+      "him",
+      "how",
+      "i",
+      "if",
+      "in",
+      "into",
+      "is",
+      "isnt",
+      "it",
+      "itll",
+      "just",
+      "last",
+      "least",
+      "like",
+      "most",
+      "my",
+      "new",
+      "no",
+      "not",
+      "now",
+      "of",
+      "on",
+      "or",
+      "should",
+      "sinc",
+      "so",
+      "some",
+      "th",
+      "than",
+      "this",
+      "that",
+      "the",
+      "their",
+      "then",
+      "those",
+      "to",
+      "told",
+      "too",
+      "true",
+      "try",
+      "until",
+      "url",
+      "us",
+      "were",
+      "when",
+      "whether",
+      "while",
+      "with",
+      "within",
+      "yes",
+      "you",
+      "youll",
+      ]
+end

data/lib/classifier/extensions/word_list.rb

@@ -0,0 +1,31 @@
+# Author::    David Fayram  (mailto:dfayram@lensmen.net)
+# Copyright:: Copyright (c) 2005 David Fayram II
+# License::   GPL
+
+module Classifier  
+  # This class keeps a word => index mapping. It is used to map stemmed words
+  # to dimensions of a vector.
+  class WordList
+    def initialize
+      @location_table = {}
+    end
+    
+    # Adds a word (if it is new) and assigns it a unique dimension.
+    def add_word(word)
+      term = word
+      @location_table[term] = @location_table.size unless @location_table[term]
+    end
+    
+    # Returns the dimension of the word or nil if the word is not in the space.
+    def [](lookup)
+      term = lookup
+      @location_table[term]
+    end
+    
+    # Returns the number of words mapped.
+    def size
+      @location_table.size
+    end
+    
+  end
+end

data/lib/classifier/lsi.rb

@@ -0,0 +1,248 @@
+# Author::    David Fayram  (mailto:dfayram@lensmen.net)
+# Copyright:: Copyright (c) 2005 David Fayram II
+# License::   GPL
+
+begin
+
+require 'gsl' # requires http://rb-gsl.rubyforge.org/
+
+require 'classifier/extensions/word_list'
+require 'classifier/extensions/vector_serialize'
+require 'classifier/lsi/content_node'
+
+module Classifier
+  
+  # This class implements a Latent Semantic Indexer, which can search, classify and cluster
+  # data based on underlying semantic relations. For more information on the algorithms used,
+  # please consult Wikipedia[http://en.wikipedia.org/wiki/Latent_Semantic_Indexing].
+  class LSI
+    
+    attr_reader :word_list
+    
+    # Create a fresh index.
+    # If you want to call #build_index manually, use
+    #      Classifier::LSI.new :auto_rebuild => false
+    #
+    def initialize(options = {})
+      @auto_rebuild = true unless options[:auto_rebuild] == false
+      @word_list, @items = WordList.new, {}
+      @version, @built_at_version = 0, -1
+    end
+    
+    # Returns true if the index needs to be rebuilt.  The index needs
+    # to be built after all informaton is added, but before you start
+    # using it for search, classification and cluster detection.
+    def needs_rebuild?
+      @version != @built_at_version
+    end
+    
+    # Adds an item to the index. item is assumed to be a string, but 
+    # any item may be indexed so long as it responds to #to_s or if
+    # you provide an optional block explaining how the indexer can 
+    # fetch fresh string data. This optional block is passed the item,
+    # so the item may only be a reference to a URL or file name.
+    # 
+    # For example:
+    #   lsi = Classifier::LSI.new
+    #   lsi.add_item "This is just plain text"
+    #   lsi.add_item "/home/me/filename.txt" { |x| File.read x }
+    #   ar = ActiveRecordObject.find( :all )
+    #   lsi.add_item ar, *ar.categories { |x| ar.content }
+    #
+    def add_item( item, *categories, &block )
+      @items[item] = ContentNode.new(item, categories, block)
+      @version += 1
+      build_index if @auto_rebuild
+    end
+
+    # A less flexible shorthand for add_item that assumes 
+    # you are passing in a string with no categorries. item
+    # will be duck typed via to_s . 
+    #
+    def <<( item )
+      add_item item
+    end
+    
+    # Removes an item from the database, if it is indexed. 
+    #
+    def remove_item( item )
+      if @items.keys.contain? item
+        @items.remove item
+        @version += 1
+      end
+    end
+    
+    # Returns an array of items that are indexed. 
+    def items
+      @items.keys
+    end
+    
+    # This function rebuilds the index if needs_rebuild? returns true.
+    # For very large document spaces, this indexing operation may take some
+    # time to complete, so it may be wise to place the operation in another 
+    # thread. 
+    #
+    # As a rule, indexing will be fairly swift on modern machines until
+    # you have well over 500 documents indexed, or have an incredibly diverse 
+    # vocabulary for your documents. 
+    #
+    # The optional parameter "cutoff" is a tuning parameter. When the index is
+    # built, a certain number of s-values are discarded from the system. The 
+    # cutoff parameter tells the indexer how many of these values to keep.
+    # A value of 1 for cutoff means that no semantic analysis will take place,
+    # turning the LSI class into a simple vector search engine.
+    def build_index( cutoff=0.75 )
+      return unless needs_rebuild?
+      make_word_list
+      
+      doc_list = @items.values
+      tda = doc_list.collect { |node| node.raw_vector_with( @word_list ) }
+      tdm = GSL::Matrix.new( *tda ).trans
+      ntdm = build_reduced_matrix(tdm, cutoff)
+      
+      ntdm.size[1].times do |col| 
+        vec = GSL::Vector.new( ntdm.column(col) ).row
+        doc_list[col].lsi_vector = vec
+        doc_list[col].lsi_norm = vec.normalize
+      end
+      
+      @built_at_version = @version
+    end
+    
+    # This function is the primitive that find_related and classify 
+    # build upon. It returns an array of 2-element arrays. The first element
+    # of this array is a document, and the second is its "score", defining
+    # how "close" it is to other indexed items.
+    # 
+    # These values are somewhat arbitrary, having to do with the vector space
+    # created by your content, so the magnitude is interpretable but not always
+    # meaningful between indexes. 
+    #
+    # The parameter doc is the content to compare. If that content is not
+    # indexed, you can pass an optional block to define how to create the 
+    # text data. See add_item for examples of how this works. 
+    def proximity_array_for_content( doc, &block )
+      return [] if needs_rebuild?
+  
+      content_node = node_for_content( doc, &block )
+      result = 
+        @items.keys.collect do |item|
+          val = content_node.search_vector * @items[item].search_vector.col
+          [item, val]
+        end
+      result.sort_by { |x| x[1] }.reverse
+    end 
+    
+    # Similar to proximity_array_for_content, this function takes similar
+    # arguments and returns a similar array. However, it uses the normalized
+    # calculated vectors instead of their full versions. This is useful when 
+    # you're trying to perform operations on content that is much smaller than
+    # the text you're working with. search uses this primitive.
+    def proximity_norms_for_content( doc, &block )
+      return [] if needs_rebuild?
+  
+      content_node = node_for_content( doc, &block )
+      result = 
+        @items.keys.collect do |item|
+          val = content_node.search_norm * @items[item].search_norm.col
+          [item, val]
+        end
+      result.sort_by { |x| x[1] }.reverse
+    end 
+    
+    # This function allows for text-based search of your index. Unlike other functions
+    # like find_related and classify, search only takes short strings. It will also ignore
+    # factors like repeated words. It is best for short, google-like search terms. 
+    # A search will first priortize lexical relationships, then semantic ones. 
+    #
+    # While this may seem backwards compared to the other functions that LSI supports,
+    # it is actually the same algorithm, just applied on a smaller document.
+    def search( string, max_nearest=3 )
+      return [] if needs_rebuild?
+      
+      carry = 
+        proximity_norms_for_content( string )
+      result = carry.collect { |x| x[0] }
+      return result[0..max_nearest-1]
+    end
+    
+    # This function takes content and finds other documents
+    # that are semantically "close", returning an array of documents sorted
+    # from most to least relavant.
+    # max_nearest specifies the number of documents to return. A value of 
+    # 0 means that it returns all the indexed documents, sorted by relavence. 
+    #
+    # This is particularly useful for identifing clusters in your document space. 
+    # For example you may want to identify several "What's Related" items for weblog
+    # articles, or find paragraphs that relate to each other in an essay.
+    def find_related( doc, max_nearest=3, &block )
+      carry = 
+        proximity_array_for_content( doc, &block ).reject { |pair| pair[0] == doc }
+      result = carry.collect { |x| x[0] }
+      return result[0..max_nearest-1]
+    end
+      
+    # This function uses a voting system to categorize documents, based on 
+    # the categories of other documents. It uses the same logic as the 
+    # find_related function to find related documents, then returns the
+    # most obvious category from this list. 
+    #
+    # cutoff signifies the number of documents to consider when clasifying 
+    # text. A cutoff of 1 means that every document in the index votes on 
+    # what category the document is in. This may not always make sense.
+    #
+    def classify( doc, cutoff=0.30, &block )
+      icutoff = (@items.size * cutoff).round
+      carry = proximity_array_for_content( doc, &block )
+      carry = carry[0..icutoff-1]
+      votes = {}
+      carry.each do |pair|
+        categories = @items[pair[0]].categories
+        categories.each do |category| 
+          votes[category] ||= 0.0
+          votes[category] += pair[1] 
+        end
+      end
+      
+      ranking = votes.keys.sort_by { |x| votes[x] }
+      return ranking[-1]
+    end
+    
+    private
+    def build_reduced_matrix( matrix, cutoff=0.75 )
+      # TODO: Check that M>=N on these dimensions! Transpose helps assure this
+      u, v, s = matrix.SV_decomp
+      # TODO: Better than 75% term, please. :\
+      s_cutoff = s.sort.reverse[(s.size * cutoff).round - 1]
+      s.size.times do |ord|
+        s[ord] = 0.0 if s[ord] < s_cutoff
+      end
+      
+      # Reconstruct the term document matrix, only with reduced rank
+      u * Matrix.diagonal( s ) * v.trans
+    end
+    
+    def node_for_content(item, &block)    
+      if @items[item]
+        return @items[item]
+      else
+        cn = ContentNode.new(item, &block) # make the node and extract the data
+        cn.raw_vector_with( @word_list ) # make the lsi raw and norm vectors
+      end
+
+      cn
+    end
+    
+    def make_word_list
+      @word_list = WordList.new
+      @items.each_value do |node|
+        node.word_hash.each_key { |key| @word_list.add_word key }
+      end
+    end
+
+  end
+end
+
+rescue LoadError
+	$stderr.puts "For LSI support, you need to install http://rb-gsl.rubyforge.org/"
+end

data/lib/classifier/lsi/content_node.rb

@@ -0,0 +1,67 @@
+# Author::    David Fayram  (mailto:dfayram@lensmen.net)
+# Copyright:: Copyright (c) 2005 David Fayram II
+# License::   GPL
+
+module Classifier
+
+
+# This is an internal data structure class for the LSI node. Save for 
+# raw_vector_with, it should be fairly straightforward to understand.
+# You should never have to use it directly.
+  class ContentNode
+    attr_accessor :word_hash, :raw_vector, :raw_norm, 
+                              :lsi_vector, :lsi_norm,
+                  :categories
+    attr_reader :source
+   
+    # If text_proc is not specified, the source will be duck-typed
+    # via source.to_s
+    def initialize( source, categories=nil, text_proc=nil )
+      text_proc = text_proc || (proc {|x| x.to_s})
+      @categories = categories || []
+      @source = source
+      @word_hash = text_proc.call( @source ).clean_word_hash
+    end
+   
+    # Use this to fetch the appropriate search vector.
+    def search_vector
+      @lsi_vector || @raw_vector
+    end
+    
+    # Use this to fetch the appropriate search vector in normalized form.
+    def search_norm
+      @lsi_norm || @raw_norm
+    end
+   
+    # Creates the raw vector out of word_hash using word_list as the
+    # key for mapping the vector space.
+    def raw_vector_with( word_list )
+      vec = Array.new(word_list.size, 0)
+
+      @word_hash.each_key do |word|
+        vec[word_list[word]] = @word_hash[word] if word_list[word]
+      end
+     
+      # Perform the scaling transform
+      total_words = vec.inject(0) { |sum,term| sum += term  }.to_f
+      
+      # Perform first-order association transform if this vector has more
+      # than one word in it. 
+      if total_words > 1.0 
+        weighted_total = vec.inject(0.0) do |sum,term|
+          if( term > 0 )
+            sum += (( term / total_words ) * Math.log( term / total_words ))
+          else
+            sum
+          end
+        end 
+        vec.map! { |val| Math.log( val + 1 ) / -weighted_total }
+      end
+
+      @raw_norm   = GSL::Vector.new( vec ).normalize
+      @raw_vector = GSL::Vector.new( vec )
+    end   
+  
+  end  
+  
+end