classifier 1.3.4 → 1.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3c53668ddd328fb78862c67723b185df9c2aa717
-  data.tar.gz: 3655405d082fdd8a01e4ca893a70360ca9f62322
+SHA256:
+  metadata.gz: 1d453b4ca9e0a0c44a2b8d3c9ef55db5b55a17efe5ff0cfbcab93f24965cd536
+  data.tar.gz: 26f5d9595ddd35c8d7c239f946afa8251a4b68f24f5b8bc41e30be13ded60547
 SHA512:
-  metadata.gz: 40b7395e2f04f56bdbabb49a4d0013dba36e9c1325ae66e5bff92451059c5b559677aaea30e50f8f2fbbae58e50bf0f084925ef38e0e3d3fb729e37e357469d4
-  data.tar.gz: 150f8f387706d870a37e86b0418c5e68ad386b82518294bdf21585ab3509fd98515648bc5e06dfb78b97f1e544099fe1da5ddcd69413826e0ccc39780d457940
+  metadata.gz: c13c1666c7d2fe92d47ab7ced0a885fec7b13719aea25f283a013d5015744d6aea7d473706af54042972ba687b214c2a3a619f58d75560d90e57c3569d38f957
+  data.tar.gz: 23261ba6708307ecf6faac636d8572c50720fbf9a2e6db6ee736a07ce3de445daa431afb3bbf2c49dd4d2bd699327ca1c419029b4972236c52a9a5c1f00ab5a2

data/LICENSE

@@ -146,7 +146,7 @@ such a program is covered only if its contents constitute a work based
 on the Library (independent of the use of the Library in a tool for
 writing it).  Whether that is true depends on what the Library does
 and what the program that uses the Library does.
-  
+
   1. You may copy and distribute verbatim copies of the Library's
 complete source code as you receive it, in any medium, provided that
 you conspicuously and appropriately publish on each copy an
@@ -426,4 +426,4 @@ the Free Software Foundation.
   14. If you wish to incorporate parts of the Library into other free
 programs whose distribution conditions are incompatible with these,
 write to the author to ask for permission.  For software which is
-copyrighted by
+copyrighted by

data/lib/classifier.rb

@@ -27,4 +27,4 @@
 require 'rubygems'
 require 'classifier/extensions/string'
 require 'classifier/bayes'
-require 'classifier/lsi'
+require 'classifier/lsi'

data/lib/classifier/bayes.rb

@@ -6,7 +6,7 @@ module Classifier
 
 class Bayes
   # The class can be created with one or more categories, each of which will be
-  # initialized and given a training method. E.g., 
+  # initialized and given a training method. E.g.,
   #      b = Classifier::Bayes.new 'Interesting', 'Uninteresting', 'Spam'
 	def initialize(*categories)
 		@categories = Hash.new
@@ -56,7 +56,7 @@ class Bayes
 			end
 		end
 	end
-		
+
 	#
 	# Returns the scores in each category the provided +text+. E.g.,
 	#    b.classifications "I hate bad words and you"
@@ -80,14 +80,14 @@ class Bayes
 	end
 
   #
-  # Returns the classification of the provided +text+, which is one of the 
+  # Returns the classification of the provided +text+, which is one of the
   # categories given in the initializer. E.g.,
   #    b.classify "I hate bad words and you"
   #    =>  'Uninteresting'
 	def classify(text)
 		(classifications(text).sort_by { |a| -a[1] })[0][0]
 	end
-	
+
 	#
 	# Provides training and untraining methods for the categories specified in Bayes#new
 	# For example:
@@ -106,7 +106,7 @@ class Bayes
 	    super  #raise StandardError, "No such method: #{name}"
 		end
 	end
-	
+
 	#
 	# Provides a list of category names
 	# For example:
@@ -115,7 +115,7 @@ class Bayes
 	def categories # :nodoc:
 		@categories.keys.collect {|c| c.to_s}
 	end
-	
+
 	#
 	# Allows you to add categories to the classifier.
 	# For example:
@@ -128,7 +128,7 @@ class Bayes
 	def add_category(category)
 		@categories[category.prepare_category_name] = Hash.new
 	end
-	
+
 	alias append_category add_category
 end
 

data/lib/classifier/extensions/vector.rb

@@ -1,5 +1,5 @@
 # Author::    Ernest Ellingson
-# Copyright:: Copyright (c) 2005 
+# Copyright:: Copyright (c) 2005
 
 # These are extensions to the std-lib 'matrix' to allow an all ruby SVD
 
@@ -9,7 +9,7 @@ require 'mathn'
 class Array
   def sum(identity = 0, &block)
     return identity unless size > 0
-  
+
     if block_given?
       map(&block).sum
     else
@@ -22,7 +22,7 @@ class Vector
   def magnitude
     sumsqs = 0.0
     self.size.times do |i|
-      sumsqs += self[i] ** 2.0 
+      sumsqs += self[i] ** 2.0
     end
     Math.sqrt(sumsqs)
   end
@@ -42,7 +42,7 @@ class Matrix
   def Matrix.diag(s)
      Matrix.diagonal(*s)
   end
-  
+
   alias :trans :transpose
 
   def SV_decomp(maxSweeps = 20)
@@ -51,7 +51,7 @@ class Matrix
     else
       q = self * self.trans
     end
-    
+
     qrot    = q.dup
     v       = Matrix.identity(q.row_size)
     azrot   = nil
@@ -75,16 +75,16 @@ class Matrix
           mzrot[col,col] = hcos
           qrot = mzrot.trans * qrot * mzrot
           v = v * mzrot
-        end 
+        end
       end
       s_old = qrot.dup if cnt == 1
-      sum_qrot = 0.0 
+      sum_qrot = 0.0
       if cnt > 1
         qrot.row_size.times do |r|
           sum_qrot += (qrot[r,r]-s_old[r,r]).abs if (qrot[r,r]-s_old[r,r]).abs > 0.001
         end
         s_old = qrot.dup
-      end 
+      end
       break if (sum_qrot <= 0.001 and cnt > 1) or cnt >= maxSweeps
     end # of do while true
     s = []
@@ -93,7 +93,7 @@ class Matrix
     end
     #puts "cnt = #{cnt}"
     if self.row_size >= self.column_size
-      mu = self *  v * Matrix.diagonal(*s).inverse     
+      mu = self *  v * Matrix.diagonal(*s).inverse
       return [mu, v, s]
     else
       puts v.row_size

data/lib/classifier/extensions/vector_serialize.rb

@@ -1,17 +1,17 @@
 module GSL
-  
+
   class Vector
     def _dump(v)
       Marshal.dump( self.to_a )
     end
-    
+
     def self._load(arr)
       arry = Marshal.load(arr)
       return GSL::Vector.alloc(arry)
     end
-    
+
   end
-  
+
   class Matrix
      class <<self
         alias :diag :diagonal

data/lib/classifier/extensions/word_hash.rb

@@ -4,20 +4,20 @@
 
 require "set"
 
-# These are extensions to the String class to provide convenience 
+# These are extensions to the String class to provide convenience
 # methods for the Classifier package.
 class String
-  
-  # Removes common punctuation symbols, returning a new string. 
+
+  # Removes common punctuation symbols, returning a new string.
   # E.g.,
   #   "Hello (greeting's), with {braces} < >...?".without_punctuation
   #   => "Hello  greetings   with  braces         "
   def without_punctuation
     tr( ',?.!;:"@#$%^&*()_=+[]{}\|<>/`~', " " ) .tr( "'\-", "")
   end
-  
+
   # Return a Hash of strings => ints. Each word in the string is stemmed,
-  # interned, and indexes to its frequency in the document.  
+  # interned, and indexes to its frequency in the document.
 	def word_hash
 		word_hash = clean_word_hash()
 		symbol_hash = word_hash_for_symbols(gsub(/[\w]/," ").split)
@@ -28,9 +28,9 @@ class String
 	def clean_word_hash
 		word_hash_for_words gsub(/[^\w\s]/,"").split
 	end
-	
+
 	private
-	
+
 	def word_hash_for_words(words)
 		d = Hash.new(0)
 		words.each do |word|
@@ -50,7 +50,7 @@ class String
 		end
 		return d
 	end
-	
+
 	CORPUS_SKIP_WORDS = Set.new([
       "a",
       "again",

data/lib/classifier/lsi.rb

@@ -4,30 +4,30 @@
 
 begin
    raise LoadError if ENV['NATIVE_VECTOR'] == "true" # to test the native vector class, try `rake test NATIVE_VECTOR=true`
-   
+
    require 'gsl' # requires http://rb-gsl.rubyforge.org/
    require 'classifier/extensions/vector_serialize'
    $GSL = true
-   
+
 rescue LoadError
 	warn "Notice: for 10x faster LSI support, please install http://rb-gsl.rubyforge.org/"
-	require 'classifier/extensions/vector'	
+	require 'classifier/extensions/vector'
 end
-   
+
 require 'classifier/lsi/word_list'
 require 'classifier/lsi/content_node'
 require 'classifier/lsi/summary'
 
 module Classifier
-  
+
   # This class implements a Latent Semantic Indexer, which can search, classify and cluster
   # data based on underlying semantic relations. For more information on the algorithms used,
   # please consult Wikipedia[http://en.wikipedia.org/wiki/Latent_Semantic_Indexing].
   class LSI
-    
+
     attr_reader :word_list
     attr_accessor :auto_rebuild
-    
+
     # Create a fresh index.
     # If you want to call #build_index manually, use
     #      Classifier::LSI.new :auto_rebuild => false
@@ -37,20 +37,20 @@ module Classifier
       @word_list, @items = WordList.new, {}
       @version, @built_at_version = 0, -1
     end
-    
+
     # Returns true if the index needs to be rebuilt.  The index needs
     # to be built after all informaton is added, but before you start
     # using it for search, classification and cluster detection.
     def needs_rebuild?
       (@items.keys.size > 1) && (@version != @built_at_version)
     end
-    
-    # Adds an item to the index. item is assumed to be a string, but 
+
+    # Adds an item to the index. item is assumed to be a string, but
     # any item may be indexed so long as it responds to #to_s or if
-    # you provide an optional block explaining how the indexer can 
+    # you provide an optional block explaining how the indexer can
     # fetch fresh string data. This optional block is passed the item,
     # so the item may only be a reference to a URL or file name.
-    # 
+    #
     # For example:
     #   lsi = Classifier::LSI.new
     #   lsi.add_item "This is just plain text"
@@ -65,14 +65,14 @@ module Classifier
       build_index if @auto_rebuild
     end
 
-    # A less flexible shorthand for add_item that assumes 
+    # A less flexible shorthand for add_item that assumes
     # you are passing in a string with no categorries. item
-    # will be duck typed via to_s . 
+    # will be duck typed via to_s .
     #
     def <<( item )
       add_item item
     end
-    
+
     # Returns the categories for a given indexed items. You are free to add and remove
     # items from this as you see fit. It does not invalide an index to change its categories.
     def categories_for(item)
@@ -80,7 +80,7 @@ module Classifier
       return @items[item].categories
     end
 
-    # Removes an item from the database, if it is indexed. 
+    # Removes an item from the database, if it is indexed.
     #
     def remove_item( item )
       if @items.keys.contain? item
@@ -88,12 +88,12 @@ module Classifier
         @version += 1
       end
     end
-    
-    # Returns an array of items that are indexed. 
+
+    # Returns an array of items that are indexed.
     def items
       @items.keys
     end
-    
+
     # Returns the categories for a given indexed items. You are free to add and remove
     # items from this as you see fit. It does not invalide an index to change its categories.
     def categories_for(item)
@@ -103,30 +103,30 @@ module Classifier
 
     # This function rebuilds the index if needs_rebuild? returns true.
     # For very large document spaces, this indexing operation may take some
-    # time to complete, so it may be wise to place the operation in another 
-    # thread. 
+    # time to complete, so it may be wise to place the operation in another
+    # thread.
     #
     # As a rule, indexing will be fairly swift on modern machines until
-    # you have well over 500 documents indexed, or have an incredibly diverse 
-    # vocabulary for your documents. 
+    # you have well over 500 documents indexed, or have an incredibly diverse
+    # vocabulary for your documents.
     #
     # The optional parameter "cutoff" is a tuning parameter. When the index is
-    # built, a certain number of s-values are discarded from the system. The 
+    # built, a certain number of s-values are discarded from the system. The
     # cutoff parameter tells the indexer how many of these values to keep.
     # A value of 1 for cutoff means that no semantic analysis will take place,
     # turning the LSI class into a simple vector search engine.
     def build_index( cutoff=0.75 )
       return unless needs_rebuild?
       make_word_list
-      
+
       doc_list = @items.values
       tda = doc_list.collect { |node| node.raw_vector_with( @word_list ) }
-      
+
       if $GSL
          tdm = GSL::Matrix.alloc(*tda).trans
          ntdm = build_reduced_matrix(tdm, cutoff)
 
-         ntdm.size[1].times do |col| 
+         ntdm.size[1].times do |col|
            vec = GSL::Vector.alloc( ntdm.column(col) ).row
            doc_list[col].lsi_vector = vec
            doc_list[col].lsi_norm = vec.normalize
@@ -134,50 +134,50 @@ module Classifier
       else
          tdm = Matrix.rows(tda).trans
          ntdm = build_reduced_matrix(tdm, cutoff)
-      
+
          ntdm.row_size.times do |col|
            doc_list[col].lsi_vector = ntdm.column(col) if doc_list[col]
            doc_list[col].lsi_norm = ntdm.column(col).normalize  if doc_list[col]
          end
       end
-   
+
       @built_at_version = @version
     end
-   
+
     # This method returns max_chunks entries, ordered by their average semantic rating.
     # Essentially, the average distance of each entry from all other entries is calculated,
     # the highest are returned.
     #
     # This can be used to build a summary service, or to provide more information about
     # your dataset's general content. For example, if you were to use categorize on the
-    # results of this data, you could gather information on what your dataset is generally 
+    # results of this data, you could gather information on what your dataset is generally
     # about.
     def highest_relative_content( max_chunks=10 )
        return [] if needs_rebuild?
-       
+
        avg_density = Hash.new
        @items.each_key { |x| avg_density[x] = proximity_array_for_content(x).inject(0.0) { |x,y| x + y[1]} }
-       
+
        avg_density.keys.sort_by { |x| avg_density[x] }.reverse[0..max_chunks-1].map
     end
 
-    # This function is the primitive that find_related and classify 
+    # This function is the primitive that find_related and classify
     # build upon. It returns an array of 2-element arrays. The first element
     # of this array is a document, and the second is its "score", defining
     # how "close" it is to other indexed items.
-    # 
+    #
     # These values are somewhat arbitrary, having to do with the vector space
     # created by your content, so the magnitude is interpretable but not always
-    # meaningful between indexes. 
+    # meaningful between indexes.
     #
     # The parameter doc is the content to compare. If that content is not
-    # indexed, you can pass an optional block to define how to create the 
-    # text data. See add_item for examples of how this works. 
+    # indexed, you can pass an optional block to define how to create the
+    # text data. See add_item for examples of how this works.
     def proximity_array_for_content( doc, &block )
       return [] if needs_rebuild?
-      
+
       content_node = node_for_content( doc, &block )
-      result = 
+      result =
         @items.keys.collect do |item|
           if $GSL
              val = content_node.search_vector * @items[item].search_vector.col
@@ -187,18 +187,18 @@ module Classifier
           [item, val]
         end
       result.sort_by { |x| x[1] }.reverse
-    end 
-    
+    end
+
     # Similar to proximity_array_for_content, this function takes similar
     # arguments and returns a similar array. However, it uses the normalized
-    # calculated vectors instead of their full versions. This is useful when 
+    # calculated vectors instead of their full versions. This is useful when
     # you're trying to perform operations on content that is much smaller than
     # the text you're working with. search uses this primitive.
     def proximity_norms_for_content( doc, &block )
       return [] if needs_rebuild?
-  
+
       content_node = node_for_content( doc, &block )
-      result = 
+      result =
         @items.keys.collect do |item|
           if $GSL
             val = content_node.search_norm * @items[item].search_norm.col
@@ -208,12 +208,12 @@ module Classifier
           [item, val]
         end
       result.sort_by { |x| x[1] }.reverse
-    end 
-    
+    end
+
     # This function allows for text-based search of your index. Unlike other functions
     # like find_related and classify, search only takes short strings. It will also ignore
-    # factors like repeated words. It is best for short, google-like search terms. 
-    # A search will first priortize lexical relationships, then semantic ones. 
+    # factors like repeated words. It is best for short, google-like search terms.
+    # A search will first priortize lexical relationships, then semantic ones.
     #
     # While this may seem backwards compared to the other functions that LSI supports,
     # it is actually the same algorithm, just applied on a smaller document.
@@ -223,30 +223,30 @@ module Classifier
       result = carry.collect { |x| x[0] }
       return result[0..max_nearest-1]
     end
-    
+
     # This function takes content and finds other documents
     # that are semantically "close", returning an array of documents sorted
     # from most to least relavant.
-    # max_nearest specifies the number of documents to return. A value of 
-    # 0 means that it returns all the indexed documents, sorted by relavence. 
+    # max_nearest specifies the number of documents to return. A value of
+    # 0 means that it returns all the indexed documents, sorted by relavence.
     #
-    # This is particularly useful for identifing clusters in your document space. 
+    # This is particularly useful for identifing clusters in your document space.
     # For example you may want to identify several "What's Related" items for weblog
     # articles, or find paragraphs that relate to each other in an essay.
     def find_related( doc, max_nearest=3, &block )
-      carry = 
+      carry =
         proximity_array_for_content( doc, &block ).reject { |pair| pair[0] == doc }
       result = carry.collect { |x| x[0] }
       return result[0..max_nearest-1]
     end
-      
-    # This function uses a voting system to categorize documents, based on 
-    # the categories of other documents. It uses the same logic as the 
+
+    # This function uses a voting system to categorize documents, based on
+    # the categories of other documents. It uses the same logic as the
     # find_related function to find related documents, then returns the
-    # most obvious category from this list. 
+    # most obvious category from this list.
     #
-    # cutoff signifies the number of documents to consider when clasifying 
-    # text. A cutoff of 1 means that every document in the index votes on 
+    # cutoff signifies the number of documents to consider when clasifying
+    # text. A cutoff of 1 means that every document in the index votes on
     # what category the document is in. This may not always make sense.
     #
     def classify( doc, cutoff=0.30, &block )
@@ -256,16 +256,16 @@ module Classifier
       votes = {}
       carry.each do |pair|
         categories = @items[pair[0]].categories
-        categories.each do |category| 
+        categories.each do |category|
           votes[category] ||= 0.0
-          votes[category] += pair[1] 
+          votes[category] += pair[1]
         end
       end
-      
+
       ranking = votes.keys.sort_by { |x| votes[x] }
       return ranking[-1]
     end
-    
+
     # Prototype, only works on indexed documents.
     # I have no clue if this is going to work, but in theory
     # it's supposed to.
@@ -289,8 +289,8 @@ module Classifier
       # Reconstruct the term document matrix, only with reduced rank
       u * ($GSL ? GSL::Matrix : ::Matrix).diag( s ) * v.trans
     end
-    
-    def node_for_content(item, &block)    
+
+    def node_for_content(item, &block)
       if @items[item]
         return @items[item]
       else
@@ -302,10 +302,10 @@ module Classifier
           cn.raw_vector_with( @word_list ) # make the lsi raw and norm vectors
         end
       end
-      
+
       return cn
     end
-    
+
     def make_word_list
       @word_list = WordList.new
       @items.each_value do |node|

data/lib/classifier/lsi/content_node.rb

@@ -4,14 +4,14 @@
 
 module Classifier
 
-# This is an internal data structure class for the LSI node. Save for 
+# This is an internal data structure class for the LSI node. Save for
 # raw_vector_with, it should be fairly straightforward to understand.
 # You should never have to use it directly.
   class ContentNode
-    attr_accessor :raw_vector, :raw_norm, 
+    attr_accessor :raw_vector, :raw_norm,
                   :lsi_vector, :lsi_norm,
-                  :categories   
-    
+                  :categories
+
     attr_reader :word_hash
     # If text_proc is not specified, the source will be duck-typed
     # via source.to_s
@@ -19,17 +19,17 @@ module Classifier
       @categories = categories || []
       @word_hash = word_hash
     end
-   
+
     # Use this to fetch the appropriate search vector.
     def search_vector
       @lsi_vector || @raw_vector
     end
-    
+
     # Use this to fetch the appropriate search vector in normalized form.
     def search_norm
       @lsi_norm || @raw_norm
     end
-   
+
     # Creates the raw vector out of word_hash using word_list as the
     # key for mapping the vector space.
     def raw_vector_with( word_list )
@@ -42,22 +42,22 @@ module Classifier
       @word_hash.each_key do |word|
         vec[word_list[word]] = @word_hash[word] if word_list[word]
       end
-     
+
       # Perform the scaling transform
       total_words = vec.sum
-      
+
       # Perform first-order association transform if this vector has more
-      # than one word in it. 
-      if total_words > 1.0 
+      # than one word in it.
+      if total_words > 1.0
         weighted_total = 0.0
         vec.each do |term|
           if ( term > 0 )
             weighted_total += (( term / total_words ) * Math.log( term / total_words ))
           end
-        end 
+        end
         vec = vec.collect { |val| Math.log( val + 1 ) / -weighted_total }
       end
-      
+
       if $GSL
          @raw_norm   = vec.normalize
          @raw_vector = vec
@@ -65,8 +65,8 @@ module Classifier
          @raw_norm   = Vector[*vec].normalize
          @raw_vector = Vector[*vec]
       end
-    end   
-  
-  end  
-  
+    end
+
+  end
+
 end

data/lib/classifier/lsi/summary.rb

@@ -14,13 +14,13 @@ class String
    def split_sentences
       split /(\.|\!|\?)/ # TODO: make this less primitive
    end
-   
+
    def split_paragraphs
       split /(\n\n|\r\r|\r\n\r\n)/ # TODO: make this less primitive
    end
-   
+
    private
-   
+
    def perform_lsi(chunks, count, separator)
       lsi = Classifier::LSI.new :auto_rebuild => false
       chunks.each { |chunk| lsi << chunk unless chunk.strip.empty? || chunk.strip.split.size == 1 }
@@ -28,4 +28,4 @@ class String
       summaries = lsi.highest_relative_content count
       return summaries.reject { |chunk| !summaries.include? chunk }.map { |x| x.strip }.join(separator)
    end
-end
+end

data/lib/classifier/lsi/word_list.rb

@@ -2,35 +2,35 @@
 # Copyright:: Copyright (c) 2005 David Fayram II
 # License::   LGPL
 
-module Classifier  
+module Classifier
   # This class keeps a word => index mapping. It is used to map stemmed words
   # to dimensions of a vector.
-    
+
   class WordList
     def initialize
       @location_table = Hash.new
     end
-    
+
     # Adds a word (if it is new) and assigns it a unique dimension.
     def add_word(word)
       term = word
       @location_table[term] = @location_table.size unless @location_table[term]
     end
-    
+
     # Returns the dimension of the word or nil if the word is not in the space.
     def [](lookup)
       term = lookup
       @location_table[term]
     end
-    
+
     def word_for_index(ind)
       @location_table.invert[ind]
     end
-    
+
     # Returns the number of words mapped.
     def size
       @location_table.size
     end
-    
+
   end
 end

metadata

@@ -1,36 +1,94 @@
 --- !ruby/object:Gem::Specification
 name: classifier
 version: !ruby/object:Gem::Version
-  version: 1.3.4
+  version: 1.3.5
 platform: ruby
 authors:
 - Lucas Carlson
-autorequire: classifier
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-31 00:00:00.000000000 Z
+date: 2018-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast-stemmer
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.0
-description: |2
-     A general classifier module to allow Bayesian and other types of classifications.
+- !ruby/object:Gem::Dependency
+  name: mathn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A general classifier module to allow Bayesian and other types of classifications.
 email: lucas@rufy.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE
+- bin/bayes.rb
+- bin/summarize.rb
 - lib/classifier.rb
 - lib/classifier/bayes.rb
 - lib/classifier/extensions/string.rb
@@ -41,19 +99,9 @@ files:
 - lib/classifier/lsi/content_node.rb
 - lib/classifier/lsi/summary.rb
 - lib/classifier/lsi/word_list.rb
-- bin/bayes.rb
-- bin/summarize.rb
-- test/bayes/bayesian_test.rb
-- test/extensions/word_hash_test.rb
-- test/lsi/lsi_test.rb
-- test/test_helper.rb
-- Gemfile
-- Gemfile.lock
-- LICENSE
-- README.markdown
-- Rakefile
-homepage: http://classifier.rufy.com/
-licenses: []
+homepage: https://github.com/cardmagic/classifier
+licenses:
+- LGPL
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -61,18 +109,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-requirements:
-- A porter-stemmer module to split word stems.
+requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A general classifier module to allow Bayesian and other types of classifications.

data/Gemfile

@@ -1,5 +0,0 @@
-source 'https://rubygems.org'
-gem 'rake'
-gem 'rspec', :require => 'spec'
-gem 'rdoc' 
-gem 'fast-stemmer'

data/Gemfile.lock

@@ -1,26 +0,0 @@
-GEM
-  remote: https://rubygems.org/
-  specs:
-    diff-lcs (1.2.5)
-    fast-stemmer (1.0.2)
-    json (1.8.1)
-    rake (10.1.1)
-    rdoc (4.1.0)
-      json (~> 1.4)
-    rspec (2.14.1)
-      rspec-core (~> 2.14.0)
-      rspec-expectations (~> 2.14.0)
-      rspec-mocks (~> 2.14.0)
-    rspec-core (2.14.7)
-    rspec-expectations (2.14.4)
-      diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.14.4)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  fast-stemmer
-  rake
-  rdoc
-  rspec

data/README.markdown

@@ -1,97 +0,0 @@
-## Welcome to Classifier
-
-Classifier is a general module to allow Bayesian and other types of classifications.
-
-## Download
-
-* https://github.com/cardmagic/classifier
-* gem install classifier
-* git clone https://github.com/cardmagic/classifier.git
-
-## Dependencies
-
-If you install Classifier from source, you'll need to install Roman Shterenzon's fast-stemmer gem with RubyGems as follows:
-
-    gem install fast-stemmer
-
-If you would like to speed up LSI classification by at least 10x, please install the following libraries:
-GNU GSL:: http://www.gnu.org/software/gsl
-rb-gsl:: http://rb-gsl.rubyforge.org
-
-Notice that LSI will work without these libraries, but as soon as they are installed, Classifier will make use of them. No configuration changes are needed, we like to keep things ridiculously easy for you.
-
-## Bayes
-
-A Bayesian classifier by Lucas Carlson. Bayesian Classifiers are accurate, fast, and have modest memory requirements.
-
-### Usage
-
-    require 'classifier'
-    b = Classifier::Bayes.new 'Interesting', 'Uninteresting'
-    b.train_interesting "here are some good words. I hope you love them"
-    b.train_uninteresting "here are some bad words, I hate you"
-    b.classify "I hate bad words and you" # returns 'Uninteresting'
-    
-    require 'madeleine'
-    m = SnapshotMadeleine.new("bayes_data") {
-        Classifier::Bayes.new 'Interesting', 'Uninteresting'
-    }
-    m.system.train_interesting "here are some good words. I hope you love them"
-    m.system.train_uninteresting "here are some bad words, I hate you"
-    m.take_snapshot
-    m.system.classify "I love you" # returns 'Interesting'
-
-Using Madeleine, your application can persist the learned data over time.
-
-### Bayesian Classification
-
-* http://www.process.com/precisemail/bayesian_filtering.htm
-* http://en.wikipedia.org/wiki/Bayesian_filtering
-* http://www.paulgraham.com/spam.html
-
-## LSI
-
-A Latent Semantic Indexer by David Fayram. Latent Semantic Indexing engines
-are not as fast or as small as Bayesian classifiers, but are more flexible, providing 
-fast search and clustering detection as well as semantic analysis of the text that 
-theoretically simulates human learning.
-
-### Usage
-
-    require 'classifier'
-    lsi = Classifier::LSI.new
-    strings = [ ["This text deals with dogs. Dogs.", :dog],
-              ["This text involves dogs too. Dogs! ", :dog],
-              ["This text revolves around cats. Cats.", :cat],
-              ["This text also involves cats. Cats!", :cat],
-              ["This text involves birds. Birds.",:bird ]]
-    strings.each {|x| lsi.add_item x.first, x.last}
-  
-    lsi.search("dog", 3)
-    # returns => ["This text deals with dogs. Dogs.", "This text involves dogs too. Dogs! ", 
-    #             "This text also involves cats. Cats!"]
-
-    lsi.find_related(strings[2], 2)
-    # returns => ["This text revolves around cats. Cats.", "This text also involves cats. Cats!"]
-  
-    lsi.classify "This text is also about dogs!"
-    # returns => :dog
-  
-Please see the Classifier::LSI documentation for more information. It is possible to index, search and classify
-with more than just simple strings. 
-
-### Latent Semantic Indexing
-
-* http://www.c2.com/cgi/wiki?LatentSemanticIndexing
-* http://www.chadfowler.com/index.cgi/Computing/LatentSemanticIndexing.rdoc
-* http://en.wikipedia.org/wiki/Latent_semantic_analysis
-
-## Authors    
-
-* Lucas Carlson  (lucas@rufy.com)
-* David Fayram II (dfayram@gmail.com)
-* Cameron McBride (cameron.mcbride@gmail.com)
-* Ivan Acosta-Rubio (ivan@softwarecriollo.com)
-
-This library is released under the terms of the GNU LGPL. See LICENSE for more details.
-

data/Rakefile

@@ -1,84 +0,0 @@
-require 'rubygems'
-require 'rake'
-require 'rake/testtask'
-require 'rdoc/task'
-require 'rake/contrib/rubyforgepublisher'
-
-desc "Default Task"
-task :default => [ :test ]
-
-# Run the unit tests
-desc "Run all unit tests"
-Rake::TestTask.new("test") { |t|
-  t.libs << "lib"
-  t.pattern = 'test/*/*_test.rb'
-  t.verbose = true
-}
-
-# Make a console, useful when working on tests
-desc "Generate a test console"
-task :console do
-   verbose( false ) { sh "irb -I lib/ -r 'classifier'" }
-end
-
-# Genereate the RDoc documentation
-desc "Create documentation"
-Rake::RDocTask.new("doc") { |rdoc|
-  rdoc.title = "Ruby Classifier - Bayesian and LSI classification library"
-  rdoc.rdoc_dir = 'html'
-  rdoc.rdoc_files.include('README')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-}
-
-# Genereate the package
-spec = Gem::Specification.new do |s|
-
-  #### Basic information.
-
-  s.name = 'classifier'
-  s.version = PKG_VERSION
-  s.summary = <<-EOF
-   A general classifier module to allow Bayesian and other types of classifications.
-  EOF
-  s.description = <<-EOF
-   A general classifier module to allow Bayesian and other types of classifications.
-  EOF
-
-  #### Which files are to be included in this gem?  Everything!  (Except CVS directories.)
-
-  s.files = PKG_FILES
-
-  #### Load-time details: library and application (you will need one or both).
-
-  s.require_path = 'lib'
-  s.autorequire = 'classifier'
-
-  #### Documentation and testing.
-
-  s.has_rdoc = true
-
-  #### Dependencies and requirements.
-
-  s.add_dependency('fast-stemmer', '>= 1.0.0')
-  s.requirements << "A porter-stemmer module to split word stems."
-
-  #### Author and project details.
-  s.author = "Lucas Carlson"
-  s.email = "lucas@rufy.com"
-  s.homepage = "http://classifier.rufy.com/"
-end
-
-desc "Report code statistics (KLOCs, etc) from the application"
-task :stats do
-  require 'code_statistics'
-  CodeStatistics.new(
-    ["Library", "lib"],
-    ["Units", "test"]
-  ).to_s
-end
-
-desc "Publish new documentation"
-task :publish do
-   `ssh rufy update-classifier-doc`
-    Rake::RubyForgePublisher.new('classifier', 'cardmagic').upload
-end

data/test/bayes/bayesian_test.rb

@@ -1,33 +0,0 @@
-require File.dirname(__FILE__) + '/../test_helper'
-class BayesianTest < Test::Unit::TestCase
-	def setup
-		@classifier = Classifier::Bayes.new 'Interesting', 'Uninteresting'
-	end
-	
-	def test_good_training
-		assert_nothing_raised { @classifier.train_interesting "love" }
-	end
-
-	def test_bad_training
-		assert_raise(StandardError) { @classifier.train_no_category "words" }
-	end
-	
-	def test_bad_method
-		assert_raise(NoMethodError) { @classifier.forget_everything_you_know "" }
-	end
-	
-	def test_categories
-		assert_equal ['Interesting', 'Uninteresting'].sort, @classifier.categories.sort
-	end
-
-	def test_add_category
-		@classifier.add_category 'Test'
-		assert_equal ['Test', 'Interesting', 'Uninteresting'].sort, @classifier.categories.sort
-	end
-
-	def test_classification
-		@classifier.train_interesting "here are some good words. I hope you love them"
-		@classifier.train_uninteresting "here are some bad words, I hate you"
-		assert_equal 'Uninteresting', @classifier.classify("I hate bad words and you")
-	end
-end

data/test/extensions/word_hash_test.rb

@@ -1,35 +0,0 @@
-require File.dirname(__FILE__) + '/../test_helper'
-class StringExtensionsTest < Test::Unit::TestCase
-	def test_word_hash
-		hash = {:good=>1, :"!"=>1, :hope=>1, :"'"=>1, :"."=>1, :love=>1, :word=>1, :them=>1, :test=>1}
-		assert_equal hash, "here are some good words of test's. I hope you love them!".word_hash
-	end
-
-   	
-	def test_clean_word_hash
-	   hash = {:good=>1, :word=>1, :hope=>1, :love=>1, :them=>1, :test=>1}
-	   assert_equal hash, "here are some good words of test's. I hope you love them!".clean_word_hash
-	end
-
-end
-
-
-class ArrayExtensionsTest < Test::Unit::TestCase
-
-  def test_plays_nicely_with_any_array
-    assert_equal [Array].sum, Array
-  end
-
-  def test_monkey_path_array_sum
-    assert_equal [1,2,3].sum, 6
-  end
-
-  def test_summing_an_empty_array
-    assert_equal [nil].sum, 0
-  end
-
-  def test_summing_an_empty_array
-    assert_equal Array[].sum, 0
-  end
-
-end

data/test/lsi/lsi_test.rb

@@ -1,123 +0,0 @@
-require File.dirname(__FILE__) + '/../test_helper'
-class LSITest < Test::Unit::TestCase
-	def setup
-	  # we repeat principle words to help weight them. 
-	  # This test is rather delicate, since this system is mostly noise.
-     @str1 = "This text deals with dogs. Dogs."
-	  @str2 = "This text involves dogs too. Dogs! "
-	  @str3 = "This text revolves around cats. Cats."
-	  @str4 = "This text also involves cats. Cats!"
-	  @str5 = "This text involves birds. Birds."
-	end
-	
-	def test_basic_indexing
-	 lsi = Classifier::LSI.new
-	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
-	  assert ! lsi.needs_rebuild?
-	  
-	 # note that the closest match to str1 is str2, even though it is not
-	 # the closest text match.
-	 assert_equal [@str2, @str5, @str3], lsi.find_related(@str1, 3)
-	end
-
-	def test_not_auto_rebuild
-	 lsi = Classifier::LSI.new :auto_rebuild => false
-	 lsi.add_item @str1, "Dog"
-	 lsi.add_item @str2, "Dog"
-	 assert lsi.needs_rebuild?
-	 lsi.build_index
-	 assert ! lsi.needs_rebuild?
-	end
-
-	def test_basic_categorizing
-	  lsi = Classifier::LSI.new
-	  lsi.add_item @str2, "Dog"
-	  lsi.add_item @str3, "Cat"
-	  lsi.add_item @str4, "Cat"
-	  lsi.add_item @str5, "Bird"
-	  
-	  assert_equal "Dog", lsi.classify( @str1 )
-	  assert_equal "Cat", lsi.classify( @str3 )
-     assert_equal "Bird", lsi.classify( @str5 )  
-	end
-	
-	def test_external_classifying
-	  lsi = Classifier::LSI.new
-	  bayes = Classifier::Bayes.new 'Dog', 'Cat', 'Bird'
-	  lsi.add_item @str1, "Dog" ; bayes.train_dog @str1
-	  lsi.add_item @str2, "Dog" ; bayes.train_dog @str2
-	  lsi.add_item @str3, "Cat" ; bayes.train_cat @str3
-	  lsi.add_item @str4, "Cat" ; bayes.train_cat @str4
-	  lsi.add_item @str5, "Bird" ; bayes.train_bird @str5
-	  
-	  # We're talking about dogs. Even though the text matches the corpus on 
-	  # cats better.  Dogs have more semantic weight than cats. So bayes
-	  # will fail here, but the LSI recognizes content.
-	  tricky_case = "This text revolves around dogs."
-	  assert_equal "Dog", lsi.classify( tricky_case )
-	  assert_not_equal "Dog", bayes.classify( tricky_case )
-	end 
-	
-	def test_recategorize_interface
-	  lsi = Classifier::LSI.new
-	  lsi.add_item @str1, "Dog"
-	  lsi.add_item @str2, "Dog"
-	  lsi.add_item @str3, "Cat"
-	  lsi.add_item @str4, "Cat"
-	  lsi.add_item @str5, "Bird"
-	  
-	  tricky_case = "This text revolves around dogs."
-	  assert_equal "Dog", lsi.classify( tricky_case )
-	  
-	  # Recategorize as needed.
-	  lsi.categories_for(@str1).clear.push "Cow"
-	  lsi.categories_for(@str2).clear.push "Cow"
-
-	  assert !lsi.needs_rebuild?
-	  assert_equal "Cow", lsi.classify( tricky_case )	  
-	end
-	
-	def test_search
-	  lsi = Classifier::LSI.new
-	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
-	  
-	  # Searching by content and text, note that @str2 comes up first, because
-	  # both "dog" and "involve" are present. But, the next match is @str1 instead
-	  # of @str4, because "dog" carries more weight than involves. 
-	  assert_equal( [@str2, @str1, @str4, @str5, @str3],
-	                lsi.search("dog involves", 100) )
-	                
-	  # Keyword search shows how the space is mapped out in relation to 
-	  # dog when magnitude is remove. Note the relations. We move from dog
-	  # through involve and then finally to other words. 
-	  assert_equal( [@str1, @str2, @str4, @str5, @str3],
-	                lsi.search("dog", 5) )
-	end
-	
-	def test_serialize_safe
-    lsi = Classifier::LSI.new
-	  [@str1, @str2, @str3, @str4, @str5].each { |x| lsi << x }
-	  
-	  lsi_md = Marshal.dump lsi
-	  lsi_m = Marshal.load lsi_md
-	  
-	  assert_equal lsi_m.search("cat", 3), lsi.search("cat", 3)
-	  assert_equal lsi_m.find_related(@str1, 3), lsi.find_related(@str1, 3)
-	end
-	
-	def test_keyword_search
-	  lsi = Classifier::LSI.new
-	  lsi.add_item @str1, "Dog"
-	  lsi.add_item @str2, "Dog"
-	  lsi.add_item @str3, "Cat"
-	  lsi.add_item @str4, "Cat"
-	  lsi.add_item @str5, "Bird"
-	
-	  assert_equal [:dog, :text, :deal], lsi.highest_ranked_stems(@str1)
-	end
-	
-	def test_summary
-	   assert_equal "This text involves dogs too [...] This text also involves cats", [@str1, @str2, @str3, @str4, @str5].join.summary(2)
-	end
-	  
-end

data/test/test_helper.rb

@@ -1,4 +0,0 @@
-$:.unshift(File.dirname(__FILE__) + '/../lib')
-
-require 'test/unit'
-require 'classifier'