classifier-reborn 2.0.0.rc1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85ca221ce7678d94a2d7020e37b260991e64aa84
-  data.tar.gz: 3260eeef860f793d575da189c3acc87f6ed96089
+  metadata.gz: b5bf47d31418ceab033a1cb67ef62dd0958d619f
+  data.tar.gz: 9fbfb8cfa723f0c124130aa15ef831e8cb30e317
 SHA512:
-  metadata.gz: 88de7ba1a323cabcf871e98f2607bc25263830d4a9808d4c2d8d9d880d36ac85547e25e93a3eb555419c543d99edd455b9c6b95599c707e2e12421d02a33dafc
-  data.tar.gz: 3c346d1aff42973651869e4738adaf673c2d258b8fdd5cf9f724f0659590ae8173abe4c5ac21a06c098b0085c3c155e1bff0f4cce16e7a1bbf3f50c869720b8c
+  metadata.gz: 79d2af7643a0184efc1f4173f4cce6e966d6e02d8ddc60ebfa03c8aed1f34d6c33ec6cb75077e46f466acf91eabe9122af4ff611d5f96f3d7bfba29923de90db
+  data.tar.gz: 80bc672d0e87fdb0a5064dcd6d8d80746923f7713f38da1c34decb3e3b4a7e11fe92e16e2c8230a1837905b91b3db3de7368c56f0f89654500f4ea242291775a

data/README.markdown

@@ -1,22 +1,35 @@
-## Welcome to Classifier
+## Welcome to Classifier Reborn
 
 Classifier is a general module to allow Bayesian and other types of classifications.
 
+Classifier Reborn is a fork of cardmagic/classifier under more active development.
+
 ## Download
 
-* https://github.com/cardmagic/classifier
-* gem install classifier
-* git clone https://github.com/cardmagic/classifier.git
+Add this line to your application's Gemfile:
+
+    gem 'classifier-reborn'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install classifier-reborn
 
 ## Dependencies
 
-If you install Classifier from source, you'll need to install Roman Shterenzon's fast-stemmer gem with RubyGems as follows:
+The only runtime dependency you'll need to install is Roman Shterenzon's fast-stemmer gem:
 
     gem install fast-stemmer
 
+This should install automatically with RubyGems.
+
 If you would like to speed up LSI classification by at least 10x, please install the following libraries:
-GNU GSL:: http://www.gnu.org/software/gsl
-rb-gsl:: http://rb-gsl.rubyforge.org
+
+* [GNU GSL](http://www.gnu.org/software/gsl)
+* [rb-gsl](http://rb-gsl.rubyforge.org)
 
 Notice that LSI will work without these libraries, but as soon as they are installed, Classifier will make use of them. No configuration changes are needed, we like to keep things ridiculously easy for you.
 
@@ -26,20 +39,22 @@ A Bayesian classifier by Lucas Carlson. Bayesian Classifiers are accurate, fast,
 
 ### Usage
 
-    require 'classifier'
-    b = ClassifierReborn::Bayes.new 'Interesting', 'Uninteresting'
-    b.train_interesting "here are some good words. I hope you love them"
-    b.train_uninteresting "here are some bad words, I hate you"
-    b.classify "I hate bad words and you" # returns 'Uninteresting'
-    
-    require 'madeleine'
-    m = SnapshotMadeleine.new("bayes_data") {
-        ClassifierReborn::Bayes.new 'Interesting', 'Uninteresting'
-    }
-    m.system.train_interesting "here are some good words. I hope you love them"
-    m.system.train_uninteresting "here are some bad words, I hate you"
-    m.take_snapshot
-    m.system.classify "I love you" # returns 'Interesting'
+```ruby
+require 'classifier'
+b = ClassifierReborn::Bayes.new 'Interesting', 'Uninteresting'
+b.train_interesting "here are some good words. I hope you love them"
+b.train_uninteresting "here are some bad words, I hate you"
+b.classify "I hate bad words and you" # returns 'Uninteresting'
+
+require 'madeleine' # use madeline to persist the data
+m = SnapshotMadeleine.new("bayes_data") {
+  ClassifierReborn::Bayes.new 'Interesting', 'Uninteresting'
+}
+m.system.train_interesting "here are some good words. I hope you love them"
+m.system.train_uninteresting "here are some bad words, I hate you"
+m.take_snapshot
+m.system.classify "I love you" # returns 'Interesting'
+```
 
 Using Madeleine, your application can persist the learned data over time.
 
@@ -52,33 +67,35 @@ Using Madeleine, your application can persist the learned data over time.
 ## LSI
 
 A Latent Semantic Indexer by David Fayram. Latent Semantic Indexing engines
-are not as fast or as small as Bayesian classifiers, but are more flexible, providing 
-fast search and clustering detection as well as semantic analysis of the text that 
+are not as fast or as small as Bayesian classifiers, but are more flexible, providing
+fast search and clustering detection as well as semantic analysis of the text that
 theoretically simulates human learning.
 
 ### Usage
 
-    require 'classifier'
-    lsi = ClassifierReborn::LSI.new
-    strings = [ ["This text deals with dogs. Dogs.", :dog],
-              ["This text involves dogs too. Dogs! ", :dog],
-              ["This text revolves around cats. Cats.", :cat],
-              ["This text also involves cats. Cats!", :cat],
-              ["This text involves birds. Birds.",:bird ]]
-    strings.each {|x| lsi.add_item x.first, x.last}
-  
-    lsi.search("dog", 3)
-    # returns => ["This text deals with dogs. Dogs.", "This text involves dogs too. Dogs! ", 
-    #             "This text also involves cats. Cats!"]
-
-    lsi.find_related(strings[2], 2)
-    # returns => ["This text revolves around cats. Cats.", "This text also involves cats. Cats!"]
-  
-    lsi.classify "This text is also about dogs!"
-    # returns => :dog
-  
+```ruby
+require 'classifier'
+lsi = ClassifierReborn::LSI.new
+strings = [ ["This text deals with dogs. Dogs.", :dog],
+            ["This text involves dogs too. Dogs! ", :dog],
+            ["This text revolves around cats. Cats.", :cat],
+            ["This text also involves cats. Cats!", :cat],
+            ["This text involves birds. Birds.",:bird ]]
+strings.each {|x| lsi.add_item x.first, x.last}
+
+lsi.search("dog", 3)
+# returns => ["This text deals with dogs. Dogs.", "This text involves dogs too. Dogs! ",
+#             "This text also involves cats. Cats!"]
+
+lsi.find_related(strings[2], 2)
+# returns => ["This text revolves around cats. Cats.", "This text also involves cats. Cats!"]
+
+lsi.classify "This text is also about dogs!"
+# returns => :dog
+```
+
 Please see the ClassifierReborn::LSI documentation for more information. It is possible to index, search and classify
-with more than just simple strings. 
+with more than just simple strings.
 
 ### Latent Semantic Indexing
 
@@ -86,12 +103,12 @@ with more than just simple strings.
 * http://www.chadfowler.com/index.cgi/Computing/LatentSemanticIndexing.rdoc
 * http://en.wikipedia.org/wiki/Latent_semantic_analysis
 
-## Authors    
+## Authors
 
 * Lucas Carlson  (lucas@rufy.com)
 * David Fayram II (dfayram@gmail.com)
 * Cameron McBride (cameron.mcbride@gmail.com)
 * Ivan Acosta-Rubio (ivan@softwarecriollo.com)
+* Parker Moore (email@byparker.com)
 
 This library is released under the terms of the GNU LGPL. See LICENSE for more details.
-

data/lib/classifier-reborn/bayes.rb

@@ -2,6 +2,8 @@
 # Copyright:: Copyright (c) 2005 Lucas Carlson
 # License::   LGPL
 
+require_relative 'extensions/string'
+
 module ClassifierReborn
   class Bayes
     # The class can be created with one or more categories, each of which will be
@@ -23,7 +25,7 @@ module ClassifierReborn
     def train(category, text)
       category = category.prepare_category_name
                   @category_counts[category] += 1
-      text.word_hash.each do |word, count|
+      Hasher.word_hash(text).each do |word, count|
         @categories[category][word]     ||=     0
         @categories[category][word]      +=     count
         @total_words += count
@@ -39,12 +41,12 @@ module ClassifierReborn
     #     b.untrain :this, "This text"
     def untrain(category, text)
       category = category.prepare_category_name
-                  @category_counts[category] -= 1
-      text.word_hash.each do |word, count|
+      @category_counts[category] -= 1
+      Hasher.word_hash(text).each do |word, count|
         if @total_words >= 0
-          orig = @categories[category][word]
-          @categories[category][word]     ||=     0
-          @categories[category][word]      -=     count
+          orig = @categories[category][word] || 0
+          @categories[category][word] ||= 0
+          @categories[category][word] -= count
           if @categories[category][word] <= 0
             @categories[category].delete(word)
             count = orig
@@ -64,7 +66,7 @@ module ClassifierReborn
       @categories.each do |category, category_words|
         score[category.to_s] = 0
         total = category_words.values.inject(0) {|sum, element| sum+element}
-        text.word_hash.each do |word, count|
+        Hasher.word_hash(text).each do |word, count|
           s = category_words.has_key?(word) ? category_words[word] : 0.1
           score[category.to_s] += Math.log(s/total.to_f)
         end

data/lib/classifier-reborn/extensions/hasher.rb

@@ -0,0 +1,134 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+require "set"
+
+module ClassifierReborn
+  module Hasher
+    extend self
+
+    # Removes common punctuation symbols, returning a new string.
+    # E.g.,
+    #   "Hello (greeting's), with {braces} < >...?".without_punctuation
+    #   => "Hello  greetings   with  braces         "
+    def without_punctuation(str)
+      str .tr( ',?.!;:"@#$%^&*()_=+[]{}\|<>/`~', " " ) .tr( "'\-", "")
+    end
+
+    # Return a Hash of strings => ints. Each word in the string is stemmed,
+    # interned, and indexes to its frequency in the document.
+    def word_hash(str)
+      word_hash   = clean_word_hash(str)
+      symbol_hash = word_hash_for_symbols(str.gsub(/[\w]/," ").split)
+      return clean_word_hash(str).merge(symbol_hash)
+    end
+
+    # Return a word hash without extra punctuation or short symbols, just stemmed words
+    def clean_word_hash(str)
+      word_hash_for_words str.gsub(/[^\w\s]/,"").split
+    end
+
+    def word_hash_for_words(words)
+      d = Hash.new(0)
+      words.each do |word|
+        word.downcase!
+        if ! CORPUS_SKIP_WORDS.include?(word) && word.length > 2
+          d[word.stem.intern] += 1
+        end
+      end
+      return d
+    end
+
+    def word_hash_for_symbols(words)
+      d = Hash.new(0)
+      words.each do |word|
+        d[word.intern] += 1
+      end
+      return d
+    end
+
+    CORPUS_SKIP_WORDS = Set.new(%w[
+      a
+      again
+      all
+      along
+      are
+      also
+      an
+      and
+      as
+      at
+      but
+      by
+      came
+      can
+      cant
+      couldnt
+      did
+      didn
+      didnt
+      do
+      doesnt
+      dont
+      ever
+      first
+      from
+      have
+      her
+      here
+      him
+      how
+      i
+      if
+      in
+      into
+      is
+      isnt
+      it
+      itll
+      just
+      last
+      least
+      like
+      most
+      my
+      new
+      no
+      not
+      now
+      of
+      on
+      or
+      should
+      sinc
+      so
+      some
+      th
+      than
+      this
+      that
+      the
+      their
+      then
+      those
+      to
+      told
+      too
+      true
+      try
+      until
+      url
+      us
+      were
+      when
+      whether
+      while
+      with
+      within
+      yes
+      you
+      youll
+    ])
+  end
+end

data/lib/classifier-reborn/extensions/string.rb

@@ -3,7 +3,7 @@
 # License::   LGPL
 
 require 'fast_stemmer'
-require 'classifier-reborn/extensions/word_hash'
+require 'classifier-reborn/extensions/hasher'
 
 class Object
   def prepare_category_name; to_s.gsub("_"," ").capitalize.intern end

data/lib/classifier-reborn/extensions/vector.rb

@@ -4,7 +4,6 @@
 # These are extensions to the std-lib 'matrix' to allow an all ruby SVD
 
 require 'matrix'
-require 'mathn'
 
 class Array
   def sum(identity = 0, &block)
@@ -13,7 +12,7 @@ class Array
     if block_given?
       map(&block).sum
     else
-      reduce(:+)
+      reduce(:+) || 0
     end
   end
 end

data/lib/classifier-reborn/lsi.rb

@@ -6,16 +6,16 @@ begin
   raise LoadError if ENV['NATIVE_VECTOR'] == "true" # to test the native vector class, try `rake test NATIVE_VECTOR=true`
 
   require 'gsl' # requires http://rb-gsl.rubyforge.org/
-  require 'classifier-reborn/extensions/vector_serialize'
+  require_relative 'extensions/vector_serialize'
   $GSL = true
 
 rescue LoadError
-  require 'classifier-reborn/extensions/vector'
+  require_relative 'extensions/vector'
 end
 
-require 'classifier-reborn/lsi/word_list'
-require 'classifier-reborn/lsi/content_node'
-require 'classifier-reborn/lsi/summary'
+require_relative 'lsi/word_list'
+require_relative 'lsi/content_node'
+require_relative 'lsi/summarizer'
 
 module ClassifierReborn
 
@@ -58,7 +58,7 @@ module ClassifierReborn
     #   lsi.add_item ar, *ar.categories { |x| ar.content }
     #
     def add_item( item, *categories, &block )
-      clean_word_hash = block ? block.call(item).clean_word_hash : item.to_s.clean_word_hash
+      clean_word_hash = Hasher.clean_word_hash(block ? block.call(item) : item.to_s)
       @items[item] = ContentNode.new(clean_word_hash, *categories)
       @version += 1
       build_index if @auto_rebuild
@@ -82,8 +82,8 @@ module ClassifierReborn
     # Removes an item from the database, if it is indexed.
     #
     def remove_item( item )
-      if @items.keys.contain? item
-        @items.remove item
+      if @items.key? item
+        @items.delete item
         @version += 1
       end
     end
@@ -293,7 +293,7 @@ module ClassifierReborn
       if @items[item]
         return @items[item]
       else
-        clean_word_hash = block ? block.call(item).clean_word_hash : item.to_s.clean_word_hash
+        clean_word_hash = Hasher.clean_word_hash(block ? block.call(item) : item.to_s)
 
         cn = ContentNode.new(clean_word_hash, &block) # make the node and extract the data
 

data/lib/classifier-reborn/lsi/content_node.rb

@@ -43,12 +43,14 @@ module ClassifierReborn
         vec[word_list[word]] = @word_hash[word] if word_list[word]
       end
 
-      # Perform the scaling transform
-      total_words = vec.sum
+      # Perform the scaling transform and force floating point arithmetic
+      total_words = vec.sum.to_f
+
+      total_unique_words = vec.count{|word| word != 0}
 
       # Perform first-order association transform if this vector has more
-      # than one word in it.
-      if total_words > 1.0
+      # then one word in it.
+      if total_words > 1.0 && total_unique_words > 1
         weighted_total = 0.0
         vec.each do |term|
           if ( term > 0 )

data/lib/classifier-reborn/lsi/summarizer.rb

@@ -0,0 +1,33 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+module ClassifierReborn
+  module Summarizer
+    extend self
+
+    def summary( str, count=10, separator=" [...] " )
+      perform_lsi split_sentences(str), count, separator
+    end
+
+    def paragraph_summary( str, count=1, separator=" [...] " )
+      perform_lsi split_paragraphs(str), count, separator
+    end
+
+    def split_sentences(str)
+      str.split /(\.|\!|\?)/ # TODO: make this less primitive
+    end
+
+    def split_paragraphs(str)
+      str.split /(\n\n|\r\r|\r\n\r\n)/ # TODO: make this less primitive
+    end
+
+    def perform_lsi(chunks, count, separator)
+      lsi = ClassifierReborn::LSI.new :auto_rebuild => false
+      chunks.each { |chunk| lsi << chunk unless chunk.strip.empty? || chunk.strip.split.size == 1 }
+      lsi.build_index
+      summaries = lsi.highest_relative_content count
+      return summaries.reject { |chunk| !summaries.include? chunk }.map { |x| x.strip }.join(separator)
+    end
+  end
+end

data/lib/classifier-reborn/version.rb

@@ -1,3 +1,3 @@
 module ClassifierReborn
-  VERSION = '2.0.0.rc1'
+  VERSION = '2.0.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: classifier-reborn
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc1
+  version: 2.0.0
 platform: ruby
 authors:
 - Lucas Carlson
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-10 00:00:00.000000000 Z
+date: 2014-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast-stemmer
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -71,13 +71,13 @@ files:
 - bin/summarize.rb
 - lib/classifier-reborn.rb
 - lib/classifier-reborn/bayes.rb
+- lib/classifier-reborn/extensions/hasher.rb
 - lib/classifier-reborn/extensions/string.rb
 - lib/classifier-reborn/extensions/vector.rb
 - lib/classifier-reborn/extensions/vector_serialize.rb
-- lib/classifier-reborn/extensions/word_hash.rb
 - lib/classifier-reborn/lsi.rb
 - lib/classifier-reborn/lsi/content_node.rb
-- lib/classifier-reborn/lsi/summary.rb
+- lib/classifier-reborn/lsi/summarizer.rb
 - lib/classifier-reborn/lsi/word_list.rb
 - lib/classifier-reborn/version.rb
 homepage: https://github.com/jekyll/classifier-reborn
@@ -96,9 +96,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.2.2

data/lib/classifier-reborn/extensions/word_hash.rb

@@ -1,136 +0,0 @@
-# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
-# Copyright:: Copyright (c) 2005 Lucas Carlson
-# License::   LGPL
-
-require "set"
-
-# These are extensions to the String class to provide convenience
-# methods for the Classifier package.
-class String
-
-  # Removes common punctuation symbols, returning a new string.
-  # E.g.,
-  #   "Hello (greeting's), with {braces} < >...?".without_punctuation
-  #   => "Hello  greetings   with  braces         "
-  def without_punctuation
-    tr( ',?.!;:"@#$%^&*()_=+[]{}\|<>/`~', " " ) .tr( "'\-", "")
-  end
-
-  # Return a Hash of strings => ints. Each word in the string is stemmed,
-  # interned, and indexes to its frequency in the document.
-	def word_hash
-		word_hash = clean_word_hash()
-		symbol_hash = word_hash_for_symbols(gsub(/[\w]/," ").split)
-		return word_hash.merge(symbol_hash)
-	end
-
-	# Return a word hash without extra punctuation or short symbols, just stemmed words
-	def clean_word_hash
-		word_hash_for_words gsub(/[^\w\s]/,"").split
-	end
-
-	private
-
-	def word_hash_for_words(words)
-		d = Hash.new(0)
-		words.each do |word|
-			word.downcase!
-			if ! CORPUS_SKIP_WORDS.include?(word) && word.length > 2
-				d[word.stem.intern] += 1
-			end
-		end
-		return d
-	end
-
-
-	def word_hash_for_symbols(words)
-		d = Hash.new(0)
-		words.each do |word|
-			d[word.intern] += 1
-		end
-		return d
-	end
-
-	CORPUS_SKIP_WORDS = Set.new([
-      "a",
-      "again",
-      "all",
-      "along",
-      "are",
-      "also",
-      "an",
-      "and",
-      "as",
-      "at",
-      "but",
-      "by",
-      "came",
-      "can",
-      "cant",
-      "couldnt",
-      "did",
-      "didn",
-      "didnt",
-      "do",
-      "doesnt",
-      "dont",
-      "ever",
-      "first",
-      "from",
-      "have",
-      "her",
-      "here",
-      "him",
-      "how",
-      "i",
-      "if",
-      "in",
-      "into",
-      "is",
-      "isnt",
-      "it",
-      "itll",
-      "just",
-      "last",
-      "least",
-      "like",
-      "most",
-      "my",
-      "new",
-      "no",
-      "not",
-      "now",
-      "of",
-      "on",
-      "or",
-      "should",
-      "sinc",
-      "so",
-      "some",
-      "th",
-      "than",
-      "this",
-      "that",
-      "the",
-      "their",
-      "then",
-      "those",
-      "to",
-      "told",
-      "too",
-      "true",
-      "try",
-      "until",
-      "url",
-      "us",
-      "were",
-      "when",
-      "whether",
-      "while",
-      "with",
-      "within",
-      "yes",
-      "you",
-      "youll",
-      ])
-end

data/lib/classifier-reborn/lsi/summary.rb

@@ -1,31 +0,0 @@
-# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
-# Copyright:: Copyright (c) 2005 Lucas Carlson
-# License::   LGPL
-
-class String
-  def summary( count=10, separator=" [...] " )
-    perform_lsi split_sentences, count, separator
-  end
-
-  def paragraph_summary( count=1, separator=" [...] " )
-    perform_lsi split_paragraphs, count, separator
-  end
-
-  def split_sentences
-    split /(\.|\!|\?)/ # TODO: make this less primitive
-  end
-
-  def split_paragraphs
-    split /(\n\n|\r\r|\r\n\r\n)/ # TODO: make this less primitive
-  end
-
-  private
-
-  def perform_lsi(chunks, count, separator)
-    lsi = ClassifierReborn::LSI.new :auto_rebuild => false
-    chunks.each { |chunk| lsi << chunk unless chunk.strip.empty? || chunk.strip.split.size == 1 }
-    lsi.build_index
-    summaries = lsi.highest_relative_content count
-    return summaries.reject { |chunk| !summaries.include? chunk }.map { |x| x.strip }.join(separator)
-  end
-end