linnaeus 1.0.2 → 1.0.3

data/Gemfile

@@ -1,4 +1,4 @@
-source "http://rubygems.org"
+source 'http://rubygems.org'
 
 gem 'redis', '~> 3.0.0'
 gem 'stemmer', '~> 1.0.0'
@@ -6,10 +6,11 @@ gem 'stemmer', '~> 1.0.0'
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
-  gem "rspec", "~> 2.11.0"
-  gem "yard", "~> 0.7"
-  gem "rdoc", "~> 3.12"
-  gem "bundler"
-  gem "jeweler"
-  gem "simplecov"
+  gem 'rspec', '~> 2.11.0'
+  gem 'yard', '~> 0.7'
+  gem 'rdoc', '~> 3.12'
+  gem 'bundler'
+  gem 'jeweler'
+  gem 'simplecov'
+  gem 'redcarpet'
 end

data/Gemfile.lock

@@ -13,6 +13,7 @@ GEM
     rake (0.9.2.2)
     rdoc (3.12)
       json (~> 1.4)
+    redcarpet (2.2.2)
     redis (3.0.2)
     rspec (2.11.0)
       rspec-core (~> 2.11.0)
@@ -36,6 +37,7 @@ DEPENDENCIES
   bundler
   jeweler
   rdoc (~> 3.12)
+  redcarpet
   redis (~> 3.0.0)
   rspec (~> 2.11.0)
   simplecov

data/README.md

@@ -0,0 +1,40 @@
+# Linnaeus [![Build Status](https://secure.travis-ci.org/djcp/linnaeus.png?branch=master)](http://travis-ci.org/djcp/linnaeus)
+
+![Carl Linnaeus](https://raw.github.com/djcp/linnaeus/master/images/linnaeus.jpg)
+
+Linnaeus is a redis-backed naive Bayesian classification system. Please see the [rdoc](http://rubydoc.info/gems/linnaeus/) for more information. Ruby 1.9 is required.
+
+Examples
+--------
+
+    lt = Linnaeus::Trainer.new      # Used to train documents
+    lc = Linnaeus::Classifier.new   # Used to classify documents
+
+    lt.train 'language', 'Ruby is a dynamic, reflective, general-purpose object-oriented programming language that combines syntax inspired by Perl with Smalltalk-like features.'
+    lt.train 'database', 'PostgreSQL, often simply Postgres, is an object-relational database management system (ORDBMS) available for many platforms including Linux, FreeBSD, Solaris, Microsoft Windows and Mac OS X.'
+
+    lc.classify 'Perl is a high-level, general-purpose, interpreted, dynamic programming language.' # returns "language"
+
+
+Contributing to linnaeus
+------------------------
+
+* Submit bugs to the github issue tracker: https://github.com/djcp/linnaeus/issues
+* If you'd like to add a feature, please submit a description of it to the issue tracker so we can discuss.
+* If the feature makes sense, fork the github repository. Write rspec tests and issue a pull request when your change is done.
+
+The Future
+----------
+
+* Create additional storage backends - sqlite, postgresql, mongodb, etc.
+* Allow for weighting tweaks.
+
+Copyright
+---------
+
+Copyright (c) 2012 Dan Collis-Puro. See LICENSE.txt for further details.
+
+Credits
+-------
+
+* Image courtesy wikipedia. About Carl Linnaeus: http://en.wikipedia.org/wiki/Linnaeus

data/VERSION

@@ -1 +1 @@
-1.0.2
+1.0.3

data/lib/linnaeus/classifier.rb

@@ -57,7 +57,12 @@ class Linnaeus::Classifier < Linnaeus
   # == Returns
   # A string representing the most likely category.
   def classify(text)
-    (classification_scores(text).sort_by { |a| -a[1] })[0][0]
+    scores = classification_scores(text)
+    if scores.any?
+      (scores.sort_by { |a| -a[1] })[0][0]
+    else
+      ''
+    end
   end
 
 end

data/lib/linnaeus/persistence.rb

@@ -116,7 +116,9 @@ class Linnaeus::Persistence < Linnaeus
     if empty_words == word_counts
       @redis.del BASE_CATEGORY_KEY + category
     else
-      @redis.hdel BASE_CATEGORY_KEY + category, empty_words.keys
+      if empty_words.any?
+        @redis.hdel BASE_CATEGORY_KEY + category, empty_words.keys
+      end
     end
   end
 

data/lib/linnaeus.rb

@@ -44,7 +44,7 @@ class Linnaeus
   #   A string or array of categories
   def normalize_categories(categories = [])
     [categories].flatten.collect do |cat|
-      cat.to_s.downcase.gsub(/[^a-z\d\.\-_]/,'')
+      cat.to_s.encode(@encoding).downcase.gsub(/[^a-z\d\.\-_ ]/,'')
     end.reject{|cat| cat == ''}.compact
   end
 

data/linnaeus.gemspec

@@ -5,16 +5,16 @@
 
 Gem::Specification.new do |s|
   s.name = "linnaeus"
-  s.version = "1.0.2"
+  s.version = "1.0.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["djcp"]
-  s.date = "2012-10-31"
+  s.date = "2012-11-02"
   s.description = "Linnaeus provides a redis-backed Bayesian classifier. Words are stemmed, stopwords are stopped, and redis is used to allow for persistent and concurrent training and classification."
   s.email = "dan@collispuro.net"
   s.extra_rdoc_files = [
     "LICENSE.txt",
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".document",
@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
     "Gemfile",
     "Gemfile.lock",
     "LICENSE.txt",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "VERSION",
     "images/linnaeus.jpg",
@@ -58,6 +58,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
       s.add_development_dependency(%q<simplecov>, [">= 0"])
+      s.add_development_dependency(%q<redcarpet>, [">= 0"])
     else
       s.add_dependency(%q<redis>, ["~> 3.0.0"])
       s.add_dependency(%q<stemmer>, ["~> 1.0.0"])
@@ -67,6 +68,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
       s.add_dependency(%q<simplecov>, [">= 0"])
+      s.add_dependency(%q<redcarpet>, [">= 0"])
     end
   else
     s.add_dependency(%q<redis>, ["~> 3.0.0"])
@@ -77,6 +79,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
     s.add_dependency(%q<simplecov>, [">= 0"])
+    s.add_dependency(%q<redcarpet>, [">= 0"])
   end
 end
 

data/spec/linnaeus_classifier_spec.rb

@@ -1,12 +1,32 @@
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe Linnaeus::Classifier do
+  context 'with no training data' do
+    it 'should return empty values when attempting to classify' do
+      Linnaeus::Persistence.new.clear_all_training_data
+      subject.classify("foo bar baz").should be_empty
+      subject.classification_scores("foo bar baz").should be_empty
+    end
+  end
+
   context 'with a very small dataset' do
-    it 'should classify easy things well' do
+    before do
       create_small_dataset
+    end
+
+    it 'should classify easy things well' do
       subject.classify('A bird that migrates').should eq('bird')
       subject.classify('This was directed by Gus Van Sant').should eq('movie')
     end
+
+    it 'should return correct classification scores' do
+      subject.classification_scores('a bird').should eq(
+        { "movie"=>-6.272877006546167, "bird"=>-4.2626798770413155 }
+      )
+      subject.classification_scores('a directorial bird').should eq(
+        { "movie"=>-10.24316892009829, "bird"=>-10.827944847076676 }
+      )
+    end
   end
 
   def create_small_dataset

data/spec/linnaeus_persistence_spec.rb

@@ -1,22 +1,88 @@
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe Linnaeus::Persistence do
-  it 'stores categories successfully' do
-    lp = Linnaeus::Persistence.new
+  before do
+    lp = get_linnaeus_persistence
+    lp.clear_all_training_data
+  end
+
+  it '#clear_all_training_data' do
+    lp = get_linnaeus_persistence
+    train_a_document_in 'testcategory'
+    lp.get_words_with_count_for_category('testcategory').should_not be_empty
     lp.clear_all_training_data
+    lp.get_words_with_count_for_category('testcategory').should be_empty
+  end
+
+  it 'stores categories successfully' do
+    lp = get_linnaeus_persistence
     add_categories lp
     lp.get_categories.sort.should eq ['bar','baz','foo']
   end
 
   it 'can remove categories' do
-    lp = Linnaeus::Persistence.new
-    lp.clear_all_training_data
+    lp = get_linnaeus_persistence
     add_categories lp
     lp.remove_category 'bar'
     lp.get_categories.sort.should eq ['baz','foo']
   end
 
+  it '#get_words_with_count_for_category' do
+    lp = get_linnaeus_persistence
+    train_a_document_in 'testcategory'
+    lp.get_words_with_count_for_category('testcategory').should eq({
+      "test"=>"1", "document"=>"1", "stuff"=>"1",
+      "bayesian"=>"1", "corpu"=>"1"
+    })
+  end
+
+  it '#increment_word_counts_for_category' do
+    lp = get_linnaeus_persistence
+    train_a_document_in 'testcategory'
+    train_a_document_in 'testcategory'
+    lp.get_words_with_count_for_category('testcategory').should eq({
+      "test"=>"2", "document"=>"2", "stuff"=>"2",
+      "bayesian"=>"2", "corpu"=>"2"
+    })
+  end
+
+  it '#decrement_word_counts_for_category' do
+    lp = get_linnaeus_persistence
+    train_a_document_in 'testcategory'
+    train_a_document_in 'testcategory'
+    untrain_a_document_in 'testcategory'
+    lp.get_words_with_count_for_category('testcategory').should eq({
+      "test"=>"1", "document"=>"1", "stuff"=>"1",
+      "bayesian"=>"1", "corpu"=>"1"
+    })
+  end
+
+  it '#cleanup_empty_words_in_category' do
+    lp = get_linnaeus_persistence
+    train_a_document_in 'testcategory'
+    untrain_a_document_in 'testcategory'
+    lp.get_words_with_count_for_category('testcategory').should eq ({})
+  end
+
   def add_categories(lp)
     lp.add_categories(['foo','bar','baz','foo', 'bar'])
   end
+
+  def get_linnaeus_persistence
+    @lp ||= Linnaeus::Persistence.new
+  end
+
+  def train_a_document_in(category)
+    lt = Linnaeus::Trainer.new
+    lt.train category, document
+  end
+
+  def untrain_a_document_in(category)
+    lt = Linnaeus::Trainer.new
+    lt.untrain category, document
+  end
+
+  def document
+    'I am a test document and I will have stuff in the bayesian corpus'
+  end
 end

data/spec/linnaeus_trainer_spec.rb

@@ -52,7 +52,7 @@ describe Linnaeus::Trainer do
 
   context 'with non-default stopwords' do
     subject { Linnaeus::Trainer.new(stopwords_class: FooStop) }
-    it 'should count word occurrencs properly' do
+    it 'should count word occurrences properly' do
       subject.count_word_occurrences('foo bar foo baz').should == { 'baz' => 1 }
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: linnaeus
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-31 00:00:00.000000000 Z
+date: 2012-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -139,6 +139,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Linnaeus provides a redis-backed Bayesian classifier. Words are stemmed,
   stopwords are stopped, and redis is used to allow for persistent and concurrent
   training and classification.
@@ -147,7 +163,7 @@ executables: []
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
-- README.rdoc
+- README.md
 files:
 - .document
 - .rspec
@@ -155,7 +171,7 @@ files:
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
-- README.rdoc
+- README.md
 - Rakefile
 - VERSION
 - images/linnaeus.jpg
@@ -186,7 +202,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2735128872899197737
+      hash: 2790078718663664512
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/README.rdoc

@@ -1,35 +0,0 @@
-= Linnaeus
-
-https://raw.github.com/djcp/linnaeus/master/images/linnaeus.jpg
-
-Linnaeus is a redis-backed Bayesian classification system. Please see the generated rdocs for more information. Ruby 1.9 is required.
-
-== Examples
-
-  lt = Linnaeus::Trainer.new      # Used to train documents
-  lc = Linnaeus::Classifier.new   # Used to classify documents
-
-  lt.train 'language', 'Ruby is a dynamic, reflective, general-purpose object-oriented programming language that combines syntax inspired by Perl with Smalltalk-like features.'
-  lt.train 'database', 'PostgreSQL, often simply Postgres, is an object-relational database management system (ORDBMS) available for many platforms including Linux, FreeBSD, Solaris, Microsoft Windows and Mac OS X.'
-
-  lc.classify 'Perl is a high-level, general-purpose, interpreted, dynamic programming language.' # returns "language"
-
-
-== Contributing to linnaeus
-
-* Submit bugs to the github issue tracker: https://github.com/djcp/linnaeus/issues
-* If you'd like to add a feature, please submit a description of it to the issue tracker so we can discuss.
-* If the feature makes sense, fork the github repository. Write rspec tests and issue a pull request when your change is done.
-
-== The Future
-
-* Create additional storage backends - sqlite, postgresql, mongodb, etc.
-* Allow for weighting tweaks.
-
-== Copyright
-
-Copyright (c) 2012 Dan Collis-Puro. See LICENSE.txt for further details.
-
-== Credits
-
-* Image courtesy wikipedia. About Carl Linnaeus: http://en.wikipedia.org/wiki/Linnaeus