linnaeus 1.0.3 → 1.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b380bea103cd6825850436184c4ab35c9c07c089
+  data.tar.gz: 147508ab8b2610250a82acf5d11721040d9c721d
+SHA512:
+  metadata.gz: 12ef0f679970ad545fc3942f26a709626ed43647bbeb454656fb39c908691434746dd3cf21c3cc658be534a814c73aa5572af1e1c466c1f89048d326516dc97f
+  data.tar.gz: d982a11641d6be711117b270850d7309881ad74b4e5278f484bced8f1703a36bd058e7de093883f520916c378eba353ae3c01b331461301f41139699657bed8a

data/.rspec

@@ -1 +1,3 @@
---color
+--colour
+--order rand
+-f d

data/.ruby-version

@@ -0,0 +1 @@
+2.0.0-p0

data/.travis.yml

@@ -1,5 +1,6 @@
 language: ruby
 rvm:
+  - 2.0.0
   - 1.9.3
   - 1.9.2
 services:

data/Gemfile.lock

@@ -1,20 +1,51 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    addressable (2.3.5)
+    builder (3.2.2)
     diff-lcs (1.1.3)
+    faraday (0.8.7)
+      multipart-post (~> 1.1)
     git (1.2.5)
-    jeweler (1.8.4)
+    github_api (0.10.1)
+      addressable
+      faraday (~> 0.8.1)
+      hashie (>= 1.2)
+      multi_json (~> 1.4)
+      nokogiri (~> 1.5.2)
+      oauth2
+    hashie (2.0.5)
+    highline (1.6.19)
+    httpauth (0.2.0)
+    jeweler (1.8.6)
+      builder
       bundler (~> 1.0)
       git (>= 1.2.5)
+      github_api (= 0.10.1)
+      highline (>= 1.6.15)
+      nokogiri (= 1.5.10)
       rake
       rdoc
-    json (1.7.5)
-    multi_json (1.3.6)
-    rake (0.9.2.2)
-    rdoc (3.12)
+    json (1.8.0)
+    jwt (0.1.8)
+      multi_json (>= 1.5)
+    multi_json (1.7.7)
+    multi_xml (0.5.4)
+    multipart-post (1.2.0)
+    nokogiri (1.5.10)
+    oauth2 (0.9.2)
+      faraday (~> 0.8)
+      httpauth (~> 0.2)
+      jwt (~> 0.1.4)
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.5.2)
+    rake (10.1.0)
+    rdoc (3.12.2)
       json (~> 1.4)
-    redcarpet (2.2.2)
-    redis (3.0.2)
+    redcarpet (3.0.0)
+    redis (3.0.4)
     rspec (2.11.0)
       rspec-core (~> 2.11.0)
       rspec-expectations (~> 2.11.0)
@@ -28,7 +59,7 @@ GEM
       simplecov-html (~> 0.7.1)
     simplecov-html (0.7.1)
     stemmer (1.0.1)
-    yard (0.8.3)
+    yard (0.8.6.2)
 
 PLATFORMS
   ruby

data/VERSION

@@ -1 +1 @@
-1.0.3
+1.1.0

data/lib/linnaeus/persistence.rb

@@ -1,10 +1,5 @@
 # The redis persistence layer.
 class Linnaeus::Persistence < Linnaeus
-  # The Set (in the redis sense) of categories are stored in this key.
-  CATEGORIES_KEY = 'Linnaeus:category'
-  # The base key for a category in the redis corpus. Word occurrence counts for a category appear under here.
-  BASE_CATEGORY_KEY = 'Linnaeus:cat:'
-
   attr_accessor :redis
 
   def initialize(opts = {})
@@ -17,9 +12,12 @@ class Linnaeus::Persistence < Linnaeus
       redis_timeout: 5.0,
       redis_password: nil,
       redis_id: nil,
-      redis_tcp_keepalive: 0
+      redis_tcp_keepalive: 0,
+      scope: nil
     }.merge(opts)
 
+    @scope = options[:scope]
+
     @redis = Redis.new(
       host: options[:redis_host],
       port: options[:redis_port],
@@ -41,7 +39,7 @@ class Linnaeus::Persistence < Linnaeus
   # categories::
   #   A string or array of categories.
   def add_categories(categories)
-    @redis.sadd CATEGORIES_KEY, categories
+    @redis.sadd category_collection_key, categories
   end
 
   # Remove categories from the bayesian corpus
@@ -50,7 +48,7 @@ class Linnaeus::Persistence < Linnaeus
   # categories::
   #   A string or array of categories.
   def remove_category(category)
-    @redis.srem CATEGORIES_KEY, category
+    @redis.srem category_collection_key, category
   end
 
   # Get categories from the bayesian corpus
@@ -59,7 +57,7 @@ class Linnaeus::Persistence < Linnaeus
   # categories::
   #   A string or array of categories.
   def get_categories
-    @redis.smembers CATEGORIES_KEY
+    @redis.smembers category_collection_key
   end
 
   # Get a list of words with their number of occurrences.
@@ -71,7 +69,7 @@ class Linnaeus::Persistence < Linnaeus
   # == Returns
   # A hash with the word counts for this category.
   def get_words_with_count_for_category(category)
-    @redis.hgetall BASE_CATEGORY_KEY + category
+    @redis.hgetall base_category_key + category
   end
 
   # Clear all training data from the backend.
@@ -79,6 +77,15 @@ class Linnaeus::Persistence < Linnaeus
     @redis.flushdb
   end
 
+  # Clear training data for the scope associated with this instance.
+  def clear_training_data
+     keys = @redis.keys(base_key.join(':') + '*')
+
+     keys.each do |key|
+       @redis.del key
+     end
+  end
+
   # Increment word counts within a category
   #
   # == Parameters
@@ -88,7 +95,7 @@ class Linnaeus::Persistence < Linnaeus
   #   A hash containing a count of the number of word occurences in a document
   def increment_word_counts_for_category(category, word_occurrences)
     word_occurrences.each do|word,count|
-      @redis.hincrby BASE_CATEGORY_KEY + category, word, count
+      @redis.hincrby base_category_key + category, word, count
     end
   end
 
@@ -101,7 +108,7 @@ class Linnaeus::Persistence < Linnaeus
   #   A hash containing a count of the number of word occurences in a document
   def decrement_word_counts_for_category(category, word_occurrences)
     word_occurrences.each do|word,count|
-      @redis.hincrby BASE_CATEGORY_KEY + category, word, - count
+      @redis.hincrby base_category_key + category, word, - count
     end
   end
 
@@ -111,15 +118,32 @@ class Linnaeus::Persistence < Linnaeus
   # category::
   #   A string representing a category.
   def cleanup_empty_words_in_category(category)
-    word_counts = @redis.hgetall BASE_CATEGORY_KEY + category
+    word_counts = @redis.hgetall base_category_key + category
     empty_words = word_counts.select{|word, count| count.to_i <= 0}
     if empty_words == word_counts
-      @redis.del BASE_CATEGORY_KEY + category
+      @redis.del base_category_key + category
     else
       if empty_words.any?
-        @redis.hdel BASE_CATEGORY_KEY + category, empty_words.keys
+        @redis.hdel base_category_key + category, empty_words.keys
       end
     end
   end
 
+  private
+
+  # The Set (in the redis sense) of categories are stored in this key.
+  def category_collection_key
+    [ base_key, 'category' ].compact.join(':')
+  end
+
+  # The base key for a category within a scope in the redis corpus. Word
+  # occurrence counts for a category appear under here.
+  def base_category_key
+    [ base_key, 'cat:' ].flatten.join(':')
+  end
+
+  def base_key
+    [ 'Linnaeus', @scope ].compact
+  end
+
 end

data/linnaeus.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "linnaeus"
-  s.version = "1.0.3"
+  s.version = "1.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["djcp"]
-  s.date = "2012-11-02"
+  s.date = "2013-07-12"
   s.description = "Linnaeus provides a redis-backed Bayesian classifier. Words are stemmed, stopwords are stopped, and redis is used to allow for persistent and concurrent training and classification."
   s.email = "dan@collispuro.net"
   s.extra_rdoc_files = [
@@ -19,6 +19,7 @@ Gem::Specification.new do |s|
   s.files = [
     ".document",
     ".rspec",
+    ".ruby-version",
     ".travis.yml",
     "Gemfile",
     "Gemfile.lock",
@@ -43,11 +44,11 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/djcp/linnaeus"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.24"
+  s.rubygems_version = "2.0.0"
   s.summary = "Another redis-backed Bayesian classifier"
 
   if s.respond_to? :specification_version then
-    s.specification_version = 3
+    s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<redis>, ["~> 3.0.0"])

data/spec/linnaeus_persistence_spec.rb

@@ -3,7 +3,58 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 describe Linnaeus::Persistence do
   before do
     lp = get_linnaeus_persistence
-    lp.clear_all_training_data
+    lp.clear_training_data
+  end
+
+  it 'sets keys properly with defaults' do
+    lp2 = get_linnaeus_persistence
+    train_a_document_in('foobar')
+    lp2.redis.keys('*').should eq ['Linnaeus:category', 'Linnaeus:cat:foobar']
+  end
+
+  context "custom scopes" do
+    it 'sets keys properly' do
+      lp2 = get_linnaeus_persistence(scope: 'new-scope')
+      lp2.clear_all_training_data
+
+      train_a_document_in('foobar', scope: 'new-scope')
+
+      lp2.redis.keys('*').sort.should eq [
+        'Linnaeus:new-scope:cat:foobar', 'Linnaeus:new-scope:category'
+      ]
+    end
+
+    it 'can clear scoped training data separately' do
+      lp = get_linnaeus_persistence
+
+      train_a_document_in('foobar')
+
+      lp2 = get_linnaeus_persistence(scope: 'new-scope')
+
+      train_a_document_in('foobar', scope: 'new-scope')
+
+      lp.redis.keys.sort.should eq [
+        "Linnaeus:cat:foobar", "Linnaeus:category",
+        "Linnaeus:new-scope:cat:foobar", "Linnaeus:new-scope:category"
+      ]
+
+      lp2.clear_training_data
+
+      lp.redis.keys.sort.should eq [
+        "Linnaeus:cat:foobar", "Linnaeus:category"
+      ]
+    end
+
+    it 'stores categories successfully into different scopes' do
+      lp = get_linnaeus_persistence
+      add_categories lp
+
+      lp2 = get_linnaeus_persistence(scope: 'new-scope')
+      add_categories lp2, ['slack' , 'frop']
+
+      lp2.get_categories.sort.should eq ['frop', 'slack']
+      lp.get_categories.sort.should eq ['bar','baz','foo']
+    end
   end
 
   it '#clear_all_training_data' do
@@ -64,21 +115,21 @@ describe Linnaeus::Persistence do
     lp.get_words_with_count_for_category('testcategory').should eq ({})
   end
 
-  def add_categories(lp)
-    lp.add_categories(['foo','bar','baz','foo', 'bar'])
+  def add_categories(lp, categories = ['foo','bar','baz','foo', 'bar'])
+    lp.add_categories(categories)
   end
 
-  def get_linnaeus_persistence
-    @lp ||= Linnaeus::Persistence.new
+  def get_linnaeus_persistence(options = {})
+    Linnaeus::Persistence.new(options)
   end
 
-  def train_a_document_in(category)
-    lt = Linnaeus::Trainer.new
+  def train_a_document_in(category, options = {})
+    lt = Linnaeus::Trainer.new(options)
     lt.train category, document
   end
 
-  def untrain_a_document_in(category)
-    lt = Linnaeus::Trainer.new
+  def untrain_a_document_in(category, options = {})
+    lt = Linnaeus::Trainer.new(options)
     lt.untrain category, document
   end
 

data/spec/linnaeus_trainer_spec.rb

@@ -17,17 +17,8 @@ describe Linnaeus::Trainer do
       subject.count_word_occurrences.should == { }
     end
 
-    it 'should train on documents properly' do
-      lp = Linnaeus::Persistence.new
-      lp.clear_all_training_data
-      subject.train 'fruit', grape
-      subject.train 'fruit', orange
-      lp.get_words_with_count_for_category('fruit').should eq(
-        {
-        "grape"=>"1", "purpl"=>"1", "blue"=>"1", "green"=>"1", 
-        "fruit"=>"2", "sweet"=>"2", "wine"=>"1", "oval"=>"1", 
-        "orang"=>"1", "round"=>"1", "citru"=>"1"
-      })
+    it 'should train documents properly' do
+      train_documents_properly
     end
 
     it 'should partially untrain properly' do
@@ -57,6 +48,26 @@ describe Linnaeus::Trainer do
     end
   end
 
+  context 'with a custom scope' do
+    it 'should train on documents properly' do
+      train_documents_properly(scope: 'new-scope')
+    end
+  end
+
+  def train_documents_properly(options = {})
+    lp = Linnaeus::Persistence.new(options)
+    lp.clear_all_training_data
+    subject = described_class.new(options)
+    subject.train 'fruit', grape
+    subject.train 'fruit', orange
+    lp.get_words_with_count_for_category('fruit').should eq(
+      {
+      "grape"=>"1", "purpl"=>"1", "blue"=>"1", "green"=>"1",
+      "fruit"=>"2", "sweet"=>"2", "wine"=>"1", "oval"=>"1",
+      "orang"=>"1", "round"=>"1", "citru"=>"1"
+    })
+  end
+
   def grape
     'grape purple blue green fruit sweet wine oval'
   end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: linnaeus
 version: !ruby/object:Gem::Version
-  version: 1.0.3
-  prerelease: 
+  version: 1.1.0
 platform: ruby
 authors:
 - djcp
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-02 00:00:00.000000000 Z
+date: 2013-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: stemmer
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -94,65 +83,57 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: jeweler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: redcarpet
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Linnaeus provides a redis-backed Bayesian classifier. Words are stemmed,
@@ -167,6 +148,7 @@ extra_rdoc_files:
 files:
 - .document
 - .rspec
+- .ruby-version
 - .travis.yml
 - Gemfile
 - Gemfile.lock
@@ -190,29 +172,25 @@ files:
 homepage: http://github.com/djcp/linnaeus
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 2790078718663664512
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Another redis-backed Bayesian classifier
 test_files: []