stuff-classifier 0.2 → 0.4

data/README.md

@@ -43,7 +43,16 @@ cls = StuffClassifier::Bayes.new("Cats or Dogs")
 
 # for the Tf-Idf based implementation
 cls = StuffClassifier::TfIdf.new("Cats or Dogs")
-```
+
+# these classifiers use word stemming by default, but if it has weird
+# behavior, then you can disable it on init:
+cls = StuffClassifier::TfIdf.new("Cats or Dogs", :stemming => false)
+
+# also by default, the parsing phase filters out stop words, to
+# disable or to come up with your own list of stop words, on a
+# classifier instance you can do this:
+cls.ignore_words = [ 'the', 'my', 'i', 'dont' ]
+ ```
 
 Training the classifier:
 
@@ -93,10 +102,46 @@ cls.classify("Who is eating my meat?")
 #=> :dog
 ```
 
-## TODO
+## Persistency
+
+2 persistency layers for saving the training data are implemented:
+
+- in memory (by default)
+- on disk
+
+To persist the data on disk, you can do this:
+
+```ruby
+store = StuffClassifier::FileStorage.new(@storage_path)
+
+# global setting
+StuffClassifier::Base.storage = store
 
-- provide more implementations
-- plugable storage mechanism (in-memory, on disk, database)
+# or alternative local setting on instantiation, by means of an
+# optional param ...
+cls = StuffClassifier::Bayes.new("Cats or Dogs", :storage => store)
+
+# after training is done, to persist the data ...
+cls.save_state
+
+# or you could just do this:
+StuffClassifier::Bayes.open("Cats or Dogs") do |cls|
+  # when done, save_state is called on END
+end
+
+# to start fresh, deleting the saved training data for this classifier
+StuffClassifier::Bayes.new("Cats or Dogs", :purge_state => true)
+```
+
+The name you give your classifier is important, as based on it the
+data will get loaded and saved. For instance, following 3 classifiers
+will be stored in different buckets, being independent of each other.
+
+```ruby
+cls1 = StuffClassifier::Bayes.new("Cats or Dogs")
+cls2 = StuffClassifier::Bayes.new("True or False")
+cls3 = StuffClassifier::Bayes.new("Spam or Ham")	
+```
 
 ## License
 

data/Rakefile

@@ -16,15 +16,15 @@ Rcov::RcovTask.new do |test|
   test.rcov_opts << '--exclude "gems/*"'
 end
 
-require 'rdoc/task'
-RDoc::Task.new do |rdoc|
-  version = StuffClassifier::VERSION
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "stuff-classifier #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
+#require 'rdoc/task'
+#RDoc::Task.new do |rdoc|
+#  version = StuffClassifier::VERSION
+#
+#  rdoc.rdoc_dir = 'rdoc'
+#  rdoc.title = "stuff-classifier #{version}"
+#  rdoc.rdoc_files.include('README*')
+#  rdoc.rdoc_files.include('lib/**/*.rb')
+#end
 
 task :default => :test
 

data/lib/stuff-classifier.rb

@@ -1,8 +1,12 @@
 module StuffClassifier
   autoload :VERSION,    'stuff-classifier/version'
   autoload :STOP_WORDS, 'stuff-classifier/stop_words'
+
   autoload :Tokenizer,  'stuff-classifier/tokenizer'
   autoload :Base,       'stuff-classifier/base'
   autoload :Bayes,      'stuff-classifier/bayes'
   autoload :TfIdf,      'stuff-classifier/tf-idf'
+
+  autoload :InMemoryStorage, 'stuff-classifier/storage'
+  autoload :FileStorage,     'stuff-classifier/storage'
 end

data/lib/stuff-classifier/base.rb

@@ -1,12 +1,22 @@
 class StuffClassifier::Base
   include StuffClassifier::Tokenizer
+  attr_reader :name
   
   def initialize(name, opts={})
-    @name = name
     @stemming = opts.key?(:stemming) ? opts[:stemming] : true
+    purge_state = opts[:purge_state]
+
+    @name = name
     @wcount = {}
     @ccount = {}
     @ignore_words = nil
+
+    @storage = opts[:storage] || StuffClassifier::Base.storage
+    unless purge_state
+      @storage.load_state(self)
+    else
+      @storage.purge_state(self)
+    end
   end
 
   def incr_word(word, category)
@@ -63,4 +73,27 @@ class StuffClassifier::Base
     # the final weighted average
     (weight * assumed_prob + totals * basic_prob) / (weight + totals)
   end
+
+  def save_state
+    @storage.save_state(self)
+  end
+
+  class << self
+    attr_writer :storage
+
+    def storage
+      @storage = StuffClassifier::InMemoryStorage.new unless defined? @storage
+      @storage
+    end
+
+    def open(name)
+      inst = self.new(name)
+      if block_given?
+        yield inst
+        inst.save_state
+      else
+        inst
+      end
+    end
+  end
 end

data/lib/stuff-classifier/bayes.rb

@@ -2,6 +2,8 @@
 class StuffClassifier::Bayes < StuffClassifier::Base
   # http://en.wikipedia.org/wiki/Naive_Bayes_classifier
 
+  attr_writer :thresholds
+
   def initialize(name, opts={})
     super(name, opts)
     @thresholds = {}

data/lib/stuff-classifier/stop_words.rb

@@ -1,3 +1,5 @@
+require 'set'
+
 StuffClassifier::STOP_WORDS = Set.new [
     'a', 'about', 'above', 'across', 'after', 'afterwards', 
     'again', 'against', 'all', 'almost', 'alone', 'along', 

data/lib/stuff-classifier/storage.rb

@@ -0,0 +1,71 @@
+require 'msgpack'
+
+module StuffClassifier
+  class InMemoryStorage
+    def initialize
+      @storage = {}
+    end
+
+    def load_state(classifier)
+      if @storage.key? classifier.name
+        _wcount, _ccount = @storage[classifier.name]
+        classifier.instance_eval do
+          @wcount = _wcount
+          @ccount = _ccount
+        end
+      end
+    end
+
+    def save_state(classifier)
+      name = classifier.name
+      wcount = classifier.instance_variable_get :@wcount
+      ccount = classifier.instance_variable_get :@ccount
+      @storage[name] = [wcount, ccount]
+    end
+
+    def purge_state(classifier)
+      @storage.delete(classifier.name)
+    end
+  end
+
+  class FileStorage
+    def initialize(path)
+      @storage = {}
+      @path = path
+    end
+
+    def load_state(classifier)
+      if @storage.length == 0 && File.exists?(@path)
+        @storage = MessagePack.unpack(File.read(@path))
+      end
+
+      if @storage.key? classifier.name
+        _wcount, _ccount = @storage[classifier.name]
+        classifier.instance_eval do
+          @wcount = _wcount
+          @ccount = _ccount
+        end
+      end
+    end
+
+    def save_state(classifier)
+      name = classifier.name
+      wcount = classifier.instance_variable_get :@wcount
+      ccount = classifier.instance_variable_get :@ccount
+      @storage[name] = [wcount, ccount]
+      _write_to_file
+    end
+
+    def purge_state(classifier)
+      @storage.delete(classifier.name)
+      _write_to_file
+    end
+
+    def _write_to_file
+      File.open(@path, 'w') do |fh|
+        fh.flock(File::LOCK_EX)
+        fh.write(@storage.to_msgpack)
+      end
+    end
+  end
+end

data/lib/stuff-classifier/tokenizer.rb

@@ -1,6 +1,7 @@
 require 'fast_stemmer'
 
 module StuffClassifier::Tokenizer
+  attr_writer :stemming
 
   def ignore_words=(value)
     @ignore_words = value
@@ -14,10 +15,6 @@ module StuffClassifier::Tokenizer
     defined?(@stemming) ? @stemming : false
   end
 
-  def stemming=(value)
-    @stemming = value
-  end
-
   def each_word(string)
     string = string.strip
     return if string == ''
@@ -37,8 +34,7 @@ module StuffClassifier::Tokenizer
           w = w.downcase
         end
 
-        yield w if block_given?
-        words << w
+        words << (block_given? ? (yield w) : w)
       end
     end
 

data/lib/stuff-classifier/version.rb

@@ -1,3 +1,3 @@
 module StuffClassifier
-  VERSION = '0.2'
+  VERSION = '0.4'
 end

data/stuff-classifier.gemspec

@@ -17,7 +17,11 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_runtime_dependency "fast-stemmer", ">= 1.0"
+  s.add_runtime_dependency "sqlite3"
+  s.add_runtime_dependency "sequel"
+  s.add_runtime_dependency "msgpack"
 
+  s.add_development_dependency "ruby-debug19"
   s.add_development_dependency "bundler"
   s.add_development_dependency "rake", ">= 0.9.2"
   s.add_development_dependency "minitest", ">= 2.10"

data/test/test_004_in_memory_storage.rb

@@ -0,0 +1,31 @@
+require 'helper'
+
+
+class Test004InMemoryStorage < TestBase
+  before do
+    StuffClassifier::Base.storage = StuffClassifier::InMemoryStorage.new
+
+    StuffClassifier::Bayes.open("Cats or Dogs") do |cls|    
+      cls.train(:dog, "Dogs are awesome, cats too. I love my dog")
+      cls.train(:cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog")
+    end
+  end
+
+  def test_for_persistance
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs").instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::InMemoryStorage),
+        "@storage should be an instance of FileStorage"
+      test.assert @wcount.length > 0, "Word count should be persisted"
+      test.assert @ccount.length > 0, "Category count should be persisted"
+    end
+  end
+
+  def test_purge_state
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs", :purge_state => true).instance_eval do
+      test.assert @wcount.length == 0, "Word count should be purged"
+      test.assert @ccount.length == 0, "Category count should be purged"
+    end
+  end
+end

data/test/test_005_file_storage.rb

@@ -0,0 +1,54 @@
+require 'helper'
+
+
+class Test005FileStorage < TestBase
+  before do
+    @storage_path = "/tmp/test_classifier.db"
+    @storage = StuffClassifier::FileStorage.new(@storage_path)
+    StuffClassifier::Base.storage = @storage
+
+    StuffClassifier::Bayes.open("Cats or Dogs") do |cls|    
+      cls.train(:dog, "Dogs are awesome, cats too. I love my dog")
+      cls.train(:cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog")
+    end
+
+    # redefining storage instance, forcing it to read from file again
+    StuffClassifier::Base.storage = StuffClassifier::FileStorage.new(@storage_path)
+  end
+
+  def teardown
+    File.unlink @storage_path if File.exists? @storage_path
+  end
+
+  def test_for_persistance    
+    assert ! @storage.equal?(StuffClassifier::Base.storage),
+      "Storage instance should not be the same"
+
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs").instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::FileStorage),
+        "@storage should be an instance of FileStorage"
+      test.assert @wcount.length > 0, "Word count should be persisted"
+      test.assert @ccount.length > 0, "Category count should be persisted"
+    end
+  end
+
+  def test_file_created
+    assert File.exist?(@storage_path), 
+      "File #@storage_path should exist"
+
+    content = File.read(@storage_path)
+    assert content.length > 100, 
+      "Serialized content should have more than 100 chars"
+  end
+
+  def test_purge_state
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs", :purge_state => true).instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::FileStorage),
+        "@storage should be an instance of FileStorage"
+      test.assert @wcount.length == 0, "Word count should be purged"
+      test.assert @ccount.length == 0, "Category count should be purged"
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stuff-classifier
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: '0.4'
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-19 00:00:00.000000000Z
+date: 2012-01-20 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast-stemmer
-  requirement: &77488640 !ruby/object:Gem::Requirement
+  requirement: &77637680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,54 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *77488640
+  version_requirements: *77637680
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: &77637470 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *77637470
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: &77637240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *77637240
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: &77637030 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *77637030
+- !ruby/object:Gem::Dependency
+  name: ruby-debug19
+  requirement: &77636820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *77636820
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &77549150 !ruby/object:Gem::Requirement
+  requirement: &77636610 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *77549150
+  version_requirements: *77636610
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &77548880 !ruby/object:Gem::Requirement
+  requirement: &77636360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +87,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *77548880
+  version_requirements: *77636360
 - !ruby/object:Gem::Dependency
   name: minitest
-  requirement: &77548630 !ruby/object:Gem::Requirement
+  requirement: &77636110 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +98,10 @@ dependencies:
         version: '2.10'
   type: :development
   prerelease: false
-  version_requirements: *77548630
+  version_requirements: *77636110
 - !ruby/object:Gem::Dependency
   name: turn
-  requirement: &77548400 !ruby/object:Gem::Requirement
+  requirement: &77635880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +109,10 @@ dependencies:
         version: 0.8.3
   type: :development
   prerelease: false
-  version_requirements: *77548400
+  version_requirements: *77635880
 - !ruby/object:Gem::Dependency
   name: rcov
-  requirement: &77548170 !ruby/object:Gem::Requirement
+  requirement: &77635650 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +120,7 @@ dependencies:
         version: '0.9'
   type: :development
   prerelease: false
-  version_requirements: *77548170
+  version_requirements: *77635650
 description: 2 methods are provided for now - (1) naive bayes implementation + (2)
   tf-idf weights
 email:
@@ -85,9 +129,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -95,15 +137,17 @@ files:
 - lib/stuff-classifier/base.rb
 - lib/stuff-classifier/bayes.rb
 - lib/stuff-classifier/stop_words.rb
+- lib/stuff-classifier/storage.rb
 - lib/stuff-classifier/tf-idf.rb
 - lib/stuff-classifier/tokenizer.rb
 - lib/stuff-classifier/version.rb
-- stuff-classifier-0.1.gem
 - stuff-classifier.gemspec
 - test/helper.rb
 - test/test_001_tokenizer.rb
 - test/test_002_naive_bayes.rb
 - test/test_003_tf_idf.rb
+- test/test_004_in_memory_storage.rb
+- test/test_005_file_storage.rb
 homepage: https://github.com/alexandru/stuff-classifier/
 licenses: []
 post_install_message: 

data/.gitignore

@@ -1,48 +0,0 @@
-# rcov generated
-coverage
-
-# rdoc generated
-rdoc
-
-# yard generated
-doc
-.yardoc
-
-# bundler
-.bundle
-
-# jeweler generated
-pkg
-
-# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
-#
-# * Create a file at ~/.gitignore
-# * Include files you want ignored
-# * Run: git config --global core.excludesfile ~/.gitignore
-#
-# After doing this, these files will be ignored in all your git projects,
-# saving you from having to 'pollute' every project you touch with them
-#
-# Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
-#
-# For MacOS:
-#
-#.DS_Store
-
-# For TextMate
-#*.tmproj
-#tmtags
-
-# For emacs:
-#*~
-#\#*
-#.\#*
-
-# For vim:
-#*.swp
-
-# For redcar:
-#.redcar
-
-# For rubinius:
-#*.rbc

data/Gemfile.lock

@@ -1,46 +0,0 @@
-PATH
-  remote: .
-  specs:
-    stuff-classifier (0.1)
-      fast-stemmer (>= 1.0)
-
-GEM
-  remote: http://rubygems.org/
-  specs:
-    ansi (1.4.1)
-    archive-tar-minitar (0.5.2)
-    columnize (0.3.4)
-    fast-stemmer (1.0.0)
-    json (1.6.5)
-    linecache19 (0.5.12)
-      ruby_core_source (>= 0.1.4)
-    minitest (2.10.1)
-    rake (0.9.2.2)
-    rcov (0.9.11)
-    rdoc (3.12)
-      json (~> 1.4)
-    ruby-debug-base19 (0.11.25)
-      columnize (>= 0.3.1)
-      linecache19 (>= 0.5.11)
-      ruby_core_source (>= 0.1.4)
-    ruby-debug19 (0.11.6)
-      columnize (>= 0.3.1)
-      linecache19 (>= 0.5.11)
-      ruby-debug-base19 (>= 0.11.19)
-    ruby_core_source (0.1.5)
-      archive-tar-minitar (>= 0.5.2)
-    turn (0.8.3)
-      ansi
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler
-  minitest (>= 2.10)
-  rake (>= 0.9.2)
-  rcov (>= 0.9)
-  rdoc (>= 3.1)
-  ruby-debug19
-  stuff-classifier!
-  turn (>= 0.8.3)