green_midget 0.0.3 → 0.1.0

data/spec/examples_spec.rb

@@ -5,8 +5,8 @@ describe GreenMidget::Examples do
   include GreenMidget
 
   before(:each) do
-    GreenMidgetRecords.delete_all
-    GreenMidgetRecords.class_variable_set("@@cache", {})
+    Records.delete_all
+    Records.cache = {}
   end
 
   describe "#[]()" do
@@ -17,17 +17,17 @@ describe GreenMidget::Examples do
     end
 
     it "should return the general feature examples if passed a (new) feature key that has no examples yet" do
-      GreenMidgetRecords.create(:key => Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ NULL }_count",        :value => 1000)
-      GreenMidgetRecords.create(:key => Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ ALTERNATIVE }_count", :value => 1000)
-      GreenMidgetRecords.find_by_key(Examples.prefix + "new::#{ NULL }_count").should == nil
-      GreenMidgetRecords.fetch_all
+      Records.create(:key => Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ NULL }_count",        :value => 1000)
+      Records.create(:key => Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ ALTERNATIVE }_count", :value => 1000)
+      Records.find_by_key(Examples.prefix + "new::#{ NULL }_count").should == nil
+      Records.fetch_all
       CATEGORIES.each do |category|
         Examples['new'][category].should == Examples[Examples::GENERAL_FEATURE_NAME][category]
       end
     end
     it "should return the feature's own example counts if these exist" do
-      GreenMidgetRecords.create(:key => Examples.prefix + "new::#{ NULL }_count",        :value => 3)
-      GreenMidgetRecords.create(:key => Examples.prefix + "new::#{ ALTERNATIVE }_count", :value => 1)
+      Records.create(:key => Examples.prefix + "new::#{ NULL }_count",        :value => 3)
+      Records.create(:key => Examples.prefix + "new::#{ ALTERNATIVE }_count", :value => 1)
       Examples['new'][NULL].should  == 3
     end
 
@@ -36,34 +36,34 @@ describe GreenMidget::Examples do
     end
 
     it "should throw an error if the general feature examples has a zero spam_count and ham_count" do
-      GreenMidgetRecords.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ NULL }_count")
+      Records.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ NULL }_count")
       @call_any.should raise_error
     end
 
     it "should throw an error if the general feature examples has a zero spam_count or ham_count" do
-      GreenMidgetRecords.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ NULL }_count", :value => 0)
+      Records.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ NULL }_count", :value => 0)
       @call_any.should raise_error
     end
 
     it "should not throw an error if both columns are positive" do
-      GreenMidgetRecords.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ NULL }_count",        :value => 2)
-      GreenMidgetRecords.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ ALTERNATIVE }_count", :value => 1)
+      Records.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ NULL }_count",        :value => 2)
+      Records.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ ALTERNATIVE }_count", :value => 1)
       @call_any.should_not raise_error
     end
   end
 
   describe "#probability_for" do
     it "should return the probability of a feature falling into category as: Examples[feature][category] / (Examples[feature][ALTERNATIVE] + Examples[feature][NULL])" do
-      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(NULL),        :value => 1000)
-      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(ALTERNATIVE), :value => 150 )
+      Records.create(:key => Examples['url_in_text'].record_key(NULL),        :value => 1000)
+      Records.create(:key => Examples['url_in_text'].record_key(ALTERNATIVE), :value => 150 )
       Examples['url_in_text'].probability_for(ALTERNATIVE).should == 150.0/(1000 + 150)
     end
   end
 
   describe "#no_examples?" do
     before(:each) do
-      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(ALTERNATIVE))
-      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(NULL))
+      Records.create(:key => Examples['url_in_text'].record_key(ALTERNATIVE))
+      Records.create(:key => Examples['url_in_text'].record_key(NULL))
       @object = Examples['url_in_text']
     end
 
@@ -72,13 +72,13 @@ describe GreenMidget::Examples do
     end
 
     it "should return true if spam_count or ham_count are zero" do
-      GreenMidgetRecords.find_by_key(@object.record_key(NULL)).update_attribute(:value, 1)
+      Records.find_by_key(@object.record_key(NULL)).update_attribute(:value, 1)
       @object.no_examples?.should be_true
     end
 
     it "should should return false if both spam_count and ham_count are positive" do
-      GreenMidgetRecords.find_by_key(@object.record_key(NULL)).update_attribute(:value, 1)
-      GreenMidgetRecords.find_by_key(@object.record_key(ALTERNATIVE)).update_attribute(:value, 1)
+      Records.find_by_key(@object.record_key(NULL)).update_attribute(:value, 1)
+      Records.find_by_key(@object.record_key(ALTERNATIVE)).update_attribute(:value, 1)
       @object.no_examples?.should be_false
     end
   end

data/spec/features_spec.rb

@@ -5,17 +5,17 @@ describe GreenMidget::Features do
   include GreenMidget
 
   before(:each) do
-    GreenMidgetRecords.delete_all
-    GreenMidgetRecords.class_variable_set("@@cache", {})
+    Records.delete_all
+    Records.cache = {}
   end
 
   describe "#probability_for" do
     it "should return Feature[feature] / Examples[feature]" do
-      GreenMidgetRecords.create(:key => Features["url_in_text"].record_key(NULL),        :value => 20  )
-      GreenMidgetRecords.create(:key => Features["url_in_text"].record_key(ALTERNATIVE), :value => 10  )
+      Records.create(:key => Features["url_in_text"].record_key(NULL),        :value => 20  )
+      Records.create(:key => Features["url_in_text"].record_key(ALTERNATIVE), :value => 10  )
 
-      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(NULL),        :value => 100 )
-      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(ALTERNATIVE), :value => 1000)
+      Records.create(:key => Examples['url_in_text'].record_key(NULL),        :value => 100 )
+      Records.create(:key => Examples['url_in_text'].record_key(ALTERNATIVE), :value => 1000)
 
       Features['url_in_text'].probability_for(NULL).should == 20.0/100
       Features['url_in_text'].probability_for(ALTERNATIVE).should  == 10.0/1000

data/spec/green_midget_records_spec.rb

@@ -1,64 +1,69 @@
 # Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
 require 'spec_helper'
 
-describe GreenMidget::GreenMidgetRecords do
+describe GreenMidget::Records do
   include GreenMidget
 
   before(:each) do
-    GreenMidgetRecords.delete_all
+    Records.delete_all
   end
 
   describe "#[]()" do
     it "should take words from data store if not found in the cache" do
       word_key, phrase_key = [ 'word', 'phrase' ].map { |w| Words[w].record_key(NULL) }
-      GreenMidgetRecords.fetch_all([ 'word' ])
-      GreenMidgetRecords.create(:key => phrase_key)
-      GreenMidgetRecords.find_by_key(word_key).should == nil
-      GreenMidgetRecords.find_by_key(phrase_key).should_not == nil
-      GreenMidgetRecords[phrase_key].should == ''
+      Records.fetch_all([ 'word' ])
+      Records.create(:key => phrase_key)
+      Records.find_by_key(word_key).should == nil
+      Records.find_by_key(phrase_key).should_not == nil
+      Records[phrase_key].should == ''
     end
+
     it "should add a {key => ''} to the cache if key not found in cache and in the data store" do
       key = Words['nonexisting'].record_key(NULL)
-      GreenMidgetRecords[key].should == ''
-      GreenMidgetRecords.find_by_key(key).should == nil
+      Records[key].should == ''
+      Records.find_by_key(key).should == nil
     end
   end
 
   describe "#fetch_all" do
     it "should empty cache before fetching" do
       bar_key = Words['bar'].record_key(ALTERNATIVE)
-      GreenMidgetRecords.fetch_all([ 'foo', 'bar' ])
-      GreenMidgetRecords.class_variable_get("@@cache").key?(bar_key).should be_true
-      GreenMidgetRecords.fetch_all([ 'foo', 'newbar' ])
-      GreenMidgetRecords.class_variable_get("@@cache").key?(bar_key).should be_false
+      Records.fetch_all([ 'foo', 'bar' ])
+      Records.cache.key?(bar_key).should be_true
+      Records.fetch_all([ 'foo', 'newbar' ])
+      Records.cache.key?(bar_key).should be_false
     end
+
     it "does a multi get on all words and keys" do
-      cache = GreenMidgetRecords.fetch_all([ 'foo', 'bar' ])
-      cache['foo'].should.eql? GreenMidgetRecords.class_eval{new('foo')}
+      cache = Records.fetch_all([ 'foo', 'bar' ])
+      cache['foo'].should.eql? Records.class_eval{new(:key => 'foo')}
     end
+
     it "should fetch the system keys along with the given words" do
       key = Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ NULL }_count"
-      GreenMidgetRecords.create(:key => key)
-      GreenMidgetRecords.fetch_all([])
-      cache = GreenMidgetRecords.class_variable_get("@@cache")
+      Records.create(:key => key)
+      Records.fetch_all([])
+      cache = Records.cache
       cache.key?(key).should be_true
       cache.count.should == 1
     end
+
     it "words with zero examples or no record in the database should be present in the cache" do
-      GreenMidgetRecords.create(:key => Words['kotoba'].record_key(NULL))
-      GreenMidgetRecords.fetch_all(['kotoba'])
-      GreenMidgetRecords.class_variable_get("@@cache").key?(Words['kotoba'].record_key(ALTERNATIVE)).should be_true
-      GreenMidgetRecords.create(:key => Words['mouichidou'].record_key(NULL),        :value => 0)
-      GreenMidgetRecords.create(:key => Words['mouichidou'].record_key(ALTERNATIVE), :value => 3)
-      GreenMidgetRecords.fetch_all(['mouichidou'])
-      GreenMidgetRecords.class_variable_get("@@cache")[Words['mouichidou'].record_key(NULL)].should_not == nil
-      GreenMidgetRecords.class_variable_get("@@cache")[Words['mouichidou'].record_key(ALTERNATIVE)].should_not == nil
+      Records.create(:key => Words['kotoba'].record_key(NULL))
+      Records.fetch_all(['kotoba'])
+      Records.cache.key?(Words['kotoba'].record_key(ALTERNATIVE)).should be_true
+      Records.create(:key => Words['mouichidou'].record_key(NULL),        :value => 0)
+      Records.create(:key => Words['mouichidou'].record_key(ALTERNATIVE), :value => 3)
+      Records.fetch_all(['mouichidou'])
+      Records.cache[Words['mouichidou'].record_key(NULL)].should_not == nil
+      Records.cache[Words['mouichidou'].record_key(ALTERNATIVE)].should_not == nil
     end
+
     it "the cache should be a hash; its keys should be strings" do
-      GreenMidgetRecords.create(:key => Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ NULL }_count")
-      GreenMidgetRecords.create(:key => Features.prefix + "url_in_text::#{ NULL }_count")
-      GreenMidgetRecords.fetch_all([])
-      cache = GreenMidgetRecords.class_variable_get("@@cache")
+      Records.create(:key => Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ NULL }_count")
+      Records.create(:key => Features.prefix + "url_in_text::#{ NULL }_count")
+      Records.fetch_all([])
+      cache = Records.cache
       cache.class.should.eql? Hash
       cache.count.should == 2
       cache.keys.each do |key|
@@ -70,11 +75,11 @@ describe GreenMidget::GreenMidgetRecords do
   describe "#increment" do
     it "should increment counts first in cache and write! to store only if explicitly called" do
       record_key = Words['stuff'].record_key(NULL)
-      GreenMidgetRecords.create(:key => record_key)
+      Records.create(:key => record_key)
 
       lambda {
-        GreenMidgetRecords.increment(record_key)
-      }.should change { GreenMidgetRecords.find_by_key(record_key).value.to_f }.by(1)
+        Records.increment(record_key)
+      }.should change { Records.find_by_key(record_key).value.to_f }.by(1)
     end
   end
 end

data/spec/spec_helper.rb

@@ -1,13 +1,16 @@
 # Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'rspec'
+
 require 'green_midget'
+require 'sqlite3'
 
-# Requires supporting files with custom matchers and macros, etc,
-# in ./support/ and its subdirectories.
-Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |file| require file }
+conn = { :adapter => 'sqlite3', :database => ':memory:' }
+ActiveRecord::Base.establish_connection(conn)
 
-RSpec.configure do |config|
+require 'green_midget/db/migrate/create_green_midget_records'
+GreenMidget::CreateGreenMidgetRecords.verbose = false
+GreenMidget::CreateGreenMidgetRecords.up
 
+GreenMidget::Records.class_eval do
+  cattr_accessor :cache
 end
+

data/spec/tester.rb

@@ -11,7 +11,7 @@ class Tester < GreenMidget::Base
   end
 
   def log_ratio
-    GreenMidgetRecords.fetch_all(words)
+    Records.fetch_all(words)
     super
   end
 end

data/spec/words_spec.rb

@@ -5,7 +5,7 @@ describe GreenMidget::Words do
   include GreenMidget
 
   before(:each) do
-    GreenMidgetRecords.delete_all
+    Records.delete_all
   end
 
   describe "self.record_keys" do
@@ -17,7 +17,7 @@ describe GreenMidget::Words do
 
   describe "#probability_for" do
     it "should return the smoother constant if the word has zero examples" do
-      GreenMidgetRecords[Words['word'].record_key(ALTERNATIVE)].should == ''
+      Records[Words['word'].record_key(ALTERNATIVE)].should == ''
       Words['word'].probability_for(ALTERNATIVE).should == (1.0 / Examples.total)
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: green_midget
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-05 00:00:00.000000000 +02:00
+date: 2012-02-17 00:00:00.000000000 +01:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &2153074740 !ruby/object:Gem::Requirement
+  requirement: &2153348200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,29 +22,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2153074740
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: &2153074320 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *2153074320
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: &2153073900 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *2153073900
+  version_requirements: *2153348200
 description: Naive Bayesian Classifier with customizable features
 email:
 - nikola@howkul.info
@@ -52,26 +30,32 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .document
 - .gitignore
+- .travis.yml
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
+- benchmark/benchmark.rb
+- benchmark/test.rb
 - green_midget.gemspec
 - lib/green_midget.rb
 - lib/green_midget/base.rb
 - lib/green_midget/constants.rb
 - lib/green_midget/db/migrate/create_green_midget_records.rb
+- lib/green_midget/default_features.rb
+- lib/green_midget/errors/feature_method_not_implemented.rb
+- lib/green_midget/errors/no_examples_given.rb
+- lib/green_midget/errors/no_text_found.rb
 - lib/green_midget/extensions/classifier.rb
 - lib/green_midget/extensions/sample.rb
-- lib/green_midget/green_midget.rb
+- lib/green_midget/heuristic_checks.rb
 - lib/green_midget/logger.rb
 - lib/green_midget/models/countable.rb
 - lib/green_midget/models/examples.rb
 - lib/green_midget/models/features.rb
-- lib/green_midget/models/green_midget_records.rb
+- lib/green_midget/models/records.rb
 - lib/green_midget/models/words.rb
 - lib/green_midget/url_detection.rb
 - lib/green_midget/version.rb
@@ -105,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: green_midget
+rubyforge_project: 
 rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3

data/.document

@@ -1,5 +0,0 @@
-lib/**/*.rb
-bin/*
-- 
-features/**/*.feature
-LICENSE.txt

data/lib/green_midget/green_midget.rb

@@ -1,6 +0,0 @@
-# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
-require File.join(File.dirname(__FILE__), 'constants')
-
-module GreenMidget
-  include Constants
-end

data/lib/green_midget/models/green_midget_records.rb

@@ -1,49 +0,0 @@
-# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
-module GreenMidget
-  class GreenMidgetRecords < ActiveRecord::Base
-    set_table_name :green_midget_records
-
-    def self.fetch_all(words = [])
-      words_keys = Words.record_keys(words)
-
-      pairs = connection.select_rows(
-        "SELECT `key`, `value` FROM %s WHERE `key` IN ('%s') OR `key` LIKE '%s' OR `key` LIKE '%s'" %
-        [ table_name, words_keys.join("', '"), "#{ Features.prefix }%", "#{ Examples.prefix }%" ]
-      )
-
-      @@cache = pairs.inject({}) do |memo, pair|
-        memo[pair.first] = pair.last
-        memo
-      end
-
-      words_keys.inject(@@cache) do |memo, word|
-        memo[word] ||= ''
-        memo
-      end
-    end
-
-    def self.[](key)
-      key = key.to_s
-      @@cache ||= {}
-      @@cache[key] || @@cache[key] = connection.select_value("SELECT `value` FROM #{ table_name } WHERE `key` = '#{ key }'") || @@cache[key] = ''
-    end
-
-    def self.increment(keys)
-      keys = Array(keys)
-      records = all(:conditions => [ "`key` IN (?)", keys ])
-
-      @@objects = records.inject({}) do |memo, record|
-        memo[record.key] = record
-        memo
-      end
-
-      keys.inject(@@objects) do |memo, key|
-        memo[key] ||= new(:key => key, :value => '0.0')
-        memo
-      end
-
-      @@objects.each { |key, record| record.update_attribute(:value, record.value.to_f + 1) }
-      @@objects = {}
-    end
-  end
-end