green_midget 0.0.1

data/lib/green_midget/models/green_midget_records.rb

@@ -0,0 +1,49 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+module GreenMidget
+  class GreenMidgetRecords < ActiveRecord::Base
+    set_table_name :green_midget_records
+
+    def self.fetch_all(words = [])
+      words_keys = Words.record_keys(words)
+
+      pairs = connection.select_rows(
+        "SELECT `key`, `value` FROM %s WHERE `key` IN ('%s') OR `key` LIKE '%s' OR `key` LIKE '%s'" %
+        [ table_name, words_keys.join("', '"), "#{ Features.prefix }%", "#{ Examples.prefix }%" ]
+      )
+
+      @@cache = pairs.inject({}) do |memo, pair|
+        memo[pair.first] = pair.last
+        memo
+      end
+
+      words_keys.inject(@@cache) do |memo, word|
+        memo[word] ||= ''
+        memo
+      end
+    end
+
+    def self.[](key)
+      key = key.to_s
+      @@cache ||= {}
+      @@cache[key] || @@cache[key] = connection.select_value("SELECT `value` FROM #{ table_name } WHERE `key` = '#{ key }'") || @@cache[key] = ''
+    end
+
+    def self.increment(keys)
+      keys = Array(keys)
+      records = all(:conditions => [ "`key` IN (?)", keys ])
+
+      @@objects = records.inject({}) do |memo, record|
+        memo[record.key] = record
+        memo
+      end
+
+      keys.inject(@@objects) do |memo, key|
+        memo[key] ||= new(:key => key, :value => '0.0')
+        memo
+      end
+
+      @@objects.each { |key, record| record.update_attribute(:value, record.value.to_f + 1) }
+      @@objects = {}
+    end
+  end
+end

data/lib/green_midget/models/words.rb

@@ -0,0 +1,21 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+module GreenMidget
+  class Words < Countable
+    self.prefix = 'word::'
+
+    def self.record_keys(words, category = nil)
+      words.map do |word|
+        Array(category || GreenMidget::CATEGORIES).map{ |category| Words[word].record_key(category) }
+      end.flatten
+    end
+
+    def probability_for(category)
+      count = self[category]
+      if count == 0.0
+        1.0 / Examples.total
+      else
+        count / Examples.send(category)
+      end
+    end
+  end
+end

data/lib/green_midget/url_detection.rb

@@ -0,0 +1,24 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+module GreenMidget
+  class UrlDetection
+    def initialize(text)
+      @text = text
+    end
+
+    def any?
+      non_tolerated_urls.size > 0
+    end
+
+    private
+
+    def urls
+      @text.scan(GreenMidget::URL_REGEX).flatten.reject(&:nil?)
+    end
+
+    def non_tolerated_urls
+      urls.reject do |url|
+        url.to_s.downcase =~ GreenMidget::TOLERATED_URLS
+      end
+    end
+  end
+end

data/lib/green_midget/version.rb

@@ -0,0 +1,3 @@
+module GreenMidget
+  VERSION = '0.0.1'
+end

data/lib/green_midget.rb

@@ -0,0 +1,16 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+require 'active_record'
+require 'green_midget/green_midget'
+require 'green_midget/base'
+require 'green_midget/models/countable'
+require 'green_midget/models/examples'
+require 'green_midget/models/features'
+require 'green_midget/models/green_midget_records'
+require 'green_midget/models/words'
+require 'extensions/green_midget_check'
+
+if (classifier = Gem.searcher.find('green_midget'))
+  path = classifier.full_gem_path
+  Dir["#{path}/lib/tasks/*.rake"].each { |ext| load ext }
+end
+

data/lib/tasks/green_midget.rake

@@ -0,0 +1,32 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+require 'fileutils'
+require 'rake'
+require File.join(File.dirname(__FILE__), '..', '..', 'db', 'migrate', 'create_green_midget_records')
+
+namespace :green_midget do
+  desc "prepare this project for a world without spam"
+  task :setup => :environment do
+    include GreenMidget
+
+    unless GreenMidgetRecords.table_exists?
+      CreateGreenMidgetRecords.up
+    end
+
+    keys = ["url_in_text", "email_in_text"].map do |feature|
+      [ Features[feature].record_key(ALTERNATIVE), Features[feature].record_key(NULL) ]
+    end.flatten
+
+    keys += [Examples::GENERAL_FEATURE_NAME, "url_in_text", "email_in_text"].map do |feature|
+      [ Examples[feature].record_key(ALTERNATIVE), Examples[feature].record_key(NULL) ]
+    end.flatten
+
+    puts '==  Creating records ==='
+    keys.each { |key|
+      unless GreenMidgetRecords.find_by_key(key)
+        GreenMidgetRecords.create(key)
+        puts "--  Created #{key}"
+      end
+    }
+    puts '==  Done ==='
+  end
+end

data/spec/base_spec.rb

@@ -0,0 +1,163 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+require 'spec_helper'
+require File.join(File.dirname(__FILE__), 'tester')
+
+describe GreenMidget::Base do
+  include GreenMidget
+
+  before(:each) do
+    GreenMidgetRecords.delete_all
+    [
+      {:key => "#{ Words.prefix    }this::#{          ALTERNATIVE  }_count", :value => 701.0  },
+      {:key => "#{ Words.prefix    }this::#{          NULL         }_count", :value => 11.0   },
+      {:key => "#{ Words.prefix    }test::#{          ALTERNATIVE  }_count", :value => 9.0    },
+      {:key => "#{ Words.prefix    }test::#{          NULL         }_count", :value => 71.0   },
+      {:key => "#{ Words.prefix    }goes::#{          ALTERNATIVE  }_count", :value => 90.0   },
+      {:key => "#{ Words.prefix    }goes::#{          NULL         }_count", :value => 90.0   },
+      {:key => "#{ Words.prefix    }rid::#{           ALTERNATIVE  }_count", :value => 311.0  },
+      {:key => "#{ Words.prefix    }rid::#{           NULL         }_count", :value => 290.0  },
+      {:key => "#{ Words.prefix    }dirty::#{         ALTERNATIVE  }_count", :value => 222.0  },
+      {:key => "#{ Words.prefix    }dirty::#{         NULL         }_count", :value => 45.0   },
+      {:key => "#{ Words.prefix    }spam::#{          ALTERNATIVE  }_count", :value => 11.0   },
+      {:key => "#{ Words.prefix    }spam::#{          NULL         }_count", :value => 133.0  },
+      {:key => "#{ Words.prefix    }words::#{         ALTERNATIVE  }_count", :value => 6.0    },
+      {:key => "#{ Words.prefix    }words::#{         NULL         }_count", :value => 811.0  },
+      {:key => "#{ Words.prefix    }zero::#{          ALTERNATIVE  }_count", :value => 0.0    },
+      {:key => "#{ Words.prefix    }zero::#{          NULL         }_count", :value => 0.0    },
+      {:key => "#{ Features.prefix }url_in_text::#{   ALTERNATIVE  }_count", :value => 440.0  },
+      {:key => "#{ Features.prefix }url_in_text::#{   NULL         }_count", :value => 40.0   },
+      {:key => "#{ Features.prefix }email_in_text::#{ ALTERNATIVE  }_count", :value => 112.0  },
+      {:key => "#{ Features.prefix }email_in_text::#{ NULL         }_count", :value => 9.0    },
+      {:key => "#{ Examples.prefix }any::#{           ALTERNATIVE  }_count", :value => 1000.0 },
+      {:key => "#{ Examples.prefix }any::#{           NULL         }_count", :value => 1000.0 },
+      {:key => "#{ Examples.prefix }url_in_text::#{   ALTERNATIVE  }_count", :value => 1000.0 },
+      {:key => "#{ Examples.prefix }url_in_text::#{   NULL         }_count", :value => 1000.0 },
+      {:key => "#{ Examples.prefix }email_in_text::#{ ALTERNATIVE  }_count", :value => 1000.0 },
+      {:key => "#{ Examples.prefix }email_in_text::#{ NULL         }_count", :value => 1000.0 },
+    ].each do |entry|
+      GreenMidgetRecords.create(entry)
+    end
+  end
+
+  describe "#log_ratio" do
+    it "should be for 'this words'" do
+      Tester.new('this words').log_ratio.should == Math::log((701.0/1000) / (11.0/1000)) + Math::log((6.0/1000) / (811.0/1000)) + Math::log((1000.0/2000) / (1000.0/2000))
+    end
+
+    it "should be smaller for a smaller number of spammy words" do
+      Tester.new('this dirty test').log_ratio.should > Tester.new('this test').log_ratio
+    end
+
+    it "considers 'test goes words' ham" do
+      Tester.new('test goes words').log_ratio.should < REJECT_ALTERNATIVE_MAX
+    end
+
+    it "considers 'rid goes dirty' spam" do
+      Tester.new('rid goes dirty').log_ratio.should >= ACCEPT_ALTERNATIVE_MIN
+    end
+
+    it "doesn't know whether 'zero goes rid' is spam or not" do
+      Tester.new('zero goes rid').log_ratio.between?(REJECT_ALTERNATIVE_MAX, ACCEPT_ALTERNATIVE_MIN).should be_true
+    end
+
+    it "thinks of 'test boss@offshore.com' as more spam than just 'test'" do
+      Tester.new('test boss@offshore.com').log_ratio.
+        should > Tester.new('test').log_ratio
+    end
+
+    it "thinks of 'test www.offshore.com' as more spam than just 'test'" do
+      Tester.new('test www.offshore.com').log_ratio.
+        should > Tester.new('test').log_ratio
+    end
+
+    it "will tolerate urls coming from known sites" do
+      Tester.new('test www.offshore.com').log_ratio.should >
+      Tester.new('test www.soundcloud.com').log_ratio
+    end
+
+    it "should say DUNNO if it doesnt have neither ALTERNATIVE nor NULL score for a message" do
+      Tester.new('zero newword heuristicspass').log_ratio.between?(REJECT_ALTERNATIVE_MAX, ACCEPT_ALTERNATIVE_MIN).should be_true
+    end
+
+    it "should say ALTERNATIVE if it has spam score for a message and doesn't have ham score for it" do
+      a = Tester.new('nosuchword nowordsuch heuristicspass')
+      a.log_ratio.between?(REJECT_ALTERNATIVE_MAX, ACCEPT_ALTERNATIVE_MIN).should be_true
+      a.classify_as!(ALTERNATIVE)
+      a.log_ratio.should >= ACCEPT_ALTERNATIVE_MIN
+    end
+
+    it "should say NULL if it has ham score for a message and doesn't have spam score for it" do
+      a = Tester.new('suchwordno nowordsuch heuristicspasss')
+      a.log_ratio.between?(REJECT_ALTERNATIVE_MAX, ACCEPT_ALTERNATIVE_MIN).should be_true
+      a.classify_as!(NULL)
+      a.log_ratio.should < REJECT_ALTERNATIVE_MAX
+    end
+  end
+
+  describe "#classify" do
+    it "should add unknown words to the dictionary before classification" do
+      Tester.new('newword needs to pass heuristics').classify
+      Words['newword'][ALTERNATIVE].should == 0
+      Words['newword'][ALTERNATIVE].should == 0
+    end
+  end
+
+  describe "#classify_as!" do
+    it "should increase the index counts of the classified words" do
+      lambda {
+        Tester.new('zero').classify_as!(NULL)
+      }.should change { GreenMidgetRecords.find_by_key(Words['zero'].record_key(NULL)).value.to_f }.by(1)
+    end
+    it "should increment the learning examples count for all features" do
+      FEATURES.each do |feature|
+        lambda {
+          Tester.new('zero').classify_as!(NULL)
+        }.should change { GreenMidgetRecords.find_by_key(Examples[feature].record_key(NULL)).value.to_f }.by(1)
+      end
+    end
+    it "should not add new records for known keys" do
+      a = Tester.new 'stuff unknown sofar'
+      lambda {
+        a.classify_as! ALTERNATIVE
+      }.should change { GreenMidgetRecords.count }.by(3)
+      lambda {
+        a.classify_as! ALTERNATIVE
+      }.should_not change { GreenMidgetRecords.count }
+    end
+  end
+
+  describe "#words" do
+    it "should ignore words less than 3 characters" do
+      Tester.new('is 2 ch').words.should == []
+    end
+    it "should break large character strings into chunks of 20 bytes" do
+      Tester.new('s'*20 + '111').words.should == ['s'*20, '111']
+    end
+    it "should bring uppercase to lowcase" do
+      Tester.new('HOWBIG').words.should == ['howbig']
+    end
+    it "should not consider parts of email address as individual words" do
+      Tester.new('friend@soundcloud.com').words.should == []
+    end
+    it "should not consider parts of website url as individual words" do
+      Tester.new('www.myguy.com http://weargeil.org').words.should == []
+    end
+  end
+
+  # describe "extreme cases" do
+  #   it "should fallback to training_examples_with_feature::any if there're no examples in the database for a particular feature" do
+  #     # a new feature should be added with no examples and make sure the classifier won't break
+  #     pending('todo')
+  #   end
+  #   it "throw an exception if no training examples were given, but it's asked for classification" do
+  #     # if GreenMidgetRecords.count(ALTERNATIVE) or GreenMidgetRecords.count(NULL) is 0.0 => throw an exception
+  #     pending('todo')
+  #   end
+  # end
+  #
+  # describe "#feature_present?" do
+  #   it "should throw NoMethodError if a feature look-up method has not been implemented" do
+  #     pending('')
+  #   end
+  # end
+end

data/spec/examples_spec.rb

@@ -0,0 +1,85 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+require 'spec_helper'
+
+describe GreenMidget::Examples do
+  include GreenMidget
+
+  before(:each) do
+    GreenMidgetRecords.delete_all
+    GreenMidgetRecords.class_variable_set("@@cache", {})
+  end
+
+  describe "#[]()" do
+    before do
+      @call_any = lambda do
+        Examples[Examples::GENERAL_FEATURE_NAME]
+      end
+    end
+
+    it "should return the general feature examples if passed a (new) feature key that has no examples yet" do
+      GreenMidgetRecords.create(:key => Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ NULL }_count",        :value => 1000)
+      GreenMidgetRecords.create(:key => Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ ALTERNATIVE }_count", :value => 1000)
+      GreenMidgetRecords.find_by_key(Examples.prefix + "new::#{ NULL }_count").should == nil
+      GreenMidgetRecords.fetch_all
+      CATEGORIES.each do |category|
+        Examples['new'][category].should == Examples[Examples::GENERAL_FEATURE_NAME][category]
+      end
+    end
+    it "should return the feature's own example counts if these exist" do
+      GreenMidgetRecords.create(:key => Examples.prefix + "new::#{ NULL }_count",        :value => 3)
+      GreenMidgetRecords.create(:key => Examples.prefix + "new::#{ ALTERNATIVE }_count", :value => 1)
+      Examples['new'][NULL].should  == 3
+    end
+
+    it "should throw an error if the general feature examples isn't found" do
+      @call_any.should raise_error
+    end
+
+    it "should throw an error if the general feature examples has a zero spam_count and ham_count" do
+      GreenMidgetRecords.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ NULL }_count")
+      @call_any.should raise_error
+    end
+
+    it "should throw an error if the general feature examples has a zero spam_count or ham_count" do
+      GreenMidgetRecords.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ NULL }_count", :value => 0)
+      @call_any.should raise_error
+    end
+
+    it "should not throw an error if both columns are positive" do
+      GreenMidgetRecords.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ NULL }_count",        :value => 2)
+      GreenMidgetRecords.create(:key => Examples.prefix + "#{ Examples::GENERAL_FEATURE_NAME }::#{ ALTERNATIVE }_count", :value => 1)
+      @call_any.should_not raise_error
+    end
+  end
+
+  describe "#probability_for" do
+    it "should return the probability of a feature falling into category as: Examples[feature][category] / (Examples[feature][ALTERNATIVE] + Examples[feature][NULL])" do
+      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(NULL),        :value => 1000)
+      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(ALTERNATIVE), :value => 150 )
+      Examples['url_in_text'].probability_for(ALTERNATIVE).should == 150.0/(1000 + 150)
+    end
+  end
+
+  describe "#no_examples?" do
+    before(:each) do
+      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(ALTERNATIVE))
+      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(NULL))
+      @object = Examples['url_in_text']
+    end
+
+    it "should return true if spam_count and ham_count are zero" do
+      @object.no_examples?.should be_true
+    end
+
+    it "should return true if spam_count or ham_count are zero" do
+      GreenMidgetRecords.find_by_key(@object.record_key(NULL)).update_attribute(:value, 1)
+      @object.no_examples?.should be_true
+    end
+
+    it "should should return false if both spam_count and ham_count are positive" do
+      GreenMidgetRecords.find_by_key(@object.record_key(NULL)).update_attribute(:value, 1)
+      GreenMidgetRecords.find_by_key(@object.record_key(ALTERNATIVE)).update_attribute(:value, 1)
+      @object.no_examples?.should be_false
+    end
+  end
+end

data/spec/features_spec.rb

@@ -0,0 +1,24 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+require 'spec_helper'
+
+describe GreenMidget::Features do
+  include GreenMidget
+
+  before(:each) do
+    GreenMidgetRecords.delete_all
+    GreenMidgetRecords.class_variable_set("@@cache", {})
+  end
+
+  describe "#probability_for" do
+    it "should return Feature[feature] / Examples[feature]" do
+      GreenMidgetRecords.create(:key => Features["url_in_text"].record_key(NULL),        :value => 20  )
+      GreenMidgetRecords.create(:key => Features["url_in_text"].record_key(ALTERNATIVE), :value => 10  )
+
+      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(NULL),        :value => 100 )
+      GreenMidgetRecords.create(:key => Examples['url_in_text'].record_key(ALTERNATIVE), :value => 1000)
+
+      Features['url_in_text'].probability_for(NULL).should == 20.0/100
+      Features['url_in_text'].probability_for(ALTERNATIVE).should  == 10.0/1000
+    end
+  end
+end

data/spec/green_midget_records_spec.rb

@@ -0,0 +1,80 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+require 'spec_helper'
+
+describe GreenMidget::GreenMidgetRecords do
+  include GreenMidget
+
+  before(:each) do
+    GreenMidgetRecords.delete_all
+  end
+
+  describe "#[]()" do
+    it "should take words from data store if not found in the cache" do
+      word_key, phrase_key = [ 'word', 'phrase' ].map { |w| Words[w].record_key(NULL) }
+      GreenMidgetRecords.fetch_all([ 'word' ])
+      GreenMidgetRecords.create(:key => phrase_key)
+      GreenMidgetRecords.find_by_key(word_key).should == nil
+      GreenMidgetRecords.find_by_key(phrase_key).should_not == nil
+      GreenMidgetRecords[phrase_key].should == ''
+    end
+    it "should add a {key => ''} to the cache if key not found in cache and in the data store" do
+      key = Words['nonexisting'].record_key(NULL)
+      GreenMidgetRecords[key].should == ''
+      GreenMidgetRecords.find_by_key(key).should == nil
+    end
+  end
+
+  describe "#fetch_all" do
+    it "should empty cache before fetching" do
+      bar_key = Words['bar'].record_key(ALTERNATIVE)
+      GreenMidgetRecords.fetch_all([ 'foo', 'bar' ])
+      GreenMidgetRecords.class_variable_get("@@cache").key?(bar_key).should be_true
+      GreenMidgetRecords.fetch_all([ 'foo', 'newbar' ])
+      GreenMidgetRecords.class_variable_get("@@cache").key?(bar_key).should be_false
+    end
+    it "does a multi get on all words and keys" do
+      cache = GreenMidgetRecords.fetch_all([ 'foo', 'bar' ])
+      cache['foo'].should.eql? GreenMidgetRecords.class_eval{new('foo')}
+    end
+    it "should fetch the system keys along with the given words" do
+      key = Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ NULL }_count"
+      GreenMidgetRecords.create(:key => key)
+      GreenMidgetRecords.fetch_all([])
+      cache = GreenMidgetRecords.class_variable_get("@@cache")
+      cache.key?(key).should be_true
+      cache.count.should == 1
+    end
+    it "words with zero examples or no record in the database should be present in the cache" do
+      GreenMidgetRecords.create(:key => Words['kotoba'].record_key(NULL))
+      GreenMidgetRecords.fetch_all(['kotoba'])
+      GreenMidgetRecords.class_variable_get("@@cache").key?(Words['kotoba'].record_key(ALTERNATIVE)).should be_true
+      GreenMidgetRecords.create(:key => Words['mouichidou'].record_key(NULL),        :value => 0)
+      GreenMidgetRecords.create(:key => Words['mouichidou'].record_key(ALTERNATIVE), :value => 3)
+      GreenMidgetRecords.fetch_all(['mouichidou'])
+      GreenMidgetRecords.class_variable_get("@@cache")[Words['mouichidou'].record_key(NULL)].should_not == nil
+      GreenMidgetRecords.class_variable_get("@@cache")[Words['mouichidou'].record_key(ALTERNATIVE)].should_not == nil
+    end
+    it "the cache should be a hash; its keys should be strings" do
+      GreenMidgetRecords.create(:key => Examples.prefix + Examples::GENERAL_FEATURE_NAME + "::#{ NULL }_count")
+      GreenMidgetRecords.create(:key => Features.prefix + "url_in_text::#{ NULL }_count")
+      GreenMidgetRecords.fetch_all([])
+      cache = GreenMidgetRecords.class_variable_get("@@cache")
+      cache.class.should.eql? Hash
+      cache.count.should == 2
+      cache.keys.each do |key|
+        key.class.should.eql? String
+      end
+    end
+  end
+
+  describe "#increment" do
+    it "should increment counts first in cache and write! to store only if explicitly called" do
+      record_key = Words['stuff'].record_key(NULL)
+      GreenMidgetRecords.create(:key => record_key)
+
+      lambda {
+        GreenMidgetRecords.increment(record_key)
+      }.should change { GreenMidgetRecords.find_by_key(record_key).value.to_f }.by(1)
+    end
+  end
+end

data/spec/green_midget_url_detection_spec.rb

@@ -0,0 +1,14 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+require 'spec_helper'
+
+include GreenMidget
+
+describe UrlDetection do
+  it 'should not detect a url' do
+    UrlDetection.new('not a url').any?.should_not be_true
+  end
+
+  it 'should detect a url' do
+    UrlDetection.new('http://foo.de/').any?.should be_true
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'green_midget'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |file| require file }
+
+RSpec.configure do |config|
+
+end

data/spec/tester.rb

@@ -0,0 +1,17 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+class Tester < GreenMidget::Base
+  attr_accessor :text
+
+  def initialize(text = '')
+    self.text = text
+  end
+
+  def words
+    super
+  end
+
+  def log_ratio
+    GreenMidgetRecords.fetch_all(words)
+    super
+  end
+end

data/spec/words_spec.rb

@@ -0,0 +1,24 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+require 'spec_helper'
+
+describe GreenMidget::Words do
+  include GreenMidget
+
+  before(:each) do
+    GreenMidgetRecords.delete_all
+  end
+
+  describe "self.record_keys" do
+    it "takes an array of words and optionally a category, returns an array of corresponding record keys wrt category" do
+      Words.record_keys([ 'one' ]).should == [ "#{ Words.prefix }one::#{ NULL }_count", "#{ Words.prefix }one::#{ ALTERNATIVE }_count" ]
+      Words.record_keys([ 'one' ], NULL).should == [ "#{ Words.prefix }one::#{ NULL }_count" ]
+    end
+  end
+
+  describe "#probability_for" do
+    it "should return the smoother constant if the word has zero examples" do
+      GreenMidgetRecords[Words['word'].record_key(ALTERNATIVE)].should == ''
+      Words['word'].probability_for(ALTERNATIVE).should == (1.0 / Examples.total)
+    end
+  end
+end