green_midget 0.0.3 → 0.1.0

data/lib/green_midget/default_features.rb

@@ -0,0 +1,40 @@
+# A mixin that implements features check and allows Base sublcasses
+# to define their own features for spam/ham detection.
+#
+# By default texts are checked for presence of external URL or email
+# references. An example of addional feature would be presence of particular
+# words or expressions.
+#
+# See the example in `lib/green_midget/extensions/sample.rb`
+#
+module GreenMidget
+  module DefaultFeatures
+
+    private
+
+    def features
+      FEATURES
+    end
+
+    def present_features
+      features.select { |feature| feature_present?(feature) }
+    end
+
+    def feature_present?(feature)
+      method = :"#{feature}?"
+      if respond_to?(method, true)
+        send(method)
+      else
+        raise FeatureMethodNotImplemented.new(feature, method)
+      end
+    end
+
+    def url_in_text?
+      UrlDetection.new(text).any?
+    end
+
+    def email_in_text?
+      text.scan(EMAIL_REGEX).size > 0
+    end
+  end
+end

data/lib/green_midget/errors/feature_method_not_implemented.rb

@@ -0,0 +1,11 @@
+module GreenMidget
+  class FeatureMethodNotImplemented < StandardError
+    def initialize(feature, method_name)
+      super <<-MSG
+Method #{method_name.inspect} not found. Either implement it or
+delete feature #{feature} from your features list.
+MSG
+    end
+  end
+end
+

data/lib/green_midget/errors/no_examples_given.rb

@@ -0,0 +1,9 @@
+module GreenMidget
+  class NoExamplesGiven < StandardError
+    def initialize
+      super <<-MSG
+Training examples must be provided for all categories before classification.
+MSG
+    end
+  end
+end

data/lib/green_midget/errors/no_text_found.rb

@@ -0,0 +1,10 @@
+module GreenMidget
+  class NoTextFound < StandardError
+    def initialize
+      super <<-MSG
+You should either implement the text method or provide an instance variable at this point.
+MSG
+    end
+  end
+end
+

data/lib/green_midget/heuristic_checks.rb

@@ -0,0 +1,23 @@
+# A mixin that implements heuritics checks for both categories.
+# If there're some conditions under which a spammable object could
+# directly be classified as one of the classification categories
+# the logic could be implemented using heuritic checks in your subclasses
+#
+# See the example in `lib/green_midget/extensions/sample.rb`
+#
+module GreenMidget
+  module HeuristicChecks
+
+    private
+
+    def heuristic_checks
+      CATEGORIES.each do |category|
+        if respond_to?(:"pass_#{category}_heuristics?") && send(:"pass_#{category}_heuristics?")
+          classify_as!(category)
+          return RESPONSES[category]
+        end
+      end
+      return false
+    end
+  end
+end

data/lib/green_midget/models/countable.rb

@@ -1,25 +1,41 @@
 # Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+#
+# This is an abstraction from Words, Examples and Features. It provides common
+# methods for building the record keys for individual countables in any
+# category.
+#
+# For example the data record key for the word 'legit' in Spam category would
+# be something like "word::legit::spam_count". The record key for a feature
+# 'url_present' in Ham would be something like "feature::url_present::ham_count"
+# The count of all training examples given for category Spam would be
+# "example::any::spam_count"
+#
+# The example counts for individual features is stored as well. For example for
+# 'url_present' we will have two records: "example::url_present::spam_count" and
+# "example::url_present::ham_count". They will store the informatino about how
+# much training the GreenMidget received for this feature in each category.
+#
+# This class is the link between countable and the Records data store adapter
+#
 module GreenMidget
   class Countable
-    include Constants
     attr_accessor :key
+    class_attribute :prefix
 
-    class << self; attr_accessor :prefix end
-
-    def self.[](key)
-      new(key)
+    def initialize(key)
+      @key = self.class.prefix + key
     end
 
-    def self.objects(keys)
-      keys.map { |key| new(key) }
-    end
+    class << self
+      alias :[] :new
 
-    def initialize(key)
-      @key = self.class.prefix + key
+      def objects(keys)
+        keys.map { |key| new(key) }
+      end
     end
 
     def [](category)
-      GreenMidgetRecords[record_key(category)].to_f
+      Records[record_key(category)].to_f
     end
 
     def log_ratio

data/lib/green_midget/models/examples.rb

@@ -1,25 +1,26 @@
 # Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+#
+# A model for Examples used in GreenMidget. Examples represent the counts for
+# how much training GreenMidget received in each respective category.
+#
+#   Example['url_present'][:spam]
+#   # the number of spam training examples having an URL
+#
+#   Example['any'][:ham]
+#   # the number of total Ham training examples 
+#
+# See Countable
+#
 module GreenMidget
   class Examples < Countable
-    NO_EXAMPLES_GIVEN_ERROR = 'Training examples must be provided for all categories before classification.'
     GENERAL_FEATURE_NAME    = 'any'
     self.prefix             = 'examples_with_feature::'
 
-    class_eval(<<-EVAL, __FILE__, __LINE__ + 1)
-      def self.#{ ALTERNATIVE }                                         # def self.ham
-        @@alternative ||= self[GENERAL_FEATURE_NAME][ALTERNATIVE]       #   @@alternative ||= self[GENERAL_FEATURE_NAME][ALTERNATIVE]
-      end                                                               # end
-
-      def self.#{ NULL }                                                # def self.spam
-        @@null ||= self[GENERAL_FEATURE_NAME][NULL]                     #   @@null ||= self[GENERAL_FEATURE_NAME][NULL]
-      end                                                               # end
-    EVAL
-
     def self.[](feature)
       object = super(feature)
 
       if object.no_examples? && (feature == GENERAL_FEATURE_NAME)
-        raise NO_EXAMPLES_GIVEN_ERROR
+        raise NoExamplesGiven
       elsif object.no_examples?
         super GENERAL_FEATURE_NAME
       else
@@ -51,5 +52,17 @@ module GreenMidget
     def no_examples?
       CATEGORIES.inject(1) { |memo, category| memo *= self[category] } == 0
     end
+
+    # These methods store the total ham and spam examples count
+    #
+    class_eval(<<-EVAL, __FILE__, __LINE__ + 1)
+      def self.#{ ALTERNATIVE }                                         # def self.ham
+        @@alternative ||= self[GENERAL_FEATURE_NAME][ALTERNATIVE]       #   @@alternative ||= self[GENERAL_FEATURE_NAME][ALTERNATIVE]
+      end                                                               # end
+
+      def self.#{ NULL }                                                # def self.spam
+        @@null ||= self[GENERAL_FEATURE_NAME][NULL]                     #   @@null ||= self[GENERAL_FEATURE_NAME][NULL]
+      end                                                               # end
+    EVAL
   end
 end

data/lib/green_midget/models/features.rb

@@ -1,4 +1,14 @@
 # Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+#
+# A model for Features used in GreenMidget. A Feature could be defined by user.
+# An example would be 'url_found_in_text' which will be true for spammable
+# objects that have url in their text and false otherwise.
+#
+#   Features['url_in_text'][:spam]
+#   # the count of spam messages that have the feature
+#
+# See Countable
+#
 module GreenMidget
   class Features < Countable
     self.prefix = 'feature::'
@@ -8,7 +18,7 @@ module GreenMidget
     end
 
     def feature
-      key.gsub(/(^#{ self.class.prefix })|(::\w+_count$)/, '')
+      key.gsub(/(^#{self.class.prefix})|(::\w+_count$)/, '')
     end
   end
 end

data/lib/green_midget/models/records.rb

@@ -0,0 +1,61 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+#
+# GreenMidget's simple data store adapter with only three public methods.
+# It's currently ActiveRecord based but a plan is to make a
+# Redis based extension as well.
+#
+module GreenMidget
+  class Records < ActiveRecord::Base
+    self.table_name = :green_midget_records
+
+    # Does a multi-get of the necessary count records for the given words.
+    # If no words are given, then only Examples and Features counts are taken
+    def self.fetch_all(words = [])
+      words_keys = Words.record_keys(words)
+
+      pairs = where(arel_table[:key].in(words_keys).
+                    or(arel_table[:key].matches("#{Features.prefix}%")).
+                    or(arel_table[:key].matches("#{Examples.prefix}%"))).
+              select(:key).select(:value)
+
+      @@cache = pairs.inject({}) do |memo, pair|
+        memo[pair['key']] = pair['value']
+        memo
+      end
+
+      words_keys.inject(@@cache) do |memo, word|
+        memo[word] ||= ''
+        memo
+      end
+    end
+
+    # Reads the value for a given key looking in the cache first and doing a
+    # database call if nothing is found.
+    def self.[](key)
+      key = key.to_s
+      @@cache ||= {}
+      @@cache[key] ||= where(:key => key).select(:value).map(&:value).first || ''
+    end
+
+    # Increment the values for given keys. The AR implementation increments each
+    # record individually, but implementing a multi-set is possible within this
+    # method.
+    def self.increment(keys)
+      keys = Array(keys)
+
+      @@objects = where(:key => keys).inject({}) do |memo, record|
+        memo[record.key] = record
+        memo
+      end
+
+      keys.inject(@@objects) do |memo, key|
+        memo[key] ||= new(:key => key, :value => 0)
+        memo
+      end
+
+      @@objects.each { |key, record| record.increment!(:value) }
+      @@objects = {}
+    end
+  end
+end
+

data/lib/green_midget/models/words.rb

@@ -1,11 +1,14 @@
 # Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+#
+# A model for Words used in GreenMidget. See Countable
+#
 module GreenMidget
   class Words < Countable
     self.prefix = 'word::'
 
     def self.record_keys(words, category = nil)
       words.map do |word|
-        Array(category || GreenMidget::CATEGORIES).map{ |category| Words[word].record_key(category) }
+        Array(category || CATEGORIES).map{ |category| Words[word].record_key(category) }
       end.flatten
     end
 

data/lib/green_midget/url_detection.rb

@@ -12,12 +12,12 @@ module GreenMidget
     private
 
     def urls
-      @text.scan(GreenMidget::URL_REGEX).flatten.reject(&:nil?)
+      @text.scan(URL_REGEX).flatten.reject(&:nil?)
     end
 
     def non_tolerated_urls
       urls.reject do |url|
-        url.to_s.downcase =~ GreenMidget::TOLERATED_URLS
+        url.to_s.downcase =~ TOLERATED_URLS
       end
     end
   end

data/lib/green_midget/version.rb

@@ -1,3 +1,3 @@
 module GreenMidget
-  VERSION = '0.0.3'
+  VERSION = '0.1.0'
 end

data/lib/tasks/green_midget.rake

@@ -1,5 +1,4 @@
 # Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
-require 'fileutils'
 require 'rake'
 require 'green_midget/db/migrate/create_green_midget_records'
 
@@ -9,24 +8,24 @@ namespace :green_midget do
     task :active_record => :environment do
       include GreenMidget
 
-      unless GreenMidgetRecords.table_exists?
+      unless Records.table_exists?
         CreateGreenMidgetRecords.up
       end
 
       keys = [ ALTERNATIVE, NULL ].map do |hypothesis|
-        [
-          "feature::url_in_text::#{hypothesis}_count",
-          "feature::email_in_text::#{hypothesis}_count",
-          "examples::any::#{hypothesis}_count",
-          "examples::url_in_text::#{hypothesis}_count",
-          "examples::email_in_text::#{hypothesis}_count",
-        ]
+        FEATURES.map do |feature|
+          [
+            "#{Features.prefix}#{feature}::#{hypothesis}_count",
+            "#{Examples.prefix}#{feature}::#{hypothesis}_count",
+            "#{Examples.prefix}any::#{hypothesis}_count",
+          ]
+        end
       end.flatten
 
       puts '==  Creating records ==='
       keys.each { |key|
-        unless GreenMidgetRecords.find_by_key(key)
-          GreenMidgetRecords.create(key)
+        unless Records.find_by_key(key)
+          Records.create(:key => key, :value => 0)
           puts "--  Created #{key}"
         end
       }

data/spec/base_spec.rb

@@ -1,41 +1,41 @@
 # Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
 require 'spec_helper'
-require File.join(File.dirname(__FILE__), 'tester')
+require 'tester'
 
 describe GreenMidget::Base do
   include GreenMidget
 
   before(:each) do
-    GreenMidgetRecords.delete_all
+    Records.delete_all
     [
-      {:key => "#{ Words.prefix    }this::#{          ALTERNATIVE  }_count", :value => 701.0  },
-      {:key => "#{ Words.prefix    }this::#{          NULL         }_count", :value => 11.0   },
-      {:key => "#{ Words.prefix    }test::#{          ALTERNATIVE  }_count", :value => 9.0    },
-      {:key => "#{ Words.prefix    }test::#{          NULL         }_count", :value => 71.0   },
-      {:key => "#{ Words.prefix    }goes::#{          ALTERNATIVE  }_count", :value => 90.0   },
-      {:key => "#{ Words.prefix    }goes::#{          NULL         }_count", :value => 90.0   },
-      {:key => "#{ Words.prefix    }rid::#{           ALTERNATIVE  }_count", :value => 311.0  },
-      {:key => "#{ Words.prefix    }rid::#{           NULL         }_count", :value => 290.0  },
-      {:key => "#{ Words.prefix    }dirty::#{         ALTERNATIVE  }_count", :value => 222.0  },
-      {:key => "#{ Words.prefix    }dirty::#{         NULL         }_count", :value => 45.0   },
-      {:key => "#{ Words.prefix    }spam::#{          ALTERNATIVE  }_count", :value => 11.0   },
-      {:key => "#{ Words.prefix    }spam::#{          NULL         }_count", :value => 133.0  },
-      {:key => "#{ Words.prefix    }words::#{         ALTERNATIVE  }_count", :value => 6.0    },
-      {:key => "#{ Words.prefix    }words::#{         NULL         }_count", :value => 811.0  },
-      {:key => "#{ Words.prefix    }zero::#{          ALTERNATIVE  }_count", :value => 0.0    },
-      {:key => "#{ Words.prefix    }zero::#{          NULL         }_count", :value => 0.0    },
-      {:key => "#{ Features.prefix }url_in_text::#{   ALTERNATIVE  }_count", :value => 440.0  },
-      {:key => "#{ Features.prefix }url_in_text::#{   NULL         }_count", :value => 40.0   },
-      {:key => "#{ Features.prefix }email_in_text::#{ ALTERNATIVE  }_count", :value => 112.0  },
-      {:key => "#{ Features.prefix }email_in_text::#{ NULL         }_count", :value => 9.0    },
-      {:key => "#{ Examples.prefix }any::#{           ALTERNATIVE  }_count", :value => 1000.0 },
-      {:key => "#{ Examples.prefix }any::#{           NULL         }_count", :value => 1000.0 },
-      {:key => "#{ Examples.prefix }url_in_text::#{   ALTERNATIVE  }_count", :value => 1000.0 },
-      {:key => "#{ Examples.prefix }url_in_text::#{   NULL         }_count", :value => 1000.0 },
-      {:key => "#{ Examples.prefix }email_in_text::#{ ALTERNATIVE  }_count", :value => 1000.0 },
-      {:key => "#{ Examples.prefix }email_in_text::#{ NULL         }_count", :value => 1000.0 },
+      {:key => "#{ Words.prefix    }this::#{          ALTERNATIVE  }_count", :value => 701  },
+      {:key => "#{ Words.prefix    }this::#{          NULL         }_count", :value => 11   },
+      {:key => "#{ Words.prefix    }test::#{          ALTERNATIVE  }_count", :value => 9    },
+      {:key => "#{ Words.prefix    }test::#{          NULL         }_count", :value => 71   },
+      {:key => "#{ Words.prefix    }goes::#{          ALTERNATIVE  }_count", :value => 90   },
+      {:key => "#{ Words.prefix    }goes::#{          NULL         }_count", :value => 90   },
+      {:key => "#{ Words.prefix    }rid::#{           ALTERNATIVE  }_count", :value => 311  },
+      {:key => "#{ Words.prefix    }rid::#{           NULL         }_count", :value => 290  },
+      {:key => "#{ Words.prefix    }dirty::#{         ALTERNATIVE  }_count", :value => 222  },
+      {:key => "#{ Words.prefix    }dirty::#{         NULL         }_count", :value => 45   },
+      {:key => "#{ Words.prefix    }spam::#{          ALTERNATIVE  }_count", :value => 11   },
+      {:key => "#{ Words.prefix    }spam::#{          NULL         }_count", :value => 133  },
+      {:key => "#{ Words.prefix    }words::#{         ALTERNATIVE  }_count", :value => 6    },
+      {:key => "#{ Words.prefix    }words::#{         NULL         }_count", :value => 811  },
+      {:key => "#{ Words.prefix    }zero::#{          ALTERNATIVE  }_count", :value => 0    },
+      {:key => "#{ Words.prefix    }zero::#{          NULL         }_count", :value => 0    },
+      {:key => "#{ Features.prefix }url_in_text::#{   ALTERNATIVE  }_count", :value => 440  },
+      {:key => "#{ Features.prefix }url_in_text::#{   NULL         }_count", :value => 40   },
+      {:key => "#{ Features.prefix }email_in_text::#{ ALTERNATIVE  }_count", :value => 112  },
+      {:key => "#{ Features.prefix }email_in_text::#{ NULL         }_count", :value => 9    },
+      {:key => "#{ Examples.prefix }any::#{           ALTERNATIVE  }_count", :value => 1000 },
+      {:key => "#{ Examples.prefix }any::#{           NULL         }_count", :value => 1000 },
+      {:key => "#{ Examples.prefix }url_in_text::#{   ALTERNATIVE  }_count", :value => 1000 },
+      {:key => "#{ Examples.prefix }url_in_text::#{   NULL         }_count", :value => 1000 },
+      {:key => "#{ Examples.prefix }email_in_text::#{ ALTERNATIVE  }_count", :value => 1000 },
+      {:key => "#{ Examples.prefix }email_in_text::#{ NULL         }_count", :value => 1000 },
     ].each do |entry|
-      GreenMidgetRecords.create(entry)
+      Records.create(entry)
     end
   end
 
@@ -106,23 +106,27 @@ describe GreenMidget::Base do
     it "should increase the index counts of the classified words" do
       lambda {
         Tester.new('zero').classify_as!(NULL)
-      }.should change { GreenMidgetRecords.find_by_key(Words['zero'].record_key(NULL)).value.to_f }.by(1)
+      }.should change { Records.find_by_key(Words['zero'].record_key(NULL)).value.to_f }.by(1)
     end
+
     it "should increment the learning examples count for all features" do
       FEATURES.each do |feature|
         lambda {
           Tester.new('zero').classify_as!(NULL)
-        }.should change { GreenMidgetRecords.find_by_key(Examples[feature].record_key(NULL)).value.to_f }.by(1)
+        }.should change { Records.find_by_key(Examples[feature].record_key(NULL)).value.to_f }.by(1)
       end
     end
+
     it "should not add new records for known keys" do
       a = Tester.new 'stuff unknown sofar'
+
       lambda {
         a.classify_as! ALTERNATIVE
-      }.should change { GreenMidgetRecords.count }.by(3)
+      }.should change { Records.count }.by(3)
+
       lambda {
         a.classify_as! ALTERNATIVE
-      }.should_not change { GreenMidgetRecords.count }
+      }.should_not change { Records.count }
     end
   end
 
@@ -130,15 +134,19 @@ describe GreenMidget::Base do
     it "should ignore words less than 3 characters" do
       Tester.new('is 2 ch').words.should == []
     end
+
     it "should break large character strings into chunks of 20 bytes" do
       Tester.new('s'*20 + '111').words.should == ['s'*20, '111']
     end
+
     it "should bring uppercase to lowcase" do
       Tester.new('HOWBIG').words.should == ['howbig']
     end
+
     it "should not consider parts of email address as individual words" do
       Tester.new('friend@soundcloud.com').words.should == []
     end
+
     it "should not consider parts of website url as individual words" do
       Tester.new('www.myguy.com http://weargeil.org').words.should == []
     end
@@ -150,7 +158,7 @@ describe GreenMidget::Base do
   #     pending('todo')
   #   end
   #   it "throw an exception if no training examples were given, but it's asked for classification" do
-  #     # if GreenMidgetRecords.count(ALTERNATIVE) or GreenMidgetRecords.count(NULL) is 0.0 => throw an exception
+  #     # if Records.count(ALTERNATIVE) or Records.count(NULL) is 0.0 => throw an exception
   #     pending('todo')
   #   end
   # end