green_midget 0.0.1

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rspec

@@ -0,0 +1 @@
+--color

data/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-1.9.2@green_midget

data/Gemfile

@@ -0,0 +1,19 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+gem "activerecord"
+
+# remove this dependency after testing.
+gem 'jberkel-mysql-ruby',    '= 2.8.1',   :require => 'mysql' # Ruby 1.9 fixes
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", "~> 2.3.0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.5.2"
+  gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,45 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activemodel (3.0.7)
+      activesupport (= 3.0.7)
+      builder (~> 2.1.2)
+      i18n (~> 0.5.0)
+    activerecord (3.0.7)
+      activemodel (= 3.0.7)
+      activesupport (= 3.0.7)
+      arel (~> 2.0.2)
+      tzinfo (~> 0.3.23)
+    activesupport (3.0.7)
+    arel (2.0.9)
+    builder (2.1.2)
+    diff-lcs (1.1.2)
+    git (1.2.5)
+    i18n (0.5.0)
+    jberkel-mysql-ruby (2.8.1)
+    jeweler (1.5.2)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.8.7)
+    rcov (0.9.9)
+    rspec (2.3.0)
+      rspec-core (~> 2.3.0)
+      rspec-expectations (~> 2.3.0)
+      rspec-mocks (~> 2.3.0)
+    rspec-core (2.3.1)
+    rspec-expectations (2.3.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.3.0)
+    tzinfo (0.3.27)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  bundler (~> 1.0.0)
+  jberkel-mysql-ruby (= 2.8.1)
+  jeweler (~> 1.5.2)
+  rcov
+  rspec (~> 2.3.0)

data/LICENSE.txt

@@ -0,0 +1,15 @@
+Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,128 @@
+On Bayesian Classification
+----------
+
+Using SoundCloud's private messaging means that you can effectively reach out to everyone on the Cloud. On top of that, you have track commenting, groups posting, forum topics, track sharing - we care about your voice being heard! And read.
+
+I'll put this in some perspective and say that we're now having daily text exchange volume in the order of hundreds of thousands. And it's also rapidly going up.
+
+And while most of this runs smoother than Berliner beer on a SoundCloud Friday, violations to our [Community guidelines][guidelines] are starting to be less and less of an exception. So I've been given the task to address this and build a system that progressively learns how to tell good community behaviour from less good - welcome to the:
+
+GreenMidget
+----------
+
+GreenMidget is a trainable, feature-full Bayesian text classifier. Out of the box it's super straightforward to use, but it also offers easy customisation options. It's a Ruby gem and today we're open sourcing it, so you can start with it within a minute after the:
+
+Installation
+----------
+
+You are very likely (but not necessarily) gonna be on a Rails app, so just add
+
+    gem 'green_midget'
+
+to your Gemfile and run
+
+    bundle install
+
+then add
+
+    require 'green_midget'
+
+to your Rakefile and run
+
+    rake green_midget:setup
+
+You're now done.
+
+How it works
+----------
+
+GreenMidget
+
+GreenMidget is a learner, so you will only expect effective classification from it only once it has received sufficient training. Training it means providing examples of messages for the categor  
+
+
+
+Use it
+----------
+
+`GreenMidget` exposes two public methods as a start: `GreenMidget#classify_as!` and `GreenMidget#classify`. Let's do a three lines classification session and illustrate them
+
+We'll start training `GreenMidget` with a spammy example
+
+    GreenMidget.new(known_spam_text).classify_as! :spam
+
+Similarly for legitimate examples
+
+    GreenMidget.new(known_legit_text).classify_as! :ham
+
+To get a classification decision we would
+
+    decision = GreenMidget.new(new_text).classify
+
+`decision` is now one of `[-1, 0, 1]` meaning respectively 'No spam', 'Not enough evidence', 'Spam'.
+
+Extend it
+----------
+
+If the above functionality is not enough for you and you want to add custom logic to GreenMidget you can do that by extending the `GreenMidget::Base` class (check `extensions/sample.b` in the [code][green_midget_github] for an example):
+
+* Implement heuristics logic, which will directly classify incoming object as a given category. Example:
+
+        def pass_ham_heuristics?
+          words.count > 5 || url_in_text?
+        end
+
+  This method will be `true` for longer text or such that contains an external url. In this case the classifier would go on to the actual testing procedure. If `false`, however, the procedure will not be done and the classifier will return the ham category as a result. Note the native `GreenMidget::Base#words` and `GreenMidget::Base#url_in_text?`
+
+  All heuristic checks return `true` by default so it's up to you whether you will define and use heuristics at all or not. However, using them can help you integrate your application context and decrease classification error chance especially at the edge cases.
+
+* Expand the source of evidence. Traditionally, _naive_ Bayesian text classifiers see individual words as evidence and calculate category-likelihoods for each word. But there could be more than that in your application context, eg. user's data or specific text features.
+
+  By default GreenMidget comes with two feature definitions `url_in_text` and `email_in_text`, but you can implement as many more as you want by writing a boolean method that checks for the feature:
+
+         def regular_user?
+           @user.sign_up_count > 10
+         end
+
+  and then implement a `features` method that returns an array with your custom feature names:
+
+         def features
+           ['regular_user', .... ]
+         end
+
+  (do make sure that the array entry is the same as the name of the method that would be checking for this feature)
+
+  The GreenMidget features definitions have more weight on shorter texts and less weight on longer thus they provide a ground source of evidence for GreenMidget's classification.
+
+If that's not enough too, you're welcome to [browse the code][green_midget_github] and either extend more parts of it or simply make your own fork of the project.
+
+Benchmarking
+----------
+
+1. GreenMidget is optimised for classification operations (`classify` method), on which it's very efficient. The results below were obtained from classification on randomly generated messages of length _1 000 words_ (that's _very_ long for SoundCloud). Since GreenMidget runs on a relational database (through ActiveRecord) by default the table size impacts data fetch and write:
+
+	* on ~ 10 000 table rows = 0.0703 seconds / message
+	* on ~ 100 000 rows = 0.2082 sec / message
+	* on ~ 500 000 rows = 0.6505 sec / message
+	* on ~ 1 000 000 rows = 0.6773 sec / messages
+
+2. Training operations (`classify_as!`) are, however, less performant because they invoke a database write per word. Under the same conditions as above, the training times of randomly generated messages follows:
+
+	* on ~ 10 000 table rows = 1.5984 seconds / message
+	* on ~ 100 000 rows = 0.1303 sec / message
+	* on ~ 500 000 rows = 1.7185 sec / message
+	* on ~ 1 000 000 rows = 2.5335 sec / message
+
+Classification Efficiency
+----------
+
+Trained on s
+
+Benchmarks = > data
+
+Efficiency = > for the sake of this article I ran a small off-production test to show some results on real data - I used 150 000 text items from records which we marked as good and records which we marked as not-the-best!
+
+We'll be next building our own SoundCloud extensions to GreenMidget and use it, so expect to hear more from the student! Meanwhile, I'll be happy to answer everything concerning the project so do feel free to get in touch.
+
+[green_midget_github]: http://github.com/chochkov/green_midget "Github repository"
+[guidelines]: http://soundcloud.com/community-guidelines "Community guidelines"

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/benchmark/benchmark.rb

@@ -0,0 +1,40 @@
+include GreenMidget
+
+TRAININGS               = 90
+CLASSIFICATIONS         = 1
+
+MESSAGE_LENGTH          = 1000
+
+@training_times         = []
+@classification_times   = []
+
+records_count_at_start  = GreenMidgetRecords.count
+
+def generate_text(message_length = 1)
+  message ||= []
+  while message.count < message_length do
+    word = ''
+    (rand(7) + 3).times { word += ('a'..'z').to_a[rand(26)] }
+    message << word unless message.include?(word)
+  end
+  text = message.join(' ')
+end
+
+TRAININGS.times do
+  a = GreenMidgetCheck.new generate_text(MESSAGE_LENGTH)
+  @training_times       << Benchmark.measure{ a.classify_as! [ ALTERNATIVE, NULL ][rand(2)] }.real
+end
+
+CLASSIFICATIONS.times do
+  a = GreenMidgetCheck.new generate_text(MESSAGE_LENGTH)
+  @classification_times << Benchmark.measure{ a.classify }.real
+end
+
+puts " ------------------------------- "
+puts " Average seconds from #{ TRAININGS } trainings and #{ CLASSIFICATIONS } classifications. #{ MESSAGE_LENGTH } words per message:"
+puts " Number of records at start: #{ records_count_at_start } and at the end: #{ GreenMidgetRecords.count }"
+puts " ------------------------------- "
+puts " Training times:                 #{ (@training_times.sum.to_f/TRAININGS).round(4) }"
+puts " ------------------------------- "
+puts " Classification times:           #{ (@classification_times.sum.to_f/CLASSIFICATIONS).round(4) }"
+puts " ------------------------------- "

data/benchmark/test.rb

@@ -0,0 +1,31 @@
+require 'sqlite3'
+
+require File.join(File.dirname(__FILE__), '..', 'spec', 'tester')
+include GreenMidget
+
+ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => '/sc/user_backup/data.db')
+
+@spam = [ 'messages', 'comments', 'posts' ].map { |table| ActiveRecord::Base.connection.execute("select body from #{table} limit 1500").inject([]) { |memo, hash| memo << hash["body"] } }
+
+ActiveRecord::Base.establish_connection(:adapter => 'mysql', :username => 'root', :password => 'root', :database => 'soundcloud_development_temp')
+
+@ham  = [ 'messages', 'comments', 'posts' ].map { |table| GreenMidgetRecords.find_by_sql("select body from #{table} limit 1500").to_a.inject([]) { |memo, hash| memo << hash["body"] } }
+
+ActiveRecord::Base.establish_connection(:adapter => 'mysql', :username => 'root', :password => 'root', :database => 'classifier_development_weird')
+# 
+# # ------ I. PERFORM TRAINING
+# puts Benchmark.measure {
+#   @spam.each { |src|
+#     src.each {|body|
+#       klass = Tester.new(body);klass.classify_as! :spam
+#     }
+#   };true
+# }
+# 
+# puts Benchmark.measure {
+#   @ham.each { |src|
+#     src.each {|body|
+#       klass = Tester.new(body);klass.classify_as! :ham
+#     }
+#   };true
+# }

data/db/migrate/create_green_midget_records.rb

@@ -0,0 +1,16 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+class CreateGreenMidgetRecords < ActiveRecord::Migration
+  def self.up
+    create_table :green_midget_records do |t|
+      t.string   :key
+      t.string   :value
+      t.datetime :updated_at
+    end
+    add_index :green_midget_records, :key
+    add_index :green_midget_records, :updated_at
+  end
+
+  def self.down
+    drop_table :green_midget_records
+  end
+end

data/green_midget.gemspec

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "green_midget/version"
+
+Gem::Specification.new do |s|
+  s.name        = "green_midget"
+  s.version     = GreenMidget::VERSION
+  s.authors     = ["nikola chochkov"]
+  s.email       = ["nikola@howkul.info"]
+  s.homepage    = "http://github.com/chochkov/GreenMidget"
+  s.licenses    = ["MIT"]
+  s.require_paths    = ["lib"]
+  s.rubygems_version = %q{1.7.2}
+  s.summary     = %q{Bayesian Text Classifier}
+  s.description = %q{Naive Bayesian Classifier with customizable features}
+
+  s.rubyforge_project = "green_midget"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "activerecord"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "bundler"
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

data/lib/extensions/green_midget_check.rb

@@ -0,0 +1,8 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+class GreenMidgetCheck < GreenMidget::Base
+  attr_accessor :text
+
+  def initialize(text)
+    self.text = text
+  end
+end

data/lib/extensions/sample.rb

@@ -0,0 +1,19 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+class Sample < GreenMidget::Base
+  attr_accessor :user
+
+  def initialize(text, user)
+    @text = text
+    @user = user
+  end
+
+  private
+
+  def features
+    %w(regular_user) + super
+  end
+
+  def regular_user?
+
+  end
+end

data/lib/green_midget/base.rb

@@ -0,0 +1,91 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+require 'green_midget/constants'
+require 'green_midget/logger'
+require 'green_midget/url_detection'
+
+module GreenMidget
+  class Base
+    include Logger
+    include Constants
+
+    def classify
+      CATEGORIES.each do |category|
+        if respond_to?(:"pass_#{category}_heuristics?") && send(:"pass_#{category}_heuristics?")
+          classify_as!(category)
+          return HYPOTHESES[category]
+        end
+      end
+
+      GreenMidgetRecords.fetch_all(words)
+      register_classification
+
+      factor = log_ratio
+      case
+      when factor >= ACCEPT_ALTERNATIVE_MIN
+        ALTERNATIVE_RESPONSE
+      when factor >= REJECT_ALTERNATIVE_MAX
+        DUNNO
+      else
+        NULL_RESPONSE
+      end
+    end
+
+    def classify_as!(category)
+      keys = [ Words.objects(words), Features.objects(present_features), Examples.objects(features, true) ].flatten.map do |object|
+        object.record_key(category)
+      end
+
+      GreenMidgetRecords.increment(keys)
+      register_training
+    end
+
+    private
+
+    # ------ Features --------
+
+    def features
+      FEATURES
+    end
+
+    def present_features
+      features.select { |feature| feature_present?(feature) }
+    end
+
+    def feature_present?(feature)
+      method = :"#{ feature }?"
+      if respond_to?(method, true)
+        send(method)
+      else
+        raise("You must implement method #{ method } or remove feature #{ feature }.")
+      end
+    end
+
+    def url_in_text?
+      UrlDetection.new(text).any?
+    end
+
+    def email_in_text?
+      text.scan(EMAIL_REGEX).size > 0
+    end
+
+    # ------ Words --------
+
+    def words
+      strip_external_links.scan(WORDS_SPLIT_REGEX).uniq.
+        map(&:downcase).
+        reject { |word| STOP_WORDS.include?(word) }
+    end
+
+    def strip_external_links
+      text.gsub(EXTERNAL_LINK_REGEX, '')
+    end
+
+    def text
+      @text || raise('You should either implement the text method or provide an instance variable at this point.')
+    end
+
+    def log_ratio
+      Examples.log_ratio + words.map{ |word| Words[word].log_ratio }.sum + present_features.map{ |feature| Features[feature].log_ratio }.sum
+    end
+  end
+end

data/lib/green_midget/constants.rb

@@ -0,0 +1,31 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+module GreenMidget
+  module Constants
+    TOLERATED_URLS  = /(soundcloud.com)|(facebook.com)|(myspace.com)|(twitter.com)/
+
+    EMAIL_REGEX     = /[a-zA-Z][\w\.-]*[a-zA-Z0-9]@[a-zA-Z0-9][\w\.-]*[a-zA-Z0-9]\.[a-zA-Z][a-zA-Z\.]*[a-zA-Z]/
+    URL_REGEX       = /(?i)\b((?:[a-z][\w-]+:(?:\/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}\/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'".,<>?]))/
+
+    EXTERNAL_LINK_REGEX     = Regexp.new(/(#{ EMAIL_REGEX })|(#{ URL_REGEX })/)
+
+    STOP_WORDS              = %w()
+
+    MIN_CHARACTERS_IN_WORD  = 3
+    MAX_CHARACTERS_IN_WORD  = 20
+
+    WORDS_SPLIT_REGEX       = Regexp.new(/\w{#{ MIN_CHARACTERS_IN_WORD },#{ MAX_CHARACTERS_IN_WORD }}/)
+    FEATURES                = %w(url_in_text email_in_text)
+
+    # Decision making: Log(Pr(alternative | text)) - Log(Pr(null | text)) <=> [ REJECT_ALTERNATIVE_MAX, ACCEPT_ALTERNATIVE_MIN ]
+    ACCEPT_ALTERNATIVE_MIN  = Math::log(3.0)
+    REJECT_ALTERNATIVE_MAX  = 0.0
+
+    ALTERNATIVE_RESPONSE    = 1
+    DUNNO                   = 0
+    NULL_RESPONSE           = -1
+
+    NULL                    = :ham
+    ALTERNATIVE             = :spam
+    CATEGORIES              = [ NULL, ALTERNATIVE ]
+  end
+end

data/lib/green_midget/green_midget.rb

@@ -0,0 +1,6 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+require 'green_midget/constants'
+
+module GreenMidget
+  include Constants
+end

data/lib/green_midget/logger.rb

@@ -0,0 +1,16 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+module GreenMidget
+  module Logger
+    private
+
+    def register_classification
+      # TODO: implement default Logging system 
+      true
+    end
+
+    def register_training
+      # TODO: the same... (obviously)
+      true
+    end
+  end
+end

data/lib/green_midget/models/countable.rb

@@ -0,0 +1,33 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+module GreenMidget
+  class Countable
+    include Constants
+    attr_accessor :key
+
+    class << self; attr_accessor :prefix end
+
+    def self.[](key)
+      new(key)
+    end
+
+    def self.objects(keys)
+      keys.map { |key| new(key) }
+    end
+
+    def initialize(key)
+      @key = self.class.prefix + key
+    end
+
+    def [](category)
+      GreenMidgetRecords[record_key(category)].to_f
+    end
+
+    def log_ratio
+      Math::log(probability_for(ALTERNATIVE) / probability_for(NULL))
+    end
+
+    def record_key(category)
+      self.key + "::#{category}_count"
+    end
+  end
+end

data/lib/green_midget/models/examples.rb

@@ -0,0 +1,55 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+module GreenMidget
+  class Examples < Countable
+    NO_EXAMPLES_GIVEN_ERROR = 'Training examples must be provided for all categories before classification.'
+    GENERAL_FEATURE_NAME    = 'any'
+    self.prefix             = 'examples_with_feature::'
+
+    class_eval(<<-EVAL, __FILE__, __LINE__ + 1)
+      def self.#{ ALTERNATIVE }                                         # def self.ham
+        @@alternative ||= self[GENERAL_FEATURE_NAME][ALTERNATIVE]       #   @@alternative ||= self[GENERAL_FEATURE_NAME][ALTERNATIVE]
+      end                                                               # end
+
+      def self.#{ NULL }                                                # def self.spam
+        @@null ||= self[GENERAL_FEATURE_NAME][NULL]                     #   @@null ||= self[GENERAL_FEATURE_NAME][NULL]
+      end                                                               # end
+    EVAL
+
+    def self.[](feature)
+      object = super(feature)
+
+      if object.no_examples? && (feature == GENERAL_FEATURE_NAME)
+        raise NO_EXAMPLES_GIVEN_ERROR
+      elsif object.no_examples?
+        super GENERAL_FEATURE_NAME
+      else
+        object
+      end
+    end
+
+    def self.objects(features, with_general = false)
+      features += with_general ? [ GENERAL_FEATURE_NAME ] : []
+      super(features)
+    end
+
+    def self.log_ratio
+      self[GENERAL_FEATURE_NAME].log_ratio
+    end
+
+    def self.total
+      @@total ||= self[GENERAL_FEATURE_NAME].total
+    end
+
+    def probability_for(category)
+      self[category] / total
+    end
+
+    def total
+      CATEGORIES.inject(0) { |memo, category| memo += self[category] }
+    end
+
+    def no_examples?
+      CATEGORIES.inject(1) { |memo, category| memo *= self[category] } == 0
+    end
+  end
+end

data/lib/green_midget/models/features.rb

@@ -0,0 +1,14 @@
+# Copyright (c) 2011, SoundCloud Ltd., Nikola Chochkov
+module GreenMidget
+  class Features < Countable
+    self.prefix = 'feature::'
+
+    def probability_for(category)
+      self[category] / Examples[feature][category]
+    end
+
+    def feature
+      key.gsub(/(^#{ self.class.prefix })|(::\w+_count$)/, '')
+    end
+  end
+end