thomas 0.1.0

data/README

@@ -0,0 +1,4 @@
+Thomas
+======
+
+Description goes here

data/lib/thomas.rb

@@ -0,0 +1,198 @@
+# Thomas
+#
+# A naive bayesian filter that works with any model that has word, corpus and amount attributes.
+# 
+# You use it like this (with the example model BayesWord):
+#
+# b = Adocca::Thomas::Bayes.new(BayesWord)
+# b.train("spam", "i am a nigerian president and i sell viagra")
+# b.train("ham", "i am a developer and i like ruby and beer")
+# b.train("ham", "i want to sell you my fancy stock")
+# b.guess(["ham","spam"], "i am a nigerian developer and i like ruby and fancy beer")
+#
+# You can also do (with the example corpuses (corpi?) "ham" and "spam"):
+# b.ham_or_spam("i am a nigerian developer and i like ruby and fancy beer")
+#
+
+require 'pp'
+
+module Adocca
+  module Thomas
+
+    class Update
+      def initialize(model, corpus, word, amount)
+        @model = model
+        @corpus = corpus
+        @word = word
+        @amount = amount
+      end
+      def execute
+        instance = @model.find_or_create_by_corpus_and_word(@corpus, @word)
+        instance.update_attribute(:amount, (instance.amount || 0) + @amount)
+      end
+    end
+
+    class Bayes
+
+      NAMESPACE = 'adocca:thomas:bayes'
+
+      attr_accessor :model
+
+      @@updates = []
+      
+      def initialize(model)
+        @model = model
+      end
+
+      def method_missing(meth, *args)
+        if meth.to_s =~ /_or_/
+          corpi = meth.to_s.split("_or_")
+          prob_by_corpus = guess(corpi, *args)
+
+          best_match = {"none" => -1}
+          next_best = {"none" => -1}
+          prob_by_corpus.each do |corpus, prob|
+            if prob > best_match.values.first
+              next_best = best_match.clone
+              best_match = {corpus => prob}
+            elsif prob > next_best.values.first
+              next_best = {corpus => prob}
+            end
+          end
+          [best_match.keys.first, best_match.values.first / next_best.values.first]
+        else
+          super
+        end
+      end
+
+      def train(corpus, text, options = {})
+        tokenize(text, options).each do |word|
+          @@updates << Update.new(@model, corpus, word, 1)
+          flush_updates
+        end
+      end
+
+      def untrain(corpus, text, options = {})
+        tokenize(text, options).each do |word|
+          @@updates << Update.new(@model, corpus, word, -1)
+          flush_updates
+        end
+      end
+
+      def guess(corpi, text, options = {})
+        corpus_probs_by_word = {}
+        tokenize(text, options).each do |word|
+          prob_by_corpus, any_occ = get_prob_by_corpus(corpi, word)
+          corpus_probs_by_word[word] = prob_by_corpus if any_occ
+        end
+        rval = {}
+        corpi.each do |corpus|
+          rval[corpus] = 1.0
+        end
+        corpus_probs_by_word.each do |word, prob_by_corpus|
+          prob_by_corpus.each do |corpus, prob|
+            rval[corpus] *= prob
+          end
+        end
+        rval
+      end
+
+      def get_occ(corpus, word)
+        instance = @model.find_by_corpus_and_word(corpus, word)
+        instance.nil? ? 0 : instance.amount
+      end
+
+      protected
+
+      def flush_updates
+        if @@updates.size > 100 || (defined?($FLUSH_THOMAS) && $FLUSH_THOMAS)
+          @model.transaction do
+            until @@updates.empty?
+              @@updates.pop.execute
+            end
+          end
+        end
+      end
+
+      def get_prob_by_corpus(corpi, word)
+        rval = {}
+        
+        occurence_by_corpus, any_occ = get_occurence_by_corpus(corpi, word)
+
+        return [{}, false] unless any_occ
+
+        total_occurences = occurence_by_corpus.values.inject(0) do |sum, occ| sum + occ end.to_f
+        total_occurences = 1.0 if total_occurences == 0.0
+
+        corpi.each do |corpus|
+          rval[corpus] = max(0.01,
+                             min(0.99,
+                                 occurence_by_corpus[corpus] / total_occurences))
+        end
+
+        [rval, true]
+      end
+
+      def get_occurence_by_corpus(corpi, word)
+        rval = {}
+        any_occ = false
+        corpi.each do |corpus|
+          occ = get_occ(corpus, word)
+          rval[corpus] = occ
+          any_occ = true if occ > 0
+        end
+        [rval, any_occ]
+      end
+
+      def min(a,b)
+        if a > b
+          b
+        else
+          a
+        end
+      end
+
+      def abs(a)
+        if a > 0
+          a
+        else
+          -a
+        end
+      end
+
+      def max(a,b)
+        if a > b
+          a
+        else
+          b
+        end
+      end
+      
+      #
+      # Tokenizes the +text+ given and returns
+      # an Array of the included words separated by a bunch of stuff.
+      #
+      # Will accept HTML, and tokenize only the text nodes within, except
+      # it will add the source of any element containing source as 'src:SOURCE'.
+      #
+      # An optional Hash can be given and will be added to the token Array
+      # at the end as 'KEY:VALUE'.
+      #
+      def tokenize(text, options = {})
+        rval = []
+        tokenizer = HTML::Tokenizer.new(text)
+        while token = tokenizer.next
+          node = HTML::Node.parse(nil, 0, 0, token, false)
+          case node
+          when HTML::Tag
+            rval << "src:#{node.attributes["src"]}" if node.attributes && node.attributes.include?("src")
+          when HTML::Text
+            rval += node.content.split(/[\s!?:;,.\/\\=+-]+/)
+          end
+        end
+        rval + options.keys.collect do |key| "#{key}:#{options[key]}" end
+      end
+
+    end
+  end
+end

data/test/thomas_test.rb

@@ -0,0 +1,8 @@
+require 'test/unit'
+
+class ThomasTest < Test::Unit::TestCase
+  # Replace this with your real tests.
+  def test_this_plugin
+    flunk
+  end
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.0
+specification_version: 1
+name: thomas
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2006-11-13 00:00:00 +01:00
+summary: Bayes classifier using memcached
+require_paths: 
+- lib
+email: 
+homepage: 
+rubyforge_project: 
+description: 
+autorequire: thomas
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Adocca Entertainment AB
+files: 
+- lib/thomas.rb
+- test/thomas_test.rb
+- README
+test_files: []
+
+rdoc_options: 
+- --line-numbers
+- --inline-source
+extra_rdoc_files: 
+- README
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: []
+