bayes_on_redis_internal 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in bayes_on_redis_internal.gemspec
+gemspec
+gem 'redis'

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bayes_on_redis_internal.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "bayes_on_redis_internal/version"
+
+Gem::Specification.new do |s|
+  s.name        = "bayes_on_redis_internal"
+  s.version     = BayesOnRedisInternal::VERSION
+  s.authors     = ["Dermot Haughey"]
+  s.email       = ["hderms@gmail.com"]
+  s.homepage    = ""
+  s.summary     = "fork of bayes_on_redis that includes more redis integration"
+  s.description = "library for performing bayesian analysis by way of redis hashes"
+
+  s.rubyforge_project = "bayes_on_redis_internal"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

data/lib/bayes_on_redis_internal.rb

@@ -0,0 +1,120 @@
+require "rubygems"
+
+class BayesOnRedis
+  CATEGORIES_KEY = "BayesOnRedis:categories"
+  ONE_OR_TWO_WORDS_RE = /\b\w{1,2}\b/mi
+  NON_ALPHANUMERIC_AND_NON_DOT_RE = /[^\w\.]/mi
+
+  attr_reader :redis, :stopwords
+
+  def initialize(options)
+    if options.is_a?(Hash)
+      @redis = Redis.new(:host => options[:redis_host], :port => options[:redis_port], :db => options[:redis_db])      
+    else
+      @redis = options
+    end
+
+    @stopwords = Stopword.new
+  end
+
+  def flushdb
+    @redis.flushdb
+  end
+
+  # training for a category
+  def train(category, text)
+    category = category.downcase
+    @redis.sadd(CATEGORIES_KEY, category)
+
+    count_occurance(text).each do |word, count|
+      @redis.hincrby(redis_category_key(category), word, count)
+    end
+  end
+  alias_method :learn, :train
+  
+  def remove(category)
+    category=category.downcase
+    @redis.del(redis_category_key(category))
+    @redis.srem(CATEGORIES_KEY, category)
+  end
+  def list_members
+    return @redis.smembers (CATEGORIES_KEY)
+  end
+  
+  def untrain(category, text)
+    category = category.downcase
+
+    count_occurance(text).each do |word, count|
+      word_count_atm = @redis.hget(redis_category_key(category), word)
+      if (word_count_atm >= count)
+        new_count = (word_count_atm - count)
+      else
+        new_count = 0
+      end
+      @redis.hset(redis_category_key(category), word, new_count)
+    end
+  end
+  alias_method :unlearn, :untrain
+
+  def score(text)
+    scores = {}
+
+    @redis.smembers(CATEGORIES_KEY).each do |category|
+      words_count_per_category = @redis.hvals(redis_category_key(category)).inject(0) {|sum, score| sum + score.to_i}
+      @redis.srem(CATEGORIES_KEY, category) if words_count_per_category <= 0
+
+      scores[category] = 0
+
+      count_occurance(text).each do |word, count|
+        tmp_score = @redis.hget(redis_category_key(category), word).to_i
+        tmp_score = 0.1 if tmp_score <= 0
+
+        scores[category] += Math.log(tmp_score / words_count_per_category.to_f)
+      end
+    end
+
+    return scores
+  end
+
+  def classify(text)
+    (score(text).sort_by { |score| -score[1] })[0][0]    # [0][0] -> first score, get the key
+  end
+
+  private
+  def redis_category_key(category)
+    "BayesOnRedis:cat:#{category}"
+  end
+
+  # Incoming text is always downcased
+  def count_occurance(text='')
+    raise "input must be instance of String" unless text.is_a?(String)
+
+    text_chunks = text.downcase.gsub(ONE_OR_TWO_WORDS_RE, '').gsub(NON_ALPHANUMERIC_AND_NON_DOT_RE, ' ').gsub(@stopwords.to_re, '').gsub(/\./, '').split
+    text_chunks.inject(Hash.new(0)) do |container, word|
+      container[word] += 1; container
+    end
+  end
+
+  def remove_stopwords
+    @redis.smembers(CATEGORIES_KEY).each do |category|
+      @stopwords.to_a.each do |stopword|
+        @redis.hdel(redis_category_key(category), stopword)
+      end
+    end
+  end
+end
+
+
+class Stopword
+  def initialize
+    @stopwords = File.read(File.expand_path(File.join(__FILE__, "..", "..", "datasets", "stopwords.txt"))).split
+  end
+
+  def to_a
+    @stopwords
+  end
+
+  def to_re
+    @to_re ||= /\b(#{@stopwords.join('|')})\b/mi
+  end
+end

data/lib/bayes_on_redis_internal/version.rb

@@ -0,0 +1,3 @@
+module BayesOnRedisInternal
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,51 @@
+--- !ruby/object:Gem::Specification
+name: bayes_on_redis_internal
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Dermot Haughey
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-11 00:00:00.000000000Z
+dependencies: []
+description: library for performing bayesian analysis by way of redis hashes
+email:
+- hderms@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- bayes_on_redis_internal.gemspec
+- lib/bayes_on_redis_internal.rb
+- lib/bayes_on_redis_internal/version.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: bayes_on_redis_internal
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: fork of bayes_on_redis that includes more redis integration
+test_files: []