classifier-reborn 2.1.0 → 2.2.0

data/data/stopwords/zh

@@ -0,0 +1,125 @@
+
+的
+一
+不
+在
+人
+有
+是
+为
+以
+于
+上
+他
+而
+后
+之
+来
+及
+了
+因
+下
+可
+到
+由
+这
+与
+也
+此
+但
+并
+个
+其
+已
+无
+小
+我
+们
+起
+最
+再
+今
+去
+好
+只
+又
+或
+很
+亦
+某
+把
+那
+你
+乃
+它
+吧
+被
+比
+别
+趁
+当
+从
+到
+得	打
+凡
+儿
+尔
+该
+各
+给
+跟
+和
+何
+还
+即
+几
+既
+看
+据
+距
+靠
+啦
+了
+另
+么
+每
+们
+嘛
+拿
+哪
+那
+您
+凭
+且
+却
+让
+仍
+啥
+如
+若
+使
+谁
+虽
+随
+同
+所
+她
+哇
+嗡
+往
+哪
+些
+向
+沿
+哟
+用
+于
+咱
+则
+怎
+曾
+至
+致
+着
+诸
+自

data/lib/classifier-reborn.rb

@@ -25,6 +25,15 @@
 # License::   LGPL
 
 require 'rubygems'
+
+case RUBY_PLATFORM
+when 'java'
+  require 'jruby-stemmer'
+else
+  require 'fast-stemmer'
+end
+
 require_relative 'classifier-reborn/category_namer'
 require_relative 'classifier-reborn/bayes'
 require_relative 'classifier-reborn/lsi'
+require_relative 'classifier-reborn/validators/classifier_validator'

data/lib/classifier-reborn/backends/bayes_memory_backend.rb

@@ -0,0 +1,75 @@
+module ClassifierReborn
+  class BayesMemoryBackend
+    attr_reader :total_words, :total_trainings
+
+    # This class provides Memory as the storage backend for the classifier data structures
+    def initialize
+      @total_words     = 0
+      @total_trainings = 0
+      @category_counts = {}
+      @categories      = {}
+    end
+
+    def update_total_words(diff)
+      @total_words += diff
+    end
+
+    def update_total_trainings(diff)
+      @total_trainings += diff
+    end
+
+    def category_training_count(category)
+      category_counts(category)[:training]
+    end
+
+    def update_category_training_count(category, diff)
+      category_counts(category)[:training] += diff
+    end
+
+    def category_has_trainings?(category)
+      @category_counts.key?(category) && category_training_count(category) > 0
+    end
+
+    def category_word_count(category)
+      category_counts(category)[:word]
+    end
+
+    def update_category_word_count(category, diff)
+      category_counts(category)[:word] += diff
+    end
+
+    def add_category(category)
+      @categories[category] ||= Hash.new(0)
+    end
+
+    def category_keys
+      @categories.keys
+    end
+
+    def category_word_frequency(category, word)
+      @categories[category][word]
+    end
+
+    def update_category_word_frequency(category, word, diff)
+      @categories[category][word] += diff
+    end
+
+    def delete_category_word(category, word)
+      @categories[category].delete(word)
+    end
+
+    def word_in_category?(category, word)
+      @categories[category].key?(word)
+    end
+
+    def reset
+      initialize
+    end
+
+    private
+
+    def category_counts(category)
+      @category_counts[category] ||= {training: 0, word: 0}
+    end
+  end
+end

data/lib/classifier-reborn/backends/bayes_redis_backend.rb

@@ -0,0 +1,107 @@
+require_relative 'no_redis_error'
+# require redis when we run #intialize. This way only people using this backend
+# will need to install and load the backend without having to
+# require 'classifier-reborn/backends/bayes_redis_backend'
+
+module ClassifierReborn
+  # This class provides Redis as the storage backend for the classifier data structures
+  class BayesRedisBackend
+    # The class can be created with the same arguments that the redis gem accepts
+    # E.g.,
+    #      b = ClassifierReborn::BayesRedisBackend.new
+    #      b = ClassifierReborn::BayesRedisBackend.new host: "10.0.1.1", port: 6380, db: 2
+    #      b = ClassifierReborn::BayesRedisBackend.new url: "redis://:secret@10.0.1.1:6380/2"
+    #
+    # Options available are:
+    #   url:                lambda { ENV["REDIS_URL"] }
+    #   scheme:             "redis"
+    #   host:               "127.0.0.1"
+    #   port:               6379
+    #   path:               nil
+    #   timeout:            5.0
+    #   password:           nil
+    #   db:                 0
+    #   driver:             nil
+    #   id:                 nil
+    #   tcp_keepalive:      0
+    #   reconnect_attempts: 1
+    #   inherit_socket:     false
+    def initialize(options = {})
+      begin # because some people don't have redis installed
+        require 'redis'
+      rescue LoadError
+        raise NoRedisError
+      end
+
+      @redis = Redis.new(options)
+      @redis.setnx(:total_words, 0)
+      @redis.setnx(:total_trainings, 0)
+    end
+
+    def total_words
+      @redis.get(:total_words).to_i
+    end
+
+    def update_total_words(diff)
+      @redis.incrby(:total_words, diff)
+    end
+
+    def total_trainings
+      @redis.get(:total_trainings).to_i
+    end
+
+    def update_total_trainings(diff)
+      @redis.incrby(:total_trainings, diff)
+    end
+
+    def category_training_count(category)
+      @redis.hget(:category_training_count, category).to_i
+    end
+
+    def update_category_training_count(category, diff)
+      @redis.hincrby(:category_training_count, category, diff)
+    end
+
+    def category_has_trainings?(category)
+      category_training_count(category) > 0
+    end
+
+    def category_word_count(category)
+      @redis.hget(:category_word_count, category).to_i
+    end
+
+    def update_category_word_count(category, diff)
+      @redis.hincrby(:category_word_count, category, diff)
+    end
+
+    def add_category(category)
+      @redis.sadd(:category_keys, category)
+    end
+
+    def category_keys
+      @redis.smembers(:category_keys).map(&:intern)
+    end
+
+    def category_word_frequency(category, word)
+      @redis.hget(category, word).to_i
+    end
+
+    def update_category_word_frequency(category, word, diff)
+      @redis.hincrby(category, word, diff)
+    end
+
+    def delete_category_word(category, word)
+      @redis.hdel(category, word)
+    end
+
+    def word_in_category?(category, word)
+      @redis.hexists(category, word)
+    end
+
+    def reset
+      @redis.flushdb
+      @redis.set(:total_words, 0)
+      @redis.set(:total_trainings, 0)
+    end
+  end
+end

data/lib/classifier-reborn/backends/no_redis_error.rb

@@ -0,0 +1,12 @@
+class NoRedisError < LoadError
+  def initialize
+    msg =
+      %q{The Redis Backend can only be used if Redis is installed.
+        This error is raised from 'lib/classifier-reborn/backends/bayes_redis_backend.rb'.
+        If you have encountered this error and would like to use the Redis Backend,
+        please run 'gem install redis' or include 'gem "redis"' in
+        your gemfile. For more info see https://github.com/jekyll/classifier-reborn#usage.
+      }
+    super(msg)
+  end
+end

data/lib/classifier-reborn/bayes.rb

@@ -2,7 +2,11 @@
 # Copyright:: Copyright (c) 2005 Lucas Carlson
 # License::   LGPL
 
+require 'set'
+
 require_relative 'category_namer'
+require_relative 'backends/bayes_memory_backend'
+require_relative 'backends/bayes_redis_backend'
 
 module ClassifierReborn
   class Bayes
@@ -13,36 +17,45 @@ module ClassifierReborn
     #      b = ClassifierReborn::Bayes.new 'Interesting', 'Uninteresting', 'Spam'
     #
     # Options available are:
-    #   language:         'en'   Used to select language specific stop words
-    #   auto_categorize:  false  When true, enables ability to dynamically declare a category
-    #   enable_threshold: false  When true, enables a threshold requirement for classifition
-    #   threshold:        0.0    Default threshold, only used when enabled
-    #   enable_stemmer:   true   When false, disables word stemming
+    #   language:         'en'                    Used to select language specific stop words
+    #   auto_categorize:  false                   When true, enables ability to dynamically declare a category; the default is true if no initial categories are provided
+    #   enable_threshold: false                   When true, enables a threshold requirement for classifition
+    #   threshold:        0.0                     Default threshold, only used when enabled
+    #   enable_stemmer:   true                    When false, disables word stemming
+    #   stopwords:        nil                     Accepts path to a text file or an array of words, when supplied, overwrites the default stopwords; assign empty string or array to disable stopwords
+    #   backend:          BayesMemoryBackend.new  Alternatively, BayesRedisBackend.new for persistent storage
     def initialize(*args)
-      @categories = {}
+      @initial_categories = []
       options = { language:         'en',
-                  auto_categorize:  false,
                   enable_threshold: false,
                   threshold:        0.0,
-                  enable_stemmer:   true
+                  enable_stemmer:   true,
+                  backend:          BayesMemoryBackend.new
                 }
       args.flatten.each do |arg|
         if arg.is_a?(Hash)
           options.merge!(arg)
         else
-          add_category(arg)
+          @initial_categories.push(arg)
         end
       end
 
-      @total_words         = 0
-      @category_counts     = Hash.new(0)
-      @category_word_count = Hash.new(0)
+      unless options.key?(:auto_categorize)
+        options[:auto_categorize] = @initial_categories.empty? ? true : false
+      end
 
       @language            = options[:language]
       @auto_categorize     = options[:auto_categorize]
       @enable_threshold    = options[:enable_threshold]
       @threshold           = options[:threshold]
       @enable_stemmer      = options[:enable_stemmer]
+      @backend             = options[:backend]
+
+      populate_initial_categories
+
+      if options.key?(:stopwords)
+        custom_stopwords options[:stopwords]
+      end
     end
 
     # Provides a general training method for all categories specified in Bayes#new
@@ -52,10 +65,12 @@ module ClassifierReborn
     #     b.train "that", "That text"
     #     b.train "The other", "The other text"
     def train(category, text)
+      word_hash = Hasher.word_hash(text, @language, @enable_stemmer)
+      return if word_hash.empty?
       category = CategoryNamer.prepare_name(category)
 
       # Add the category dynamically or raise an error
-      unless @categories.key?(category)
+      unless category_keys.include?(category)
         if @auto_categorize
           add_category(category)
         else
@@ -63,12 +78,13 @@ module ClassifierReborn
         end
       end
 
-      @category_counts[category] += 1
-      Hasher.word_hash(text, @language, @enable_stemmer).each do |word, count|
-        @categories[category][word] += count
-        @category_word_count[category] += count
-        @total_words += count
+      word_hash.each do |word, count|
+        @backend.update_category_word_frequency(category, word, count)
+        @backend.update_category_word_count(category, count)
+        @backend.update_total_words(count)
       end
+      @backend.update_total_trainings(1)
+      @backend.update_category_training_count(category, 1)
     end
 
     # Provides a untraining method for all categories specified in Bayes#new
@@ -79,20 +95,23 @@ module ClassifierReborn
     #     b.train :this, "This text"
     #     b.untrain :this, "This text"
     def untrain(category, text)
+      word_hash = Hasher.word_hash(text, @language, @enable_stemmer)
+      return if word_hash.empty?
       category = CategoryNamer.prepare_name(category)
-      @category_counts[category] -= 1
-      Hasher.word_hash(text, @language, @enable_stemmer).each do |word, count|
-        next if @total_words < 0
-        orig = @categories[category][word] || 0
-        @categories[category][word] -= count
-        if @categories[category][word] <= 0
-          @categories[category].delete(word)
+      word_hash.each do |word, count|
+        next if @backend.total_words < 0
+        orig = @backend.category_word_frequency(category, word) || 0
+        @backend.update_category_word_frequency(category, word, -count)
+        if @backend.category_word_frequency(category, word) <= 0
+          @backend.delete_category_word(category, word)
           count = orig
         end
 
-        @category_word_count[category] -= count if @category_word_count[category] >= count
-        @total_words -= count
+        @backend.update_category_word_count(category, -count) if @backend.category_word_count(category) >= count
+        @backend.update_total_words(-count)
       end
+      @backend.update_total_trainings(-1)
+      @backend.update_category_training_count(category, -1)
     end
 
     # Returns the scores in each category the provided +text+. E.g.,
@@ -102,17 +121,22 @@ module ClassifierReborn
     def classifications(text)
       score = {}
       word_hash = Hasher.word_hash(text, @language, @enable_stemmer)
-      training_count = @category_counts.values.reduce(:+).to_f
-      @categories.each do |category, category_words|
+      if word_hash.empty?
+        category_keys.each do |category|
+          score[category.to_s] = Float::INFINITY
+        end
+        return score
+      end
+      category_keys.each do |category|
         score[category.to_s] = 0
-        total = (@category_word_count[category] || 1).to_f
+        total = (@backend.category_word_count(category) || 1).to_f
         word_hash.each do |word, _count|
-          s = category_words.key?(word) ? category_words[word] : 0.1
+          s = @backend.word_in_category?(category, word) ? @backend.category_word_frequency(category, word) : 0.1
           score[category.to_s] += Math.log(s / total)
         end
         # now add prior probability for the category
-        s = @category_counts.key?(category) ? @category_counts[category] : 0.1
-        score[category.to_s] += Math.log(s / training_count)
+        s = @backend.category_has_trainings?(category) ? @backend.category_training_count(category) : 0.1
+        score[category.to_s] += Math.log(s / @backend.total_trainings.to_f)
       end
       score
     end
@@ -178,7 +202,7 @@ module ClassifierReborn
     def method_missing(name, *args)
       cleaned_name = name.to_s.gsub(/(un)?train_([\w]+)/, '\2')
       category = CategoryNamer.prepare_name(cleaned_name)
-      if @categories.key? category
+      if category_keys.include?(category)
         args.each { |text| eval("#{Regexp.last_match(1)}train(category, text)") }
       elsif name.to_s =~ /(un)?train_([\w]+)/
         raise StandardError, "No such category: #{category}"
@@ -190,9 +214,17 @@ module ClassifierReborn
     # Provides a list of category names
     # For example:
     #     b.categories
-    #     =>   ['This', 'That', 'the_other']
-    def categories # :nodoc:
-      @categories.keys.collect(&:to_s)
+    #     =>   ["This", "That", "The other"]
+    def categories
+      category_keys.collect(&:to_s)
+    end
+
+    # Provides a list of category keys as symbols
+    # For example:
+    #     b.categories
+    #     =>   [:This, :That, :"The other"]
+    def category_keys
+      @backend.category_keys
     end
 
     # Allows you to add categories to the classifier.
@@ -204,9 +236,37 @@ module ClassifierReborn
     # more criteria than the trained selective categories. In short,
     # try to initialize your categories at initialization.
     def add_category(category)
-      @categories[CategoryNamer.prepare_name(category)] ||= Hash.new(0)
+      category = CategoryNamer.prepare_name(category)
+      @backend.add_category(category)
     end
 
     alias_method :append_category, :add_category
+
+    def reset
+      @backend.reset
+      populate_initial_categories
+    end
+
+    private
+
+    def populate_initial_categories
+      @initial_categories.each do |c|
+        add_category(c)
+      end
+    end
+
+    # Overwrites the default stopwords for current language with supplied list of stopwords or file
+    def custom_stopwords(stopwords)
+      unless stopwords.is_a?(Enumerable)
+        if stopwords.strip.empty?
+          stopwords = []
+        elsif File.exist?(stopwords)
+          stopwords = File.read(stopwords).force_encoding("utf-8").split
+        else
+          return # Do not overwrite the default
+        end
+      end
+      Hasher::STOPWORDS[@language] = Set.new stopwords
+    end
   end
 end