NaiveText 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84a7df9d9c4c9195b0fa10bfc3073ec5defbdf55
-  data.tar.gz: ee3659c08cbd21cd16e8a2d5fdedbc080c6f90bb
+  metadata.gz: 24121c5efbc4119de814e59e3080b3770c822929
+  data.tar.gz: 2638006388fe41e21918f6859545efdb5e9f4526
 SHA512:
-  metadata.gz: 2595416159567c728317e57e0e139ed73796c32e6248d7333bb3f5210d6a7430fec6fafab595e9606daf9975d9326980db1197440cbb59a058ce130164d2f093
-  data.tar.gz: d8cf7edac37f604f8263eedb3f6bf1b988811c67d216b7b730eb0e1a2d0afe0ec0f0b663ea4cac3f2038dda6f6fd606226b4b0683ecb7ec530059c26d21b6213
+  metadata.gz: 18fecd3d98353ed3d8010af35795a7bda536bf48cd072ad9674b807f78d8988e8f2b7babc3f56bd7f2d6f4caee614998450ade5dada44cfef7eb79b746f84c7a
+  data.tar.gz: c94b0036e3ce3f145ac08beca161ec0d39227ab229938bb5466acbc8f8d0364808f0115dcb6aba162ead52d815da577565681bea71339ef735aa6b6775463f53

data/README.md

@@ -5,8 +5,8 @@ A naive Bayes Textclassifier written in Ruby
 1. What does it do?
 ----
 
-It sorts text into predefined categories (i.e. interesting/boring).
-The algorithm bases its decisions on given text examples.
+It sorts texts into predefined categories (i.e. interesting/boring).
+The algorithm bases its decisions on classified trainingdata (text files, ActiveRecord models,...).
 
 ## Installation
 
@@ -26,34 +26,39 @@ Or install it yourself as:
 
 ## Usage
 
-Create a directory for the trainings-data containing subdirectories for every category (i.e. 'interesting' and 'boring'). Fill the directories with examples for interesting/boring texts.
+The algorithm needs some examples for training. An example is a object with an id that responds to the text message with a string (i.e. ActiveRecord models with an text attribute will do).
+You can also use local files as examples (via ExamplesFactory.from_files('path/to/dir')).
+
+
+
+### Example
+
+Lets pretend you write some kind of forum. A user can write posts and can vote them up or down.
+
+
+We will build a system which predicts if a new post is interesting to the user or if this post will bore him a sleep.
+
+In your system (an rails app of course) you haven a *Post* model with a text attribute containing the posts content. There are also two scopes on Post: *up_voted* and *down_voted*, which return all up/down voted posts.
+
 
-Next up, the code:
 
 ```ruby
 require 'NaiveText'
-```
-Now build the systems with your categories and training texts:
 
-```ruby
-categories_config = [{name: 'interesting', path: 'spec/training/positive'},
-                     {name: 'boring', path: 'spec/training/negative'}]
-classifier = NaiveText.build(categories_config)
-```
-Now you can start classifying texts:
+interesting_examples = Post.up_voted.to_a
+boring_examples = Post.down_voted.to_a
 
-```ruby
-classifier.classify('Seems to be interesting')
-classifier.classify('Seems to be boring')
-```
-Classify will return a category-object on which you can call name to get the name of the category as a string.
+categories = [{name: 'interesting', examples: interesting_examples},
+                     {name: 'boring', examples: boring_examples}];
 
-```ruby
-category = classifier.classify('Something interesting')
+classifier = NaiveText.build(categories: categories)
+
+category = classifier.classify(new_interesting_post.text)
 category.name
  => 'interesting'
 ```
-
+Checkout the full example and some more in the
+[NaiveText-example repo](https://github.com/RicciFlowing/NaiveText-examples).
 Have fun using it!
 
 

data/lib/NaiveText.rb

@@ -1,4 +1,6 @@
 require "NaiveText/version"
+require "NaiveText/Example"
+require "NaiveText/ExamplesFactory"
 require "NaiveText/ExamplesGroup"
 require "NaiveText/PropabilityCollection"
 require "NaiveText/PropabilityCalculator"
@@ -12,12 +14,7 @@ require "NaiveText/CategoriesFactory"
 module NaiveText
 
   def self.build(config)
-    begin
       @categories = CategoriesFactory.build(config)
       @test_classifier = TextClassifier.new(categories: @categories)
-    rescue
-      puts "Their seems to be an error in your config.
-       The expectedt format is [{name: name_of_category, path: path_to_trainings_data}]"
-    end
   end
 end

data/lib/NaiveText/CategoriesFactory.rb

@@ -1,17 +1,30 @@
 class CategoriesFactory
   def self.build(config)
     categories = []
-
-    config.each do |category_config|
-      begin
-        examples = ExamplesGroup.new(category_config[:path])
-        categories << Category.new(name: category_config[:name], examples: examples)
-      rescue
-        puts "You haven't provided trainingsdata for the category" + category_config[:name]
-        puts "This category was not created."
+    if config.is_a?(Array)
+      puts "The format [{name: name_of_category, path: path_to_trainings_data}] is deprecated and will be removed in future versions. Use the following arguments instead: categories: [name: 'the name', examples:'An example']"
+      config.each do |category_config|
+        begin
+          examples = ExamplesFactory.from_files(category_config[:path])
+          group = ExamplesGroup.new(examples: examples)
+          categories << Category.new(name: category_config[:name], examples: group)
+        rescue
+          puts "You haven't provided trainingsdata for the category" + category_config[:name]
+          puts "This category was not created."
+        end
       end
+      Categories.new(categories: categories)
+    else
+      config[:categories].each do |category_config|
+        begin
+          group = ExamplesGroup.new(examples: category_config[:examples])
+          categories << Category.new(name: category_config[:name], examples: group)
+        rescue
+          puts "You haven't provided trainingsdata for the category" + category_config[:name]
+          puts "This category was not created."
+        end
+      end
+      Categories.new(categories: categories)
     end
-
-    Categories.new(categories: categories)
   end
 end

data/lib/NaiveText/Category.rb

@@ -33,7 +33,7 @@ class NullCategory
   attr_reader :id
 
   def initialize
-    @name = "No Category"
+    @name = "No category"
     @id = 0
   end
 end

data/lib/NaiveText/Example.rb

@@ -0,0 +1,19 @@
+class Example
+  attr_reader :text
+
+  def initialize(args)
+    load_text(args)
+  end
+
+  private
+    def load_text(args)
+
+    end
+end
+
+class FileExample < Example
+  private
+    def load_text(args)
+      @text = File.read(args[:path])
+    end
+end

data/lib/NaiveText/ExamplesFactory.rb

@@ -0,0 +1,14 @@
+class ExamplesFactory
+  def self.from_files(dir_path)
+      begin
+        examples = []
+        Dir.foreach(dir_path) do |file_path|
+          next if file_path == '.' or file_path == '..'
+          examples.push FileExample.new(path: dir_path+'/'+file_path)
+        end
+      rescue
+        puts "Failed laoding" + dir_path
+      end
+    examples
+  end
+end

data/lib/NaiveText/ExamplesGroup.rb

@@ -1,14 +1,16 @@
 class ExamplesGroup
-  def initialize(path)
-    @text = load_text(path)
-    @words = @text.split(/\W+/)
+  def initialize(args)
+    @examples = args[:examples] || []
+    load_text
+    split_text_into_words
+    format_words
     if @words.length == 0
       raise 'Empty_Trainingsdata'
     end
   end
 
   def count(word)
-    @words.count(word)
+    @words.count(word.downcase)
   end
 
   def word_count
@@ -17,12 +19,18 @@ class ExamplesGroup
 
   private
 
-    def load_text(path)
-      text = ""
-      Dir.foreach(path) do |example_file|
-        next if example_file == '.' or example_file == '..'
-        text += File.read(path +'/'+ example_file)
+    def load_text
+      @text = ''
+      @examples.each do |example|
+        @text += ' ' + example.text
       end
-      text
+    end
+
+    def split_text_into_words
+      @words = @text.split(/\W+/)
+    end
+
+    def format_words
+      @words.map! {|word| word.downcase}
     end
 end

data/lib/NaiveText/PropabilityCalculator.rb

@@ -6,34 +6,40 @@ class PropabilityCalculator
 
   def get_propabilities_for(text)
     calculateProbabilities(text)
-    normalize unless @propabilities.sum < 0
+    normalize unless @propabilities.sum <= 0
     @propabilities
   end
 
 
   private
-    def minimum
-      minimum = 1.to_f/(10*@categories.total_word_count)
+    def protect_factor(factor)
+      [factor, minimum].max
     end
 
-    def min_factor(factor)
-      if factor.to_f < minimum
-        factor = minimum
-      end
-      factor
+    def minimum
+      1.to_f/(10*@categories.total_word_count)
     end
 
     def calculateProbabilities(text)
+      set_apriori_propabilities
       list_of_words = text.split(/\W+/)
-      @categories.each do |category|
-        @propabilities.set(category: category, value: p_apriori(category))
-      end
-
       list_of_words.each do |word|
         @categories.each do |category|
-          @propabilities.multiply(category: category, factor: min_factor(category.p(word)) )
+          @propabilities.multiply(category: category, factor: protect_factor(category.p(word)) )
         end
       end
+      remove_minimum(text)
+    end
+
+    def set_apriori_propabilities
+      @categories.each do |category|
+        @propabilities.set(category: category, value: p_apriori(category))
+      end
+    end
+
+    def remove_minimum(text)
+      times = text.split(/\W+/).length
+      @propabilities.greater_then(minimum**times)
     end
 
     def normalize
@@ -44,6 +50,4 @@ class PropabilityCalculator
     def p_apriori(category)
       @categories.p_apriori(category)
     end
-
-
 end

data/lib/NaiveText/PropabilityCollection.rb

@@ -28,14 +28,28 @@ class PropabilityCollection
   end
 
   def category_with_max
-    id = @propabilities.find_index(@propabilities.max)
-    @categories.find {|category| category.id == id}
+    if @propabilities.max > 0
+      id = @propabilities.find_index(@propabilities.max)
+      @categories.find {|category| category.id == id}
+    else
+      NullCategory.new
+    end
   end
 
   def max
     @propabilities.max
   end
 
+  def greater_then(value)
+    @propabilities.map! do |p|
+       if p > value
+         p
+       else
+         0
+       end
+    end
+  end
+
   def sum
     @propabilities.reduce(:+)
   end
@@ -61,7 +75,4 @@ class PropabilityCollection
       @propabilities << 0
     end
   end
-
-
-
 end

data/lib/NaiveText/version.rb

@@ -1,3 +1,3 @@
 module NaiveText
-  VERSION = "0.3.3"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: NaiveText
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - RicciFlowing
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-10-18 00:00:00.000000000 Z
+date: 2015-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,6 +102,8 @@ files:
 - lib/NaiveText/Categories.rb
 - lib/NaiveText/CategoriesFactory.rb
 - lib/NaiveText/Category.rb
+- lib/NaiveText/Example.rb
+- lib/NaiveText/ExamplesFactory.rb
 - lib/NaiveText/ExamplesGroup.rb
 - lib/NaiveText/PropabilityCalculator.rb
 - lib/NaiveText/PropabilityCollection.rb