NaiveText 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9a4ff5607deb99718f721a0e6b62636feb91f007
+  data.tar.gz: 6a0dd76a73cc56784bd333b6766ae2375ff6154d
+SHA512:
+  metadata.gz: ad7d2e7dc253ebef99f9ee9fd2dcf87b63f41cd0fc7800102cc071b780a22ace3f554a939fcc90ee65acdeeeea4def99bfe86d3813ff9e59aa2d12f2067a2510
+  data.tar.gz: 4e54cb6f0bd1a0d091bd2b19e0f5afd10cdc85ea37c7cef74d2629d9613fe3a71e540da9fe41f7ed1329b7133933be168e913526c59cafa2563bf569c60b84da

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.2.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in NaiveText.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,48 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exists?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+# Note: The cmd option is now required due to the increasing number of ways
+#       rspec may be run, below are examples of the most common uses.
+#  * bundler: 'bundle exec rspec'
+#  * bundler binstubs: 'bin/rspec'
+#  * spring: 'bin/rspec' (This will use spring if running and you have
+#                          installed the spring binstubs per the docs)
+#  * zeus: 'zeus rspec' (requires the server to be started separately)
+#  * 'just' rspec: 'rspec'
+
+guard :rspec, cmd: "bundle exec rspec" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Feel free to open issues for suggestions and improvements
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+
+end
+
+guard :rubocop, keep_failed: false do
+  watch(%r{(.+\.rb)$}) 
+  watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+end

data/NaiveText.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'NaiveText/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "NaiveText"
+  spec.version       = NaiveText::VERSION
+  spec.authors       = ["RicciFlowing"]
+  spec.email         = ["benjamin@mathe-sellin.de"]
+
+  spec.summary       = "A NaiveText text classifier"
+  spec.description   = "Sort texts based on expample texts in predefined categories"
+  spec.homepage      = "https://github.com/RicciFlowing/NaiveText"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  end
+
+  spec.add_development_dependency "bundler", "~> 1.8"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency "guard-rubocop"
+end

data/README.md

@@ -0,0 +1,75 @@
+# NaiveText
+
+A naive Bayes Textclassifier written in Ruby
+
+1. What does it do?
+----
+
+It sorts text into to categories predefined categories (i.e. interesting/boring).
+The Algorithm bases the decisions on given text examples.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'NaiveText'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install NaiveText
+
+## Usage
+
+In brief the process is as follows:
+
+Create a directory (naming convention: training) containing subdirectories for every category (i.e. 'interesting' and 'boring').
+Fill the directories with examples for interesting/boring texts. This will the training material for the algorithm.
+
+Next up, the code:
+
+```ruby
+require 'NaiveText'
+```
+Now build the systems with your categories and training texts:
+
+```ruby
+categories_config = [{name: 'interesting', path: 'spec/training/positive'},
+                     {name: 'boring', path: 'spec/training/negative'}]
+NaiveText.build(categories_config)
+```
+Now you can start classifying texts:
+
+```ruby
+NaiveText.classify('Seems to be interesting')
+NaiveText.classify('Seems to be boring')
+```
+Classify will return a category-object on which you can call name to get the name of the category as a string.
+
+```ruby
+category = NaiveText.classify('Something interesting')
+category.name
+ => 'interesting'
+```
+
+Have fun using it!
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/RicciFlowing/NaiveText/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "NaiveText"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/NaiveText.rb

@@ -0,0 +1,32 @@
+require "NaiveText/version"
+require "NaiveText/ExamplesGroup"
+require "NaiveText/PropabilityCollection"
+require "NaiveText/PropabilityCalculator"
+require "NaiveText/TextClassifier"
+require "NaiveText/Text"
+require "NaiveText/Category"
+require "NaiveText/Categories"
+require "NaiveText/CategoriesFactory"
+
+module NaiveText
+  extend self
+
+  def build(config)
+    begin
+      @categories = CategoriesFactory.build(config)
+      @test_classifier = TextClassifier.new(categories: @categories)
+    rescue
+      puts "Their seems to be an error in your config.
+       The expectedt format is [{name: name_of_category, path: path_to_trainings_data}]"
+    end
+  end
+
+  def classify(text)
+    begin
+      @text = Text.new(text: text, classifier: @test_classifier)
+      @text.classify
+    rescue
+      puts "An Error occured. Did you call NaiveText.build before using classify"
+    end
+  end
+end

data/lib/NaiveText/Categories.rb

@@ -0,0 +1,23 @@
+class Categories
+  include Enumerable
+
+  def initialize(args)
+    @categories = args[:categories] || []
+  end
+
+  def p_apriori(category)
+    calculate_apriori_propability_for(category)
+  end
+
+  def each(&block)
+    @categories.each(&block)
+  end
+
+  private
+
+   def calculate_apriori_propability_for(category)
+     sum_of_words = @categories.inject(0) {|sum, category| sum + category.word_count }
+     category.word_count.to_f / sum_of_words
+   end
+
+end

data/lib/NaiveText/CategoriesFactory.rb

@@ -0,0 +1,12 @@
+class CategoriesFactory
+  def self.build(config)
+    categories = []
+
+    config.each do |category_config|
+      examples = ExamplesGroup.new(category_config[:path])
+      categories << Category.new(name: category_config[:name], examples: examples)
+    end
+
+    Categories.new(categories: categories)
+  end
+end

data/lib/NaiveText/Category.rb

@@ -0,0 +1,21 @@
+class Category
+  @@id_counter = 1
+
+  attr_reader :name
+  attr_reader :id
+
+  def initialize(args)
+    @name = args[:name]
+    @examples = args[:examples]
+    @id = @@id_counter
+    @@id_counter += 1
+  end
+
+  def p(word)
+    @examples.count(word).to_f / @examples.word_count
+  end
+
+  def word_count
+    @examples.word_count
+  end
+end

data/lib/NaiveText/ExamplesGroup.rb

@@ -0,0 +1,25 @@
+class ExamplesGroup
+  def initialize(path)
+    @text = load_text(path)
+    @words = @text.split(/\W+/)
+  end
+
+  def count(word)
+    @words.count(word)
+  end
+
+  def word_count
+    @words.count
+  end
+
+  private
+
+    def load_text(path)
+      text = ""
+      Dir.foreach(path) do |example_file|
+        next if example_file == '.' or example_file == '..'
+        text += File.read(path +'/'+ example_file)
+      end
+      text
+    end
+end

data/lib/NaiveText/PropabilityCalculator.rb

@@ -0,0 +1,36 @@
+class PropabilityCalculator
+  def initialize(args)
+    @categories = args[:categories] || []
+    @propabilities = PropabilityCollection.new(categories: @categories)
+  end
+
+  def get_propabilities_for(words)
+    calculateProbabilities(words)
+  end
+
+  private
+
+    def calculateProbabilities(list_of_words)
+      @categories.each do |category|
+        @propabilities.set(category: category, value: p_apriori(category))
+      end
+
+      list_of_words.each do |word|
+        @categories.each do |category|
+          @propabilities.multiply(category: category, factor: category.p(word) )
+        end
+      end
+
+      normalization_factor = 1.to_f / @propabilities.sum
+      @propabilities.multiply(factor: normalization_factor)
+
+      @propabilities
+    end
+
+    def p_apriori(category)
+      @categories.p_apriori(category)
+    end
+
+
+
+end

data/lib/NaiveText/PropabilityCollection.rb

@@ -0,0 +1,52 @@
+class PropabilityCollection
+  def initialize(args)
+    @categories  =  args[:categories] || []
+    initialize_ids
+    @propabilities = []
+    initalize_propabilities(@ids)
+  end
+
+  def find(category)
+    return @propabilities[category.id]
+  end
+
+
+  def set(args)
+    category  = args[:category]
+    value = args[:value]
+   @propabilities[category.id] = value
+  end
+
+  def multiply(args)
+    category = args[:category]
+    factor = args[:factor]
+    if category
+      @propabilities[category.id] *= factor
+    else
+      @propabilities.map! {|el| el*factor}
+    end
+  end
+
+  def max
+    id = @propabilities.find_index(@propabilities.max)
+    @categories.find {|category| category.id == id}
+  end
+
+  def sum
+    @propabilities.reduce(:+)
+  end
+
+  private
+  def initialize_ids
+    @ids =  @categories.map { |category| category.id }
+  end
+
+  def initalize_propabilities(ids)
+    ids.max.times do
+      @propabilities << 0
+    end
+  end
+
+
+
+end

data/lib/NaiveText/Text.rb

@@ -0,0 +1,32 @@
+class Text
+  attr_reader :words
+  def initialize( args  )
+    @text = args[:text] || ""
+    path  = args[:path]
+    @classifier = args[:classifier] || TextClassifier.new
+    # If both path and text is given both will be concatenated
+    @text += loadText(path) if path
+
+    @words = @text.split(/\W+/)
+  end
+
+  def sample
+    @text.slice(0,50)
+  end
+
+  def classify
+    @classifier.get_category_for(words)
+  end
+
+  private
+
+    def loadText(path)
+      begin
+      loaded_text = File.read(path)
+      rescue
+        puts "You tried to load the file #{path} for classification. This file was not found.
+         Please make sure, that the path is correctly spelled and that you have reading-access to the path given"
+      end
+    end
+
+end

data/lib/NaiveText/TextClassifier.rb

@@ -0,0 +1,12 @@
+class TextClassifier
+  def initialize( args )
+    @categories = args[:categories]
+    @calculator = args[:calculator] || PropabilityCalculator.new(categories: @categories)
+  end
+
+  def get_category_for(list_of_words)
+    propabilities = @calculator.get_propabilities_for(list_of_words)
+    propabilities.max
+  end
+
+end

data/lib/NaiveText/version.rb

@@ -0,0 +1,3 @@
+module NaiveText
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: NaiveText
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- RicciFlowing
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-10-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Sort texts based on expample texts in predefined categories
+email:
+- benjamin@mathe-sellin.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Guardfile
+- NaiveText.gemspec
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/NaiveText.rb
+- lib/NaiveText/Categories.rb
+- lib/NaiveText/CategoriesFactory.rb
+- lib/NaiveText/Category.rb
+- lib/NaiveText/ExamplesGroup.rb
+- lib/NaiveText/PropabilityCalculator.rb
+- lib/NaiveText/PropabilityCollection.rb
+- lib/NaiveText/Text.rb
+- lib/NaiveText/TextClassifier.rb
+- lib/NaiveText/version.rb
+homepage: https://github.com/RicciFlowing/NaiveText
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: A NaiveText text classifier
+test_files: []