omnicat 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1468554ebab3a7d69abfb6b806ed2acc474a0fbb
+  data.tar.gz: 7cafcd36ce8030ba19dc1f48f3be8cc016f49c1d
+SHA512:
+  metadata.gz: ce946b646239bfec96ce58bc296d9b866c72603d5113b56fb4c65418b14457fad6072026fe90330250e04dc4467476d874fdd29327253793b10fdf5a90526f37
+  data.tar.gz: 1281d217ed03836696b9025e368f274941b481ba42d94af5de045fb7c06a6ee231ae8c6bec4e0758fe6e438f707559322f955f60b4c26b7f0e3c1ad3993328b1

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - jruby-19mode
+  - rbx-19mode

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in omnicat.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Mustafa Turan
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# OmniCat
+
+A generalized framework for text classifications. For now, it only supports Naive Bayes algorithm for text classification.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'omnicat'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install omnicat
+
+## Usage
+
+See rdoc for detailed usage.
+
+### Bayes classifier
+Create a Bayes classifier object.
+
+    bayes = OmniCat::Classifiers::Bayes.new
+
+### Create categories
+Create a classification category.
+
+    bayes.add_category('positive')
+    bayes.add_category('negative')
+
+### Train
+Train category with a document.
+
+    bayes.train('positive', 'great if you are in a slap happy mood .')
+    bayes.train('negative', 'bad tracking issue')
+
+### Train batch
+Train category with multiple documents.
+
+    bayes.train_batch('positive', [
+      'a feel-good picture in the best sense of the term...',
+      'it is a feel-good movie about which you can actually feel good.',
+      'love and money both of them are good choises'
+    ])
+    bayes.train_batch('negative', [
+      'simplistic , silly and tedious .',
+      'interesting , but not compelling . ',
+      'seems clever but not especially compelling'
+    ])
+
+### Classify
+Classify a document.
+
+    result = bayes.classify('I feel so good and happy')
+    => #<OmniCat::Result:0x007fe59b97b548 @category={:name=>"negative", :percentage=>99}, @scores={"positive"=>1.749909854122994e-07, "negative"=>0.014084507042253521}, @total_score=0.014084682033238934>
+    result.to_hash
+    => {:category=>{:name=>"negative", :percentage=>99}, :scores=>{"positive"=>1.749909854122994e-07, "negative"=>0.014084507042253521}, :total_score=>0.014084682033238934}
+
+### Classify batch
+Classify multiple documents at a time.
+
+    results = bayes.classify_batch(
+      [
+        'the movie is silly so not compelling enough',
+        'a good piece of work'
+      ]
+    )
+    => [#<OmniCat::Result:0x007fe59b949d90 @category={:name=>"negative", :percentage=>75}, @scores={"positive"=>7.962089836259623e-06, "negative"=>2.5145916163515512e-05}, @total_score=3.3108005999775135e-05>, #<OmniCat::Result:0x007fe59c9d7d10 @category={:name=>"positive", :percentage=>100}, @scores={"positive"=>0.0005434126313247192, "negative"=>0}, @total_score=0.0005434126313247192>]
+
+### Convert to hash
+Convert full Bayes object to hash.
+
+    # For storing, restoring modal data
+    bayes_hash = bayes.to_hash
+
+### Load from hash
+Load full Bayes object from hash.
+
+    another_bayes_obj = OmniCat::Classifiers::Bayes.new(bayes_hash)
+    another_bayes_obj.classify('best senses')
+
+## Todo
+* Add more text classification modules such as Support Vector Machine (SVM).
+* Add text cleaning/manipulating extensions such as stopwords cleaner, stemmer, and pos-tagger, etc...
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Copyright
+Copyright © 2013 Mustafa Turan. See LICENSE for details.
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/omnicat.rb

@@ -0,0 +1,7 @@
+require File.dirname(__FILE__) + '/omnicat/version'
+require File.dirname(__FILE__) + '/omnicat/string'
+require File.dirname(__FILE__) + '/omnicat/array'
+require File.dirname(__FILE__) + '/omnicat/hash'
+require File.dirname(__FILE__) + '/omnicat/base'
+require File.dirname(__FILE__) + '/omnicat/result'
+require File.dirname(__FILE__) + '/omnicat/bayes'

data/lib/omnicat/array.rb

@@ -0,0 +1,9 @@
+class Array
+  def hashify_with_counts
+    hash = {}
+    self.each do |item|
+      hash[item] = hash.has_key?(item) ? (hash[item] + 1) : 1
+    end
+    hash
+  end
+end

data/lib/omnicat/base.rb

@@ -0,0 +1,16 @@
+module OmniCat
+  class Base
+    # Returns Base object as Hash
+    #
+    # @return Base object instance variables in a Hash
+    def to_hash
+      hash = {}
+      self.instance_variables.each do |key|
+        if val = instance_variable_get(key)
+          hash[key[1..-1].to_sym] = val.class.to_s.include?('OmniCat') ? val.to_hash : val
+        end
+      end
+      hash
+    end
+  end
+end

data/lib/omnicat/bayes.rb

@@ -0,0 +1,3 @@
+require File.dirname(__FILE__) + '/classifiers/base'
+require File.dirname(__FILE__) + '/classifiers/bayes'
+require File.dirname(__FILE__) + '/classifiers/bayes_internals/category'

data/lib/omnicat/classifiers/base.rb

@@ -0,0 +1,55 @@
+module OmniCat
+  module Classifiers
+    class Base < ::OmniCat::Base
+      # Allows adding multiple classification categories
+      #
+      # ==== Parameters
+      #
+      # * +names+ - Array of categories
+      #
+      # ==== Examples
+      #
+      #   # Add multiple categories for classification
+      #   bayes.add_categories(["positive", "negative", "neutral"])
+      def add_categories(names)
+        names.each { |name| add_category(name) }
+      end
+
+      # Train the desired category with multiple documents
+      #
+      # ==== Parameters
+      #
+      # * +category+ - Name of the category from added categories list
+      # * +docs+ - Array of documents
+      #
+      # ==== Examples
+      #
+      #   # Add multiple docs for training the category
+      #   bayes.train("positive", ["clear documentation", "good, very well"])
+      #   bayes.train("negative", ["bad interface", "damn"])
+      def train_batch(category, docs)
+        docs.each { |doc| train(category, doc) }
+      end
+
+      # Classify the multiple documents at a time
+      #
+      # ==== Parameters
+      #
+      # * +docs+ - Array of documents
+      #
+      # ==== Returns
+      #
+      # * +result_set+ - Array of OmniCat::Result objects
+      #
+      # ==== Examples
+      #
+      #   # Classify multiple documents
+      #   bayes.classify_batch(["good documentation", "damn workin again"])
+      #   =>
+      def classify_batch(docs)
+        docs.collect { |doc| classify(doc) }
+      end
+
+    end
+  end
+end

data/lib/omnicat/classifiers/bayes.rb

@@ -0,0 +1,127 @@
+module OmniCat
+  module Classifiers
+    class Bayes < ::OmniCat::Classifiers::Base
+
+      attr_accessor :categories, :category_count, :doc_count, :token_count
+      attr_accessor :k_value # helper val for skipping some Bayes theorem errors
+
+      def initialize(bayes_hash = {})
+        self.categories = ::OmniCat::Hash.new
+        if bayes_hash.has_key?(:categories)
+          bayes_hash[:categories].each do |name, category|
+            self.categories[name] = ::OmniCat::Classifiers::BayesInternals::Category.new(category)
+          end
+        end
+        self.category_count = bayes_hash[:category_count].to_i
+        self.doc_count = bayes_hash[:doc_count].to_i
+        self.k_value = bayes_hash[:k_value] || 1.0
+        self.token_count = bayes_hash[:token_count].to_i
+      end
+
+      # Allows adding new classification category
+      #
+      # ==== Parameters
+      #
+      # * +name+ - Name for category
+      #
+      # ==== Examples
+      #
+      #   # Create a classification category
+      #   bayes = Bayes.new
+      #   bayes.add_category("positive")
+      def add_category(name)
+        if category_exists?(name)
+          raise StandardError,
+                "Category with name '#{name}' is already exists!"
+        else
+          self.category_count +=1
+          self.categories[name] = ::OmniCat::Classifiers::BayesInternals::Category.new
+        end
+      end
+
+      # Train the desired category with a document
+      #
+      # ==== Parameters
+      #
+      # * +category+ - Name of the category from added categories list
+      # * +doc+ - Document text
+      #
+      # ==== Examples
+      #
+      #   # Train the desired category
+      #   bayes.train("positive", "clear documentation")
+      #   bayes.train("positive", "good, very well")
+      #   bayes.train("negative", "bad dog")
+      #   bayes.train("neutral", "how is the management gui")
+      def train(category, doc)
+        if category_exists?(category)
+          self.doc_count += 1
+          categories[category].doc_count += 1
+          doc.tokenize_with_counts.each do |token, count|
+            self.token_count += count
+            self.categories[category].tokens[token] = self.categories[category].tokens[token].to_i + count
+            self.categories[category].token_count += count
+          end
+        else
+          raise StandardError,
+                "Category with name '#{category}' does not exist!"
+        end
+      end
+
+      # Classify the given document
+      #
+      # ==== Parameters
+      #
+      # * +doc+ - The document for classification
+      #
+      # ==== Returns
+      #
+      # * +result+ - OmniCat::Result object
+      #
+      # ==== Examples
+      #
+      #   # Classify a document
+      #   bayes.classify("good documentation")
+      #   =>
+      def classify(doc)
+        if category_count < 2
+          return raise StandardError,
+                       "At least 2 categories needed for classification process!"
+        end
+        score = -1000000
+        result = ::OmniCat::Result.new
+        categories.each do |name, category|
+          prior = category.doc_count / doc_count.to_f
+          result.scores[name] = k_value
+          doc.tokenize_with_counts.each do |token, count|
+            result.scores[name] *= (
+              (category.tokens[token].to_i + k_value) /
+              (category.token_count + token_count)
+            ) if category.tokens.has_key?(token)
+          end
+          result.scores[name] = (
+            result.scores[name].to_f == 1.0 ? 0 : (prior * result.scores[name])
+          )
+          if result.scores[name] > score
+            result.category[:name] = name;
+            score = result.scores[name];
+          end
+          result.total_score += result.scores[name]
+        end
+        result.total_score = 1 if result.total_score == 0
+        result.category[:percentage] = (
+          result.scores[result.category[:name]] * 100.0 /
+          result.total_score
+        ).floor
+        result
+      end
+
+      private
+        # nodoc
+        def category_exists?(category_name)
+          categories.has_key?(category_name)
+        end
+
+    end
+  end
+end

data/lib/omnicat/classifiers/bayes_internals/category.rb

@@ -0,0 +1,15 @@
+module OmniCat
+  module Classifiers
+    module BayesInternals
+      class Category < ::OmniCat::Base
+        attr_accessor :doc_count, :tokens, :token_count
+
+        def initialize(category_hash = {})
+          self.doc_count = category_hash[:doc_count].to_i
+          self.tokens = category_hash[:tokens] || {}
+          self.token_count = category_hash[:token_count].to_i
+        end
+      end
+    end
+  end
+end

data/lib/omnicat/hash.rb

@@ -0,0 +1,13 @@
+module OmniCat
+  class Hash < ::Hash
+    def to_hash
+      hash = {}
+      self.keys.each do |key|
+        if val = self[key]
+          hash[key] = val.class.to_s.include?('OmniCat') ? val.to_hash : val
+        end
+      end
+      hash
+    end
+  end
+end

data/lib/omnicat/result.rb

@@ -0,0 +1,11 @@
+module OmniCat
+  class Result < ::OmniCat::Base
+    attr_accessor :category, :scores, :total_score
+
+    def initialize
+      self.category = {}
+      self.scores = {}
+      self.total_score = 0
+    end
+  end
+end

data/lib/omnicat/string.rb

@@ -0,0 +1,9 @@
+class String
+  def omnicat_tokenize
+    self.scan(/([\p{L}\-0-9]{2,})/).collect{ |str_arr| str_arr.first }
+  end
+
+  def tokenize_with_counts
+    self.omnicat_tokenize.hashify_with_counts
+  end
+end

data/lib/omnicat/version.rb

@@ -0,0 +1,3 @@
+module OmniCat
+  VERSION = "0.1.0"
+end

data/lib/test/test_helper.rb

@@ -0,0 +1,2 @@
+require 'test/unit'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'omnicat'))

data/lib/test/unit/array_test.rb

@@ -0,0 +1,10 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'test_helper'))
+
+class TestArray < Test::Unit::TestCase
+  def test_hashify_with_counts
+    assert_equal(
+      {"omnicat" => 2, "written" => 1, "at" => 1, "2011" => 1},
+      ["omnicat", "written", "at", "2011", "omnicat"].hashify_with_counts
+    )
+  end
+end

data/lib/test/unit/base_test.rb

@@ -0,0 +1,49 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'test_helper'))
+
+class TestBase < Test::Unit::TestCase
+  def setup
+    @bayes = OmniCat::Classifiers::Bayes.new
+  end
+
+  def test_add_categories
+    @bayes.add_categories ["neutral", "positive", "negative"]
+    assert_not_nil(@bayes.categories["neutral"])
+    assert_equal(
+      ["neutral", "positive", "negative"],
+      @bayes.categories.keys
+    )
+  end
+
+  def test_train_batch
+    @bayes.add_category "positive"
+    @bayes.train_batch "positive", ["good job ever", "valid syntax",
+      "best moments of my life"]
+    assert_equal(
+      3,
+      @bayes.categories["positive"].doc_count
+    )
+  end
+
+  def test_classify_batch
+    @bayes.add_category "positive"
+    @bayes.add_category "negative"
+    @bayes.train_batch "positive", ["good job ever", "valid syntax",
+      "best moments of my life"]
+    @bayes.train_batch("negative", ["bad work", "awfull day", "never liked it"])
+    results = @bayes.classify_batch(
+      ["good sytanx research", "bad words"]
+    )
+
+    assert_equal(2, results.count)
+
+    assert_equal(
+      "positive",
+      results[0].category[:name]
+    )
+    assert_equal(
+      "negative",
+      results[1].category[:name]
+    )
+
+  end
+end

data/lib/test/unit/bayes_test.rb

@@ -0,0 +1,85 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'test_helper'))
+
+class TestBayes < Test::Unit::TestCase
+  def setup
+    @bayes = OmniCat::Classifiers::Bayes.new
+  end
+
+  def test_add_category
+    @bayes.add_category "neutral"
+    assert_not_nil(@bayes.categories["neutral"])
+    assert_equal(
+      ["neutral"],
+      @bayes.categories.keys
+    )
+    assert_equal(
+      0,
+      @bayes.categories["neutral"].doc_count
+    )
+    assert_equal(
+      {},
+      @bayes.categories["neutral"].tokens
+    )
+    assert_equal(
+      0,
+      @bayes.categories["neutral"].token_count
+    )
+  end
+
+  def test_add_category_that_already_exists
+    @bayes.add_category "neutral"
+    assert_raise(StandardError) { @bayes.add_category "neutral" }
+  end
+
+  def test_train_valid_category
+    @bayes.add_category "neutral"
+    @bayes.train "neutral", "how are you?"
+    assert_equal(
+      1,
+      @bayes.categories["neutral"].doc_count
+    )
+    assert_equal(
+      {"how" => 1, "are" => 1, "you" => 1},
+      @bayes.categories["neutral"].tokens
+    )
+    assert_equal(
+      3,
+      @bayes.categories["neutral"].token_count
+    )
+  end
+
+  def test_train_missing_category
+    assert_raise(StandardError) { @bayes.train "neutral", "how are you?" }
+  end
+
+  def test_classify
+    @bayes.add_category "positive"
+    @bayes.add_category "negative"
+    @bayes.train("positive", "good job")
+    @bayes.train("negative", "bad work")
+    assert_equal(
+      "positive",
+      @bayes.classify("very good position for this sentence").category[:name]
+    )
+    assert_equal(
+      "negative",
+      @bayes.classify("bad words").category[:name]
+    )
+  end
+
+  def test_initialize_with_hash
+    bayes1 = ::OmniCat::Classifiers::Bayes.new
+    bayes1.add_category "positive"
+    bayes1.add_category "negative"
+    bayes1.train("positive", "good job")
+    bayes1.train("negative", "bad work")
+    h1 = bayes1.to_hash
+
+    bayes2 = ::OmniCat::Classifiers::Bayes.new(h1)
+    assert_equal(h1, bayes2.to_hash)
+  end
+
+  def test_classify_with_insufficient_categories
+    assert_raise(StandardError) { @bayes.classify "blank" }
+  end
+end

data/lib/test/unit/hash_test.rb

@@ -0,0 +1,10 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'test_helper'))
+
+class TestHash < Test::Unit::TestCase
+  def test_to_hash
+    categories_hash = { "pos" => { doc_count: 0, tokens: {}, token_count: 0 } }
+    categories = OmniCat::Hash.new
+    categories["pos"] = OmniCat::Classifiers::BayesInternals::Category.new(categories_hash["pos"])
+    assert_equal(categories_hash, categories.to_hash)
+  end
+end

data/lib/test/unit/string_test.rb

@@ -0,0 +1,17 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'test_helper'))
+
+class TestString < Test::Unit::TestCase
+  def test_omnicat_tokenize
+    assert_equal(
+      ["mustafa", "turan", "omni-cat-v0", "1986"],
+      "mustafa turan omni-cat-v0 1986 1 a s d".omnicat_tokenize
+    )
+  end
+
+  def test_tokenize_with_counts
+    assert_equal(
+      {"omnicat" => 2, "written" => 1, "at" => 1, "2011" => 1},
+      "omnicat written at 2011, omnicat".tokenize_with_counts
+    )
+  end
+end

data/omnicat.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'omnicat/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "omnicat"
+  spec.version       = OmniCat::VERSION
+  spec.authors       = ["Mustafa Turan"]
+  spec.email         = ["mustafaturan.net@gmail.com"]
+  spec.description   = %q{A generalized framework for text classifications.}
+  spec.summary       = spec.description
+  spec.homepage      = "https://github.com/mustafaturan/omnicat"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: omnicat
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Mustafa Turan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A generalized framework for text classifications.
+email:
+- mustafaturan.net@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/omnicat.rb
+- lib/omnicat/array.rb
+- lib/omnicat/base.rb
+- lib/omnicat/bayes.rb
+- lib/omnicat/classifiers/base.rb
+- lib/omnicat/classifiers/bayes.rb
+- lib/omnicat/classifiers/bayes_internals/category.rb
+- lib/omnicat/hash.rb
+- lib/omnicat/result.rb
+- lib/omnicat/string.rb
+- lib/omnicat/version.rb
+- lib/test/test_helper.rb
+- lib/test/unit/array_test.rb
+- lib/test/unit/base_test.rb
+- lib/test/unit/bayes_test.rb
+- lib/test/unit/hash_test.rb
+- lib/test/unit/string_test.rb
+- omnicat.gemspec
+homepage: https://github.com/mustafaturan/omnicat
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: A generalized framework for text classifications.
+test_files: []