stuff-classifier-zh 0.5.2

data/test/test_001_tokenizer.rb

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+require 'helper.rb'
+
+class Test001Tokenizer < TestBase
+  before do
+    @en_tokenizer = StuffClassifier::Tokenizer.new
+    @fr_tokenizer = StuffClassifier::Tokenizer.new(:language => "fr")
+    @zh_tokenizer = StuffClassifier::Tokenizer.new(:language => 'zh', :stemming => false)
+  end
+
+  def test_simple_tokens
+     words =  @en_tokenizer.each_word('Hello world! How are you?')
+     should_return = ["hello", "world"]
+
+     assert_equal should_return, words
+  end    
+
+  def test_with_stemming
+    words =  @en_tokenizer.each_word('Lots of dogs, lots of cats! This really is the information highway')
+    should_return =["lot", "dog", "lot", "cat", "realli" ,"inform", "highway" ]
+
+    assert_equal should_return, words
+
+  end
+
+  def test_complicated_tokens 
+    words = @en_tokenizer.each_word("I don't really get what you want to
+      accomplish. There is a class TestEval2, you can do test_eval2 =
+      TestEval2.new afterwards. And: class A ... end always yields nil, so
+      your output is ok I guess ;-)")
+    
+    should_return = [
+      "realli", "want", "accomplish", "class",
+      "testeval2",  "test", "eval2","testeval2", "new", "class", "end",
+      "yield", "nil", "output", "ok", "guess"]
+    
+    assert_equal should_return, words
+  end  
+
+  def test_unicode
+  
+    words = @fr_tokenizer.each_word("il s'appelle le vilain petit canard : en référence à Hans Christian Andersen, se démarquer négativement")
+
+    should_return = [
+      "appel", "vilain", "pet", "canard", "référent", 
+      "han", "christian", "andersen", "démarqu", "négat"]
+
+    assert_equal should_return, words      
+  end
+
+  def test_chinese
+  
+    words = @zh_tokenizer.each_word("今天我得到二个最重要的人生启示")
+
+    should_return = [
+      "今天", "我", "得到", "二个", "最重要的", "人生", "启示"
+    ]
+
+    assert_equal should_return, words
+  end
+
+end

data/test/test_002_base.rb

@@ -0,0 +1,38 @@
+require 'helper'
+
+
+class Test002Base < TestBase
+  before do
+    @cls = StuffClassifier::Bayes.new("Cats or Dogs")
+    set_classifier @cls
+    
+    train :dog, "Dogs are awesome, cats too. I love my dog"
+    train :cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog"    
+    train :dog, "My dog's name is Willy. He likes to play with my wife's cat all day long. I love dogs"
+    train :cat, "Cats are difficult animals, unlike dogs, really annoying, I hate them all"
+    train :dog, "So which one should you choose? A dog, definitely."
+    train :cat, "The favorite food for cats is bird meat, although mice are good, but birds are a delicacy"
+    train :dog, "A dog will eat anything, including birds or whatever meat"
+    train :cat, "My cat's favorite place to purr is on my keyboard"
+    train :dog, "My dog's favorite place to take a leak is the tree in front of our house"
+  end
+
+  def test_count 
+    assert @cls.total_cat_count == 9
+    assert @cls.categories.map {|c| @cls.cat_count(c)}.inject(0){|s,count| s+count} == 9
+    
+
+    # compare word count sum to word by cat count sum 
+    assert @cls.word_list.map  {|w| @cls.total_word_count(w[0]) }.inject(0)  {|s,count| s+count}  == 58
+    assert @cls.categories.map {|c| @cls.total_word_count_in_cat(c) }.inject(0){|s,count| s+count}  == 58
+
+    # test word count by categories
+    assert @cls.word_list.map {|w| @cls.word_count(w[0],:dog) }.inject(0)  {|s,count| s+count}  == 29
+    assert @cls.word_list.map {|w| @cls.word_count(w[0],:cat) }.inject(0)  {|s,count| s+count}  == 29
+
+    # for all categories
+    assert @cls.categories.map {|c| @cls.word_list.map {|w| @cls.word_count(w[0],c) }.inject(0) {|s,count| s+count} }.inject(0){|s,count| s+count}  == 58
+
+  end
+
+end

data/test/test_003_naive_bayes.rb

@@ -0,0 +1,56 @@
+require 'helper'
+
+
+class Test003NaiveBayesClassification < TestBase
+  before do
+    set_classifier StuffClassifier::Bayes.new("Cats or Dogs")
+    
+    train :dog, "Dogs are awesome, cats too. I love my dog"
+    train :cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog"    
+    train :dog, "My dog's name is Willy. He likes to play with my wife's cat all day long. I love dogs"
+    train :cat, "Cats are difficult animals, unlike dogs, really annoying, I hate them all"
+    train :dog, "So which one should you choose? A dog, definitely."
+    train :cat, "The favorite food for cats is bird meat, although mice are good, but birds are a delicacy"
+    train :dog, "A dog will eat anything, including birds or whatever meat"
+    train :cat, "My cat's favorite place to purr is on my keyboard"
+    train :dog, "My dog's favorite place to take a leak is the tree in front of our house"
+  end
+
+  def test_for_cats 
+    should_be :cat, "This test is about cats."
+    should_be :cat, "I hate ..."
+    should_be :cat, "The most annoying animal on earth."
+    should_be :cat, "The preferred company of software developers."
+    should_be :cat, "My precious, my favorite!"
+    should_be :cat, "Kill that bird!"
+  end
+
+  def test_for_dogs
+    should_be :dog, "This test is about dogs."
+    should_be :dog, "Cats or Dogs?" 
+    should_be :dog, "What pet will I love more?"    
+    should_be :dog, "Willy, where the heck are you?"
+    should_be :dog, "I like big buts and I cannot lie." 
+    should_be :dog, "Why is the front door of our house open?"
+    should_be :dog, "Who ate my meat?"
+  end
+
+  def test_min_prob
+    classifier.min_prob = 0.001
+    should_be :cat, "This test is about cats."
+    should_be :cat, "I hate ..."
+    should_be nil, "The most annoying animal on earth."
+    should_be nil, "The preferred company of software developers."
+    should_be :cat, "My precious, my favorite!"
+    should_be :cat, "Kill that bird!"
+    should_be :dog, "This test is about dogs."
+    should_be :dog, "Cats or Dogs?" 
+    should_be :dog, "What pet will I love more?"    
+    should_be :dog, "Willy, where the heck are you?"
+    should_be nil, "I like big buts and I cannot lie." 
+    should_be nil, "Why is the front door of our house open?"
+    should_be :dog, "Who ate my meat?"
+  end
+
+
+end

data/test/test_004_tf_idf.rb

@@ -0,0 +1,37 @@
+require 'helper'
+
+
+class Test004TfIdfClassification < TestBase
+  before do
+    set_classifier StuffClassifier::TfIdf.new("Cats or Dogs")
+    
+    train :dog, "Dogs are awesome, cats too. I love my dog"
+    train :cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog"    
+    train :dog, "My dog's name is Willy. He likes to play with my wife's cat all day long. I love dogs"
+    train :cat, "Cats are difficult animals, unlike dogs, really annoying, I hate them all"
+    train :dog, "So which one should you choose? A dog, definitely."
+    train :cat, "The favorite food for cats is bird meat, although mice are good, but birds are a delicacy"
+    train :dog, "A dog will eat anything, including birds or whatever meat"
+    train :cat, "My cat's favorite place to purr is on my keyboard"
+    train :dog, "My dog's favorite place to take a leak is the tree in front of our house"
+  end
+
+  def test_for_cats 
+    should_be :cat, "This test is about cats."
+    should_be :cat, "I hate ..."
+    should_be :cat, "The most annoying animal on earth."
+    should_be :cat, "The preferred company of software developers."
+    should_be :cat, "My precious, my favorite!"
+    should_be :cat, "Kill that bird!"
+  end
+
+  def test_for_dogs
+    should_be :dog, "This test is about dogs."
+    should_be :dog, "Cats or Dogs?" 
+    should_be :dog, "What pet will I love more?"    
+    should_be :dog, "Willy, where the heck are you?"
+    should_be :dog, "I like big buts and I cannot lie." 
+    should_be :dog, "Why is the front door of our house open?"
+    should_be :dog, "Who is eating my meat?"
+  end
+end

data/test/test_005_in_memory_storage.rb

@@ -0,0 +1,31 @@
+require 'helper'
+
+
+class Test005InMemoryStorage < TestBase
+  before do
+    StuffClassifier::Base.storage = StuffClassifier::InMemoryStorage.new
+
+    StuffClassifier::Bayes.open("Cats or Dogs") do |cls|    
+      cls.train(:dog, "Dogs are awesome, cats too. I love my dog")
+      cls.train(:cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog")
+    end
+  end
+
+  def test_for_persistance
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs").instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::InMemoryStorage),
+        "@storage should be an instance of FileStorage"
+      test.assert @word_list.length > 0, "Word count should be persisted"
+      test.assert @category_list.length > 0, "Category count should be persisted"
+    end
+  end
+
+  def test_purge_state
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs", :purge_state => true).instance_eval do
+      test.assert @word_list.length == 0, "Word count should be purged"
+      test.assert @category_list.length == 0, "Category count should be purged"
+    end
+  end
+end

data/test/test_006_file_storage.rb

@@ -0,0 +1,77 @@
+require 'helper'
+
+
+class Test006FileStorage < TestBase
+  before do
+    @storage_path = "/tmp/test_classifier.db"
+    @storage = StuffClassifier::FileStorage.new(@storage_path)
+    StuffClassifier::Base.storage = @storage
+
+    StuffClassifier::Bayes.open("Cats or Dogs") do |cls|    
+      cls.train(:dog, "Dogs are awesome, cats too. I love my dog.")
+      cls.train(:dog, "My dog's name is Willy. He likes to play with my wife's cat all day long. I love dogs")
+      cls.train(:dog, "So which one should you choose? A dog, definitely.")
+      cls.train(:dog, "A dog will eat anything, including birds or whatever meat")
+      cls.train(:dog, "My dog's favorite place to take a leak is the tree in front of our house")
+
+      cls.train(:cat, "My cat's favorite place to purr is on my keyboard")
+      cls.train(:cat, "The favorite food for cats is bird meat, although mice are good, but birds are a delicacy")
+      cls.train(:cat, "Cats are difficult animals, unlike dogs, really annoying, I hate them all")
+      cls.train(:cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog")    
+    end
+
+    # redefining storage instance, forcing it to read from file again
+    StuffClassifier::Base.storage = StuffClassifier::FileStorage.new(@storage_path)
+  end
+
+  def teardown
+    File.unlink @storage_path if File.exists? @storage_path
+  end
+
+  def test_result    
+    set_classifier StuffClassifier::Bayes.new("Cats or Dogs")
+    
+    should_be :cat, "This test is about cats."
+    should_be :cat, "I hate ..."
+    should_be :cat, "The most annoying animal on earth."
+    should_be :cat, "The preferred company of software developers."
+    should_be :cat, "My precious, my favorite!"
+    should_be :cat, "Kill that bird!"
+
+    should_be :dog, "This test is about dogs."
+    should_be :dog, "Cats or Dogs?" 
+    should_be :dog, "What pet will I love more?"    
+    should_be :dog, "Willy, where the heck are you?"
+    should_be :dog, "I like big buts and I cannot lie." 
+    should_be :dog, "Why is the front door of our house open?"
+    should_be :dog, "Who ate my meat?"
+    
+  end
+
+  def test_for_persistance    
+    assert ! @storage.equal?(StuffClassifier::Base.storage),"Storage instance should not be the same"
+
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs").instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::FileStorage),"@storage should be an instance of FileStorage"
+      test.assert @word_list.length > 0, "Word count should be persisted"
+      test.assert @category_list.length > 0, "Category count should be persisted"
+    end
+  end
+
+  def test_file_created
+    assert File.exist?(@storage_path), "File #@storage_path should exist"
+
+    content = File.read(@storage_path)
+    assert content.length > 100, "Serialized content should have more than 100 chars"
+  end
+
+  def test_purge_state
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs", :purge_state => true).instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::FileStorage),"@storage should be an instance of FileStorage"
+      test.assert @word_list.length == 0, "Word count should be purged"
+      test.assert @category_list.length == 0, "Category count should be purged"
+    end
+  end
+end

data/test/test_007_redis_storage.rb

@@ -0,0 +1,81 @@
+require 'helper'
+require 'redis'
+
+
+class Test007RedisStorage < TestBase
+  before do
+    @key = "test_classifier"
+    @redis_options = { host: 'localhost', port: 6379 }
+    @redis = Redis.new(@redis_options)
+
+    @storage = StuffClassifier::RedisStorage.new(@key, @redis_options)
+    StuffClassifier::Base.storage = @storage
+
+    StuffClassifier::Bayes.open("Cats or Dogs") do |cls|
+      cls.train(:dog, "Dogs are awesome, cats too. I love my dog.")
+      cls.train(:dog, "My dog's name is Willy. He likes to play with my wife's cat all day long. I love dogs")
+      cls.train(:dog, "So which one should you choose? A dog, definitely.")
+      cls.train(:dog, "A dog will eat anything, including birds or whatever meat")
+      cls.train(:dog, "My dog's favorite place to take a leak is the tree in front of our house")
+
+      cls.train(:cat, "My cat's favorite place to purr is on my keyboard")
+      cls.train(:cat, "The favorite food for cats is bird meat, although mice are good, but birds are a delicacy")
+      cls.train(:cat, "Cats are difficult animals, unlike dogs, really annoying, I hate them all")
+      cls.train(:cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog")
+    end
+
+    # redefining storage instance, forcing it to read from file again
+    StuffClassifier::Base.storage = StuffClassifier::RedisStorage.new(@key, @redis_options)
+  end
+
+  def teardown
+    @redis.del(@key)
+  end
+
+  def test_result
+    set_classifier StuffClassifier::Bayes.new("Cats or Dogs")
+
+    should_be :cat, "This test is about cats."
+    should_be :cat, "I hate ..."
+    should_be :cat, "The most annoying animal on earth."
+    should_be :cat, "The preferred company of software developers."
+    should_be :cat, "My precious, my favorite!"
+    should_be :cat, "Kill that bird!"
+
+    should_be :dog, "This test is about dogs."
+    should_be :dog, "Cats or Dogs?"
+    should_be :dog, "What pet will I love more?"
+    should_be :dog, "Willy, where the heck are you?"
+    should_be :dog, "I like big buts and I cannot lie."
+    should_be :dog, "Why is the front door of our house open?"
+    should_be :dog, "Who ate my meat?"
+
+  end
+
+  def test_for_persistance
+    assert !@storage.equal?(StuffClassifier::Base.storage),"Storage instance should not be the same"
+
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs").instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::RedisStorage),"@storage should be an instance of RedisStorage"
+      test.assert @word_list.length > 0, "Word count should be persisted"
+      test.assert @category_list.length > 0, "Category count should be persisted"
+    end
+  end
+
+  def test_key_created
+    assert @redis.exists(@key), "Redis key #{@key} should exist"
+
+    content = @redis.get(@key)
+    assert content.length > 100, "Serialized content should have more than 100 chars"
+  end
+
+  def test_purge_state
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs", :purge_state => true).instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::RedisStorage),"@storage should be an instance of RedisStorage"
+      test.assert @word_list.length == 0, "Word count should be purged"
+      test.assert @category_list.length == 0, "Category count should be purged"
+    end
+  end
+end

metadata

@@ -0,0 +1,228 @@
+--- !ruby/object:Gem::Specification
+name: stuff-classifier-zh
+version: !ruby/object:Gem::Version
+  version: 0.5.2
+platform: ruby
+authors:
+- Alexandru Nedelcu
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ruby-stemmer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rmmseg-cpp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 2 methods are provided for now - (1) naive bayes implementation + (2)
+  tf-idf weights
+email:
+- github@contact.bionicspirit.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/stuff-classifier.rb
+- lib/stuff-classifier/base.rb
+- lib/stuff-classifier/bayes.rb
+- lib/stuff-classifier/storage.rb
+- lib/stuff-classifier/tf-idf.rb
+- lib/stuff-classifier/tokenizer.rb
+- lib/stuff-classifier/tokenizer/tokenizer_properties.rb
+- lib/stuff-classifier/version.rb
+- stuff-classifier.gemspec
+- test/helper.rb
+- test/test_001_tokenizer.rb
+- test/test_002_base.rb
+- test/test_003_naive_bayes.rb
+- test/test_004_tf_idf.rb
+- test/test_005_in_memory_storage.rb
+- test/test_006_file_storage.rb
+- test/test_007_redis_storage.rb
+homepage: https://github.com/alexandru/stuff-classifier/
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Simple text classifier(s) implemetation
+test_files:
+- test/helper.rb
+- test/test_001_tokenizer.rb
+- test/test_002_base.rb
+- test/test_003_naive_bayes.rb
+- test/test_004_tf_idf.rb
+- test/test_005_in_memory_storage.rb
+- test/test_006_file_storage.rb
+- test/test_007_redis_storage.rb