stuff-classifier-chinese 0.51

data/test/test_002_base.rb

@@ -0,0 +1,39 @@
+# -*- encoding : utf-8 -*-
+require './helper'
+
+
+class Test002Base < TestBase
+  before do
+    @cls = StuffClassifier::Bayes.new("Cats or Dogs")
+    set_classifier @cls
+    
+    train :dog, "Dogs are awesome, cats too. I love my dog"
+    train :cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog"    
+    train :dog, "My dog's name is Willy. He likes to play with my wife's cat all day long. I love dogs"
+    train :cat, "Cats are difficult animals, unlike dogs, really annoying, I hate them all"
+    train :dog, "So which one should you choose? A dog, definitely."
+    train :cat, "The favorite food for cats is bird meat, although mice are good, but birds are a delicacy"
+    train :dog, "A dog will eat anything, including birds or whatever meat"
+    train :cat, "My cat's favorite place to purr is on my keyboard"
+    train :dog, "My dog's favorite place to take a leak is the tree in front of our house"
+  end
+
+  def test_count 
+    assert @cls.total_cat_count == 9
+    assert @cls.categories.map {|c| @cls.cat_count(c)}.inject(0){|s,count| s+count} == 9
+    
+
+    # compare word count sum to word by cat count sum 
+    assert @cls.word_list.map  {|w| @cls.total_word_count(w[0]) }.inject(0)  {|s,count| s+count}  == 58
+    assert @cls.categories.map {|c| @cls.total_word_count_in_cat(c) }.inject(0){|s,count| s+count}  == 58
+
+    # test word count by categories
+    assert @cls.word_list.map {|w| @cls.word_count(w[0],:dog) }.inject(0)  {|s,count| s+count}  == 29
+    assert @cls.word_list.map {|w| @cls.word_count(w[0],:cat) }.inject(0)  {|s,count| s+count}  == 29
+
+    # for all categories
+    assert @cls.categories.map {|c| @cls.word_list.map {|w| @cls.word_count(w[0],c) }.inject(0) {|s,count| s+count} }.inject(0){|s,count| s+count}  == 58
+
+  end
+
+end

data/test/test_003_naive_bayes.rb

@@ -0,0 +1,57 @@
+# -*- encoding : utf-8 -*-
+require './helper'
+
+
+class Test003NaiveBayesClassification < TestBase
+  before do
+    set_classifier StuffClassifier::Bayes.new("Cats or Dogs")
+    
+    train :dog, "Dogs are awesome, cats too. I love my dog"
+    train :cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog"    
+    train :dog, "My dog's name is Willy. He likes to play with my wife's cat all day long. I love dogs"
+    train :cat, "Cats are difficult animals, unlike dogs, really annoying, I hate them all"
+    train :dog, "So which one should you choose? A dog, definitely."
+    train :cat, "The favorite food for cats is bird meat, although mice are good, but birds are a delicacy"
+    train :dog, "A dog will eat anything, including birds or whatever meat"
+    train :cat, "My cat's favorite place to purr is on my keyboard"
+    train :dog, "My dog's favorite place to take a leak is the tree in front of our house"
+  end
+
+  def test_for_cats 
+    should_be :cat, "This test is about cats."
+    should_be :cat, "I hate ..."
+    should_be :cat, "The most annoying animal on earth."
+    should_be :cat, "The preferred company of software developers."
+    should_be :cat, "My precious, my favorite!"
+    should_be :cat, "Kill that bird!"
+  end
+
+  def test_for_dogs
+    should_be :dog, "This test is about dogs."
+    should_be :dog, "Cats or Dogs?" 
+    should_be :dog, "What pet will I love more?"    
+    should_be :dog, "Willy, where the heck are you?"
+    should_be :dog, "I like big buts and I cannot lie." 
+    should_be :dog, "Why is the front door of our house open?"
+    should_be :dog, "Who ate my meat?"
+  end
+
+  def test_min_prob
+    classifier.min_prob = 0.001
+    should_be :cat, "This test is about cats."
+    should_be :cat, "I hate ..."
+    should_be nil, "The most annoying animal on earth."
+    should_be nil, "The preferred company of software developers."
+    should_be :cat, "My precious, my favorite!"
+    should_be :cat, "Kill that bird!"
+    should_be :dog, "This test is about dogs."
+    should_be :dog, "Cats or Dogs?" 
+    should_be :dog, "What pet will I love more?"    
+    should_be :dog, "Willy, where the heck are you?"
+    should_be nil, "I like big buts and I cannot lie." 
+    should_be nil, "Why is the front door of our house open?"
+    should_be :dog, "Who ate my meat?"
+  end
+
+
+end

data/test/test_004_tf_idf.rb

@@ -0,0 +1,38 @@
+# -*- encoding : utf-8 -*-
+require './helper'
+
+
+class Test004TfIdfClassification < TestBase
+  before do
+    set_classifier StuffClassifier::TfIdf.new("Cats or Dogs")
+    
+    train :dog, "Dogs are awesome, cats too. I love my dog"
+    train :cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog"    
+    train :dog, "My dog's name is Willy. He likes to play with my wife's cat all day long. I love dogs"
+    train :cat, "Cats are difficult animals, unlike dogs, really annoying, I hate them all"
+    train :dog, "So which one should you choose? A dog, definitely."
+    train :cat, "The favorite food for cats is bird meat, although mice are good, but birds are a delicacy"
+    train :dog, "A dog will eat anything, including birds or whatever meat"
+    train :cat, "My cat's favorite place to purr is on my keyboard"
+    train :dog, "My dog's favorite place to take a leak is the tree in front of our house"
+  end
+
+  def test_for_cats 
+    should_be :cat, "This test is about cats."
+    should_be :cat, "I hate ..."
+    should_be :cat, "The most annoying animal on earth."
+    should_be :cat, "The preferred company of software developers."
+    should_be :cat, "My precious, my favorite!"
+    should_be :cat, "Kill that bird!"
+  end
+
+  def test_for_dogs
+    should_be :dog, "This test is about dogs."
+    should_be :dog, "Cats or Dogs?" 
+    should_be :dog, "What pet will I love more?"    
+    should_be :dog, "Willy, where the heck are you?"
+    should_be :dog, "I like big buts and I cannot lie." 
+    should_be :dog, "Why is the front door of our house open?"
+    should_be :dog, "Who is eating my meat?"
+  end
+end

data/test/test_005_in_memory_storage.rb

@@ -0,0 +1,32 @@
+# -*- encoding : utf-8 -*-
+require './helper'
+
+
+class Test005InMemoryStorage < TestBase
+  before do
+    StuffClassifier::Base.storage = StuffClassifier::InMemoryStorage.new
+
+    StuffClassifier::Bayes.open("Cats or Dogs") do |cls|    
+      cls.train(:dog, "Dogs are awesome, cats too. I love my dog")
+      cls.train(:cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog")
+    end
+  end
+
+  def test_for_persistance
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs").instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::InMemoryStorage),
+        "@storage should be an instance of FileStorage"
+      test.assert @word_list.length > 0, "Word count should be persisted"
+      test.assert @category_list.length > 0, "Category count should be persisted"
+    end
+  end
+
+  def test_purge_state
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs", :purge_state => true).instance_eval do
+      test.assert @word_list.length == 0, "Word count should be purged"
+      test.assert @category_list.length == 0, "Category count should be purged"
+    end
+  end
+end

data/test/test_006_file_storage.rb

@@ -0,0 +1,78 @@
+# -*- encoding : utf-8 -*-
+require './helper'
+
+
+class Test006FileStorage < TestBase
+  before do
+    @storage_path = "/tmp/test_classifier.db"
+    @storage = StuffClassifier::FileStorage.new(@storage_path)
+    StuffClassifier::Base.storage = @storage
+
+    StuffClassifier::Bayes.open("Cats or Dogs") do |cls|    
+      cls.train(:dog, "Dogs are awesome, cats too. I love my dog.")
+      cls.train(:dog, "My dog's name is Willy. He likes to play with my wife's cat all day long. I love dogs")
+      cls.train(:dog, "So which one should you choose? A dog, definitely.")
+      cls.train(:dog, "A dog will eat anything, including birds or whatever meat")
+      cls.train(:dog, "My dog's favorite place to take a leak is the tree in front of our house")
+
+      cls.train(:cat, "My cat's favorite place to purr is on my keyboard")
+      cls.train(:cat, "The favorite food for cats is bird meat, although mice are good, but birds are a delicacy")
+      cls.train(:cat, "Cats are difficult animals, unlike dogs, really annoying, I hate them all")
+      cls.train(:cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog")    
+    end
+
+    # redefining storage instance, forcing it to read from file again
+    StuffClassifier::Base.storage = StuffClassifier::FileStorage.new(@storage_path)
+  end
+
+  def teardown
+    File.unlink @storage_path if File.exists? @storage_path
+  end
+
+  def test_result    
+    set_classifier StuffClassifier::Bayes.new("Cats or Dogs")
+    
+    should_be :cat, "This test is about cats."
+    should_be :cat, "I hate ..."
+    should_be :cat, "The most annoying animal on earth."
+    should_be :cat, "The preferred company of software developers."
+    should_be :cat, "My precious, my favorite!"
+    should_be :cat, "Kill that bird!"
+
+    should_be :dog, "This test is about dogs."
+    should_be :dog, "Cats or Dogs?" 
+    should_be :dog, "What pet will I love more?"    
+    should_be :dog, "Willy, where the heck are you?"
+    should_be :dog, "I like big buts and I cannot lie." 
+    should_be :dog, "Why is the front door of our house open?"
+    should_be :dog, "Who ate my meat?"
+    
+  end
+
+  def test_for_persistance    
+    assert ! @storage.equal?(StuffClassifier::Base.storage),"Storage instance should not be the same"
+
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs").instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::FileStorage),"@storage should be an instance of FileStorage"
+      test.assert @word_list.length > 0, "Word count should be persisted"
+      test.assert @category_list.length > 0, "Category count should be persisted"
+    end
+  end
+
+  def test_file_created
+    assert File.exist?(@storage_path), "File #@storage_path should exist"
+
+    content = File.read(@storage_path)
+    assert content.length > 100, "Serialized content should have more than 100 chars"
+  end
+
+  def test_purge_state
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs", :purge_state => true).instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::FileStorage),"@storage should be an instance of FileStorage"
+      test.assert @word_list.length == 0, "Word count should be purged"
+      test.assert @category_list.length == 0, "Category count should be purged"
+    end
+  end
+end

data/test/test_007_redis_storage.rb

@@ -0,0 +1,82 @@
+# -*- encoding : utf-8 -*-
+require './helper'
+require 'redis'
+
+
+class Test007RedisStorage < TestBase
+  before do
+    @key = "test_classifier"
+    @redis_options = { host: 'localhost', port: 6379 }
+    @redis = Redis.new(@redis_options)
+
+    @storage = StuffClassifier::RedisStorage.new(@key, @redis_options)
+    StuffClassifier::Base.storage = @storage
+
+    StuffClassifier::Bayes.open("Cats or Dogs") do |cls|
+      cls.train(:dog, "Dogs are awesome, cats too. I love my dog.")
+      cls.train(:dog, "My dog's name is Willy. He likes to play with my wife's cat all day long. I love dogs")
+      cls.train(:dog, "So which one should you choose? A dog, definitely.")
+      cls.train(:dog, "A dog will eat anything, including birds or whatever meat")
+      cls.train(:dog, "My dog's favorite place to take a leak is the tree in front of our house")
+
+      cls.train(:cat, "My cat's favorite place to purr is on my keyboard")
+      cls.train(:cat, "The favorite food for cats is bird meat, although mice are good, but birds are a delicacy")
+      cls.train(:cat, "Cats are difficult animals, unlike dogs, really annoying, I hate them all")
+      cls.train(:cat, "Cats are more preferred by software developers. I never could stand cats. I have a dog")
+    end
+
+    # redefining storage instance, forcing it to read from file again
+    StuffClassifier::Base.storage = StuffClassifier::RedisStorage.new(@key, @redis_options)
+  end
+
+  def teardown
+    @redis.del(@key)
+  end
+
+  def test_result
+    set_classifier StuffClassifier::Bayes.new("Cats or Dogs")
+
+    should_be :cat, "This test is about cats."
+    should_be :cat, "I hate ..."
+    should_be :cat, "The most annoying animal on earth."
+    should_be :cat, "The preferred company of software developers."
+    should_be :cat, "My precious, my favorite!"
+    should_be :cat, "Kill that bird!"
+
+    should_be :dog, "This test is about dogs."
+    should_be :dog, "Cats or Dogs?"
+    should_be :dog, "What pet will I love more?"
+    should_be :dog, "Willy, where the heck are you?"
+    should_be :dog, "I like big buts and I cannot lie."
+    should_be :dog, "Why is the front door of our house open?"
+    should_be :dog, "Who ate my meat?"
+
+  end
+
+  def test_for_persistance
+    assert !@storage.equal?(StuffClassifier::Base.storage),"Storage instance should not be the same"
+
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs").instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::RedisStorage),"@storage should be an instance of RedisStorage"
+      test.assert @word_list.length > 0, "Word count should be persisted"
+      test.assert @category_list.length > 0, "Category count should be persisted"
+    end
+  end
+
+  def test_key_created
+    assert @redis.exists(@key), "Redis key #{@key} should exist"
+
+    content = @redis.get(@key)
+    assert content.length > 100, "Serialized content should have more than 100 chars"
+  end
+
+  def test_purge_state
+    test = self
+    StuffClassifier::Bayes.new("Cats or Dogs", :purge_state => true).instance_eval do
+      test.assert @storage.instance_of?(StuffClassifier::RedisStorage),"@storage should be an instance of RedisStorage"
+      test.assert @word_list.length == 0, "Word count should be purged"
+      test.assert @category_list.length == 0, "Category count should be purged"
+    end
+  end
+end

metadata

@@ -0,0 +1,253 @@
+--- !ruby/object:Gem::Specification
+name: stuff-classifier-chinese
+version: !ruby/object:Gem::Version
+  version: '0.51'
+  prerelease: 
+platform: ruby
+authors:
+- Tim Lang
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ruby-stemmer
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rmmseg-cpp-huacnlee
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debugger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: forked from https://github.com/alexandru/stuff-classifier, 2 methods
+  are provided for now - (1) naive bayes implementation + (2) tf-idf weights
+email:
+- langyong135@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/stuff-classifier.rb
+- lib/stuff-classifier/base.rb
+- lib/stuff-classifier/bayes.rb
+- lib/stuff-classifier/storage.rb
+- lib/stuff-classifier/tf-idf.rb
+- lib/stuff-classifier/tokenizer.rb
+- lib/stuff-classifier/tokenizer/tokenizer_properties.rb
+- lib/stuff-classifier/version.rb
+- stuff-classifier.gemspec
+- test/helper.rb
+- test/test_001_tokenizer.rb
+- test/test_002_base.rb
+- test/test_003_naive_bayes.rb
+- test/test_004_tf_idf.rb
+- test/test_005_in_memory_storage.rb
+- test/test_006_file_storage.rb
+- test/test_007_redis_storage.rb
+homepage: https://github.com/TimLang/stuff-classifier/
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Simple text classifier(s) implemetation Chinese version
+test_files:
+- test/helper.rb
+- test/test_001_tokenizer.rb
+- test/test_002_base.rb
+- test/test_003_naive_bayes.rb
+- test/test_004_tf_idf.rb
+- test/test_005_in_memory_storage.rb
+- test/test_006_file_storage.rb
+- test/test_007_redis_storage.rb
+has_rdoc: