simple_classifier 1.0.0 → 1.3.4

data/Rakefile

@@ -0,0 +1,52 @@
+require 'rubygems'
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+PKG_VERSION = "1.3.1"
+
+PKG_FILES = FileList[
+    "lib/**/*", "bin/*", "test/**/*", "[A-Z]*", "Rakefile", "html/**/*"
+]
+
+desc "Default Task"
+task :default => [ :test ]
+
+# Run the unit tests
+desc "Run all unit tests"
+Rake::TestTask.new("test") { |t|
+  t.libs << "lib"
+  t.pattern = 'test/*/*_test.rb'
+  t.verbose = true
+}
+
+# Make a console, useful when working on tests
+desc "Generate a test console"
+task :console do
+   verbose( false ) { sh "irb -I lib/ -r 'classifier'" }
+end
+
+# Genereate the RDoc documentation
+desc "Create documentation"
+Rake::RDocTask.new("doc") { |rdoc|
+  rdoc.title = "simple_classifier - A simple Bayesian classifier"
+  rdoc.rdoc_dir = 'html'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+}
+
+desc "Generate gemspec with jeweler"
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "simple_classifier"
+    gemspec.summary = "A simple bayesian classifier"
+    gemspec.email = "ben.orenstein@gmail.com"
+    gemspec.homepage = "http://github.com/r00k/simple_classifier"
+    gemspec.authors = ["Ben Orenstein", "Lucas Carlson", "David Fayram II"]
+    gemspec.version = '1.3.4'
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+2.0.0

data/bin/bayes.rb

@@ -2,9 +2,9 @@
 
 begin
   require 'rubygems'
-  require 'classifier'
+  require 'simple_classifier'
 rescue
-  require 'classifier'
+  require 'simple_classifier'
 end
 
 require 'madeleine'

data/bin/summarize.rb

@@ -1,10 +1,10 @@
 #!/usr/bin/env ruby
 
 begin
-        require 'rubygems'
-        require 'classifier'
+  require 'rubygems'
+  require 'simple_classifier'
 rescue
-        require 'classifier'
+  require 'simple_classifier'
 end
 
 require 'open-uri'

data/install.rb

@@ -0,0 +1,49 @@
+require 'rbconfig'
+require 'find'
+require 'ftools'
+
+include Config
+
+# this was adapted from rdoc's install.rb by ways of Log4r
+
+$sitedir = CONFIG["sitelibdir"]
+unless $sitedir
+  version = CONFIG["MAJOR"] + "." + CONFIG["MINOR"]
+  $libdir = File.join(CONFIG["libdir"], "ruby", version)
+  $sitedir = $:.find {|x| x =~ /site_ruby/ }
+  if !$sitedir
+    $sitedir = File.join($libdir, "site_ruby")
+  elsif $sitedir !~ Regexp.quote(version)
+    $sitedir = File.join($sitedir, version)
+  end
+end
+
+makedirs = %w{ simple_classifier }
+makedirs = %w{ simple_classifier/extensions }
+makedirs.each {|f| File::makedirs(File.join($sitedir, *f.split(/\//)))}
+
+Dir.chdir("lib")
+begin
+  require 'rubygems'
+  require 'rake'
+rescue LoadError
+  puts
+  puts "Please install Gem and Rake from http://rubyforge.org/projects/rubygems and http://rubyforge.org/projects/rake"
+  puts
+  exit(-1)
+end
+
+files = FileList["**/*"]
+
+# File::safe_unlink *deprecated.collect{|f| File.join($sitedir, f.split(/\//))}
+files.each {|f| 
+  File::install(f, File.join($sitedir, *f.split(/\//)), 0644, true)
+}
+
+begin
+  require 'stemmer'
+rescue LoadError
+  puts
+  puts "Please install Stemmer from http://rubyforge.org/projects/stemmer or via 'gem install stemmer'"
+  puts
+end

data/lib/{classifier.rb → simple_classifier.rb}

@@ -25,5 +25,5 @@
 # License::   LGPL
 
 require 'rubygems'
-require 'classifier/extensions/string'
-require 'classifier/bayes'
+require 'simple_classifier/extensions/string'
+require 'simple_classifier/bayes'

data/lib/{classifier → simple_classifier}/bayes.rb

@@ -2,7 +2,7 @@
 # Copyright:: Copyright (c) 2005 Lucas Carlson
 # License::   LGPL
 
-module Classifier
+module DfhccClassifier
 
   class Bayes
     # The class can be created with one or more categories, each of which will be

data/lib/simple_classifier/extensions/string.rb

@@ -0,0 +1,16 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+begin
+  require 'stemmer'
+rescue LoadError
+  puts "Please install stemmer from http://rubyforge.org/projects/stemmer or 'gem install stemmer'"
+  exit(-1)
+end
+
+require 'simple_classifier/extensions/word_hash'
+
+class Object
+  def prepare_category_name; to_s.gsub("_"," ").capitalize.intern end
+end

data/lib/simple_classifier/extensions/word_hash.rb

@@ -0,0 +1,125 @@
+# Author::    Lucas Carlson  (mailto:lucas@rufy.com)
+# Copyright:: Copyright (c) 2005 Lucas Carlson
+# License::   LGPL
+
+# These are extensions to the String class to provide convenience 
+# methods for the Classifier package.
+class String
+  
+  # Removes common punctuation symbols, returning a new string. 
+  # E.g.,
+  #   "Hello (greeting's), with {braces} < >...?".without_punctuation
+  #   => "Hello  greetings   with  braces         "
+  def without_punctuation
+    tr( ',?.!;:"@#$%^&*()_=+[]{}\|<>/`~', " " ) .tr( "'\-", "")
+  end
+  
+  # Return a Hash of strings => ints. Each word in the string is stemmed,
+  # interned, and indexes to its frequency in the document.  
+  def word_hash
+    word_hash_for_words(gsub(/[^\w\s]/,"").split + gsub(/[\w]/," ").split)
+  end
+
+  # Return a word hash without extra punctuation or short symbols, just stemmed words
+  def clean_word_hash
+    word_hash_for_words gsub(/[^\w\s]/,"").split
+  end
+  
+  private
+  
+  def word_hash_for_words(words)
+    d = Hash.new
+    words.each do |word|
+      word.downcase! if word =~ /[\w]+/
+      key = word.stem.intern
+      if word =~ /[^\w]/ || ! CORPUS_SKIP_WORDS.include?(word) && word.length > 2
+        d[key] ||= 0
+        d[key] += 1
+      end
+    end
+    return d
+  end
+  
+  CORPUS_SKIP_WORDS = [
+    "a",
+    "again",
+    "all",
+    "along",
+    "are",
+    "also",
+    "an",
+    "and",
+    "as",
+    "at",
+    "but",
+    "by",
+    "came",
+    "can",
+    "cant",
+    "couldnt",
+    "did",
+    "didn",
+    "didnt",
+    "do",
+    "doesnt",
+    "dont",
+    "ever",
+    "first",
+    "from",
+    "have",
+    "her",
+    "here",
+    "him",
+    "how",
+    "i",
+    "if",
+    "in",
+    "into",
+    "is",
+    "isnt",
+    "it",
+    "itll",
+    "just",
+    "last",
+    "least",
+    "like",
+    "most",
+    "my",
+    "new",
+    "no",
+    "not",
+    "now",
+    "of",
+    "on",
+    "or",
+    "should",
+    "sinc",
+    "so",
+    "some",
+    "th",
+    "than",
+    "this",
+    "that",
+    "the",
+    "their",
+    "then",
+    "those",
+    "to",
+    "told",
+    "too",
+    "true",
+    "try",
+    "until",
+    "url",
+    "us",
+    "were",
+    "when",
+    "whether",
+    "while",
+    "with",
+    "within",
+    "yes",
+    "you",
+    "youll",
+  ]
+end

data/test/bayes/bayesian_test.rb

@@ -1,7 +1,7 @@
 require File.dirname(__FILE__) + '/../test_helper'
 class BayesianTest < Test::Unit::TestCase
   def setup
-    @classifier = Classifier::Bayes.new 'Interesting', 'Uninteresting'
+    @classifier = DfhccClassifier::Bayes.new 'Interesting', 'Uninteresting'
   end
   
   def test_good_training
@@ -12,7 +12,7 @@ class BayesianTest < Test::Unit::TestCase
     assert_raise(StandardError) { @classifier.train_no_category "words" }
   end
   
-  def test_bad_method
+  def test_method_missing_calls_super
     assert_raise(NoMethodError) { @classifier.forget_everything_you_know "" }
   end
   

data/test/test_helper.rb

@@ -1,7 +1,7 @@
 $:.unshift(File.dirname(__FILE__) + '/../lib')
 
 require 'test/unit'
-require 'classifier'
+require 'simple_classifier'
 require 'ruby-debug'
 
 # Colorize test ouput

metadata

@@ -1,7 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: simple_classifier
 version: !ruby/object:Gem::Version 
-  version: 1.0.0
+  prerelease: false
+  segments: 
+  - 1
+  - 3
+  - 4
+  version: 1.3.4
 platform: ruby
 authors: 
 - Ben Orenstein
@@ -11,20 +16,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-31 00:00:00 -04:00
+date: 2010-02-26 00:00:00 -05:00
 default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: stemmer
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
-    version: 
-description: Simple bayesian filtering, without a lot of fuss.
+dependencies: []
+
+description: 
 email: ben.orenstein@gmail.com
 executables: 
 - bayes.rb
@@ -35,12 +31,22 @@ extra_rdoc_files:
 - LICENSE
 - README
 files: 
-- lib/classifier.rb
-- lib/classifier/bayes.rb
 - LICENSE
 - README
+- Rakefile
+- VERSION
+- bin/bayes.rb
+- bin/summarize.rb
+- install.rb
+- lib/simple_classifier.rb
+- lib/simple_classifier/bayes.rb
+- lib/simple_classifier/extensions/string.rb
+- lib/simple_classifier/extensions/word_hash.rb
+- test/bayes/bayesian_test.rb
+- test/extensions/word_hash_test.rb
+- test/test_helper.rb
 has_rdoc: true
-homepage: 
+homepage: http://github.com/r00k/simple_classifier
 licenses: []
 
 post_install_message: 
@@ -52,21 +58,23 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      version: 1.3.5
-  version: 
+      segments: 
+      - 0
+      version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
-summary: A simple bayesian classifier for rubyists
+summary: A simple bayesian classifier
 test_files: 
 - test/test_helper.rb
 - test/bayes/bayesian_test.rb