twss 0.0.1

data/lib/twss.rb

@@ -0,0 +1,31 @@
+require 'rubygems'
+require 'forwardable'
+
+require File.join(File.dirname(__FILE__), 'twss/engine')
+
+module TWSS
+
+  Version = '0.0.1'
+
+  class << self
+
+    extend Forwardable
+
+    def_delegators :engine, :threshold, :threshold=
+
+    def classify(str)
+      engine.classify(str)
+    end
+
+    def engine
+      @engine ||= TWSS::Engine.new
+    end
+
+  end
+
+end
+
+# Shortcut for TWSS.classify(str)
+def TWSS(str)
+  TWSS.classify(str)
+end

data/lib/twss/engine.rb

@@ -0,0 +1,63 @@
+require 'classifier'
+
+module TWSS
+
+  class Engine
+
+    extend Forwardable
+
+    def_delegators :@classifier, :train, :untrain, :classifications
+
+    DATA_FILE = File.join(File.dirname(__FILE__), '../../data/classifier')
+
+    TRUE = '1'
+    FALSE = '0'
+
+    attr_accessor :threshold
+    
+    def initialize(options = {})
+      @data_file = options[:data_file] || DATA_FILE
+      @threshold ||= options[:threshold] || 5.0
+      @classifier = load_classifier_from_file!(@data_file) || new_classifier
+    end
+
+    def classify(str)
+      if basic_conditions_met?(str)
+        c = @classifier.classifications(str)
+        c[TRUE] - c[FALSE] > threshold
+      else
+        false
+      end
+    end
+
+    # Dumps the current classifier state to specified path
+    def dump_classifier_to_file(f = @data_file)
+      o = File.open(f, 'w')
+      o.write(Marshal.dump(@classifier))
+      o.close
+    end
+
+    # Clears out the current classifier instance and nukes the data file
+    def clear_state!
+      File.delete(@data_file) if File.exists?(@data_file)
+      @classifier = new_classifier
+    end
+
+    private
+
+    def new_classifier
+      Classifier::Bayes.new(TRUE, FALSE)
+    end
+
+    def basic_conditions_met?(str)
+      str.split(' ').length > 3 # more than 3 words
+    end
+
+    # Given a path to a classifier file, load the instance into memory
+    def load_classifier_from_file!(f)
+      Marshal.load(File.read(f)) rescue nil if File.exists?(f)
+    end
+
+  end
+
+end

data/lib/twss/trainer.rb

@@ -0,0 +1,68 @@
+#require 'twitter'
+
+module TWSS
+
+  class Trainer
+
+    attr_reader :engine
+
+    def initialize(engine, options = {})
+      @engine = engine
+      engine.clear_state!
+      @training_set_size = options[:training_set_size] || 100
+    end
+
+    def train
+      path = File.join(File.dirname(__FILE__), '../../data/')
+
+      puts "Clearing state..."
+      engine.clear_state!
+
+      puts "Training NON-TWSS strings..."
+      File.read(File.join(path, 'non_twss.txt')).each_line do |l|
+        engine.train(TWSS::Engine::FALSE, strip_tweet(l))
+      end
+
+      puts "Training TWSS strings..."
+      File.read(File.join(path, 'twss.txt')).each_line do |l|
+        engine.train(TWSS::Engine::TRUE, strip_tweet(l))
+      end
+
+      puts "Writing to file..."
+      engine.dump_classifier_to_file
+
+      puts "Done."
+      puts
+
+      run_examples
+    end
+
+    # A little cleanup of the text before we train on it.
+    def strip_tweet(text)
+      t = text.gsub(/[\@\#]\w+\b/i, '') # strip mentions and hashtags
+      t.gsub!(/(RT|OH)\W/i, '') # strip RT's and OH's
+      t.gsub!(/twss/i, '') # strip out twss itself
+      t.gsub!(/http:\/\/[A-Za-z0-9\.\/]+/, '') # URLs
+      t.gsub!(/[\W\d]/, ' ') # now all non word chars and numbers
+      t.strip!
+      t
+    end
+
+    def run_examples
+      ["how big is that thing going to get?",
+       "umm... that's the not the right hole",
+       "did you resolve the ticket?",
+       "did you fix the bug?",
+       "you're going to need to go faster",
+       "I'm almost there, keep going",
+       "Ok, send me a pull request",
+       "The president issued a decree",
+       "I don't get it, this isn't working correctly",
+       "finished specialties in the warehouse"].each do |s|
+         puts '"' + s + '" => ' + TWSS(s).to_s
+       end
+    end
+
+  end
+
+end

data/lib/twss/tweet_collector.rb

@@ -0,0 +1,29 @@
+require 'twitter'
+
+module TWSS
+
+  class TweetCollector
+
+    attr_reader :search, :filename, :limit
+
+    def initialize(search, filename, limit = 1500)
+      @search, @filename, @limit = search, filename, limit
+    end
+
+    def run
+      o = File.open(filename, 'a')
+      page, per_page = 1, 100
+      begin
+        Twitter::Search.new(search).per_page(per_page).page(page).each do |tweet|
+          puts tweet.text
+          o.puts tweet.text
+        end
+        page += 1
+        sleep 2
+      end while page * per_page < limit
+      o.close
+    end
+
+  end
+
+end

data/script/collect_non_twss.rb

@@ -0,0 +1,4 @@
+require File.join(File.dirname(__FILE__), '../lib/twss')
+require File.join(File.dirname(__FILE__), '../lib/twss/tweet_collector')
+
+TWSS::TweetCollector.new(':)', File.join(File.dirname(__FILE__), '../data/non_twss.txt')).run

data/script/collect_twss.rb

@@ -0,0 +1,4 @@
+require File.join(File.dirname(__FILE__), '../lib/twss')
+require File.join(File.dirname(__FILE__), '../lib/twss/tweet_collector')
+
+TWSS::TweetCollector.new('#twss', File.join(File.dirname(__FILE__), '../data/twss.txt')).run

data/script/train.rb

@@ -0,0 +1,7 @@
+
+require File.join(File.dirname(__FILE__), '../lib/twss')
+require File.join(File.dirname(__FILE__), '../lib/twss/trainer')
+
+engine = TWSS::Engine.new
+trainer = TWSS::Trainer.new(engine)
+trainer.train

data/twss.gemspec

@@ -0,0 +1,61 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{twss}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Ben VandenBos"]
+  s.date = %q{2010-08-07}
+  s.description = %q{Pre-trained "That's What She Said" bayes classifier.
+      Given a string, returns true if it's a TWSS joke.  Pre-trained from
+      Twitter #twss.  Let the twss mashups begin!}
+  s.email = %q{bvandenbos@gmail.com}
+  s.extra_rdoc_files = [
+    "README.markdown"
+  ]
+  s.files = [
+    ".gitignore",
+     "README.markdown",
+     "Rakefile",
+     "data/classifier",
+     "data/non_twss.txt",
+     "data/twss.txt",
+     "lib/twss.rb",
+     "lib/twss/engine.rb",
+     "lib/twss/trainer.rb",
+     "lib/twss/tweet_collector.rb",
+     "script/collect_non_twss.rb",
+     "script/collect_twss.rb",
+     "script/train.rb",
+     "twss.gemspec"
+  ]
+  s.homepage = %q{http://github.com/bvandenbos/twss}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Pre-trained That's What She Said classifier}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<classifier>, [">= 1.3.1"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+      s.add_development_dependency(%q<twitter>, [">= 0"])
+    else
+      s.add_dependency(%q<classifier>, [">= 1.3.1"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+      s.add_dependency(%q<twitter>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<classifier>, [">= 1.3.1"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+    s.add_dependency(%q<twitter>, [">= 0"])
+  end
+end
+

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification 
+name: twss
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Ben VandenBos
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-08-07 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: classifier
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 25
+        segments: 
+        - 1
+        - 3
+        - 1
+        version: 1.3.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: twitter
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+description: |-
+  Pre-trained "That's What She Said" bayes classifier.
+        Given a string, returns true if it's a TWSS joke.  Pre-trained from
+        Twitter #twss.  Let the twss mashups begin!
+email: bvandenbos@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+files: 
+- .gitignore
+- README.markdown
+- Rakefile
+- data/classifier
+- data/non_twss.txt
+- data/twss.txt
+- lib/twss.rb
+- lib/twss/engine.rb
+- lib/twss/trainer.rb
+- lib/twss/tweet_collector.rb
+- script/collect_non_twss.rb
+- script/collect_twss.rb
+- script/train.rb
+- twss.gemspec
+has_rdoc: true
+homepage: http://github.com/bvandenbos/twss
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Pre-trained That's What She Said classifier
+test_files: []
+