feature_set 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.2@feature_set --create

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in feature_set.gemspec
+gemspec

data/README.markdown

@@ -0,0 +1,8 @@
+## FeatureSet
+
+
+Helpful tasks:
+
+- rake build
+- rake install
+- rake release

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/feature_set.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "feature_set/version"
+
+Gem::Specification.new do |s|
+  s.name        = "feature_set"
+  s.version     = FeatureSet::VERSION
+  s.authors     = ["Andrew Cantino"]
+  s.email       = ["andrew@iterationlabs.com"]
+  s.homepage    = "https://github.com/iterationlabs/feature_set"
+  s.summary     = %q{Generate feature vectors from textual data}
+  s.description = %q{FeatureSet is a Ruby library for generating feature vectors from textual data.  It can output in ARFF format for experimentation with Weka.}
+
+  s.rubyforge_project = "feature_set"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "rspec"
+  s.add_runtime_dependency "wwood-rarff"
+  s.add_runtime_dependency "activesupport"
+  s.add_runtime_dependency "i18n"
+end

data/lib/feature_set/builder.rb

@@ -0,0 +1,70 @@
+require 'active_support'
+require 'active_support/inflector'
+
+require "feature_set/feature_builder/word_vector"
+require "feature_set/feature_builder/cuss"
+
+require "feature_set/datum"
+
+module FeatureSet
+  class Builder
+    BUILTIN_FEATURE_BUILDERS = %w[FeatureSet::FeatureBuilder::Cuss 
+                                  FeatureSet::FeatureBuilder::WordVector].map(&:constantize)
+
+    attr_accessor :options, :feature_builders, :data, :features
+
+    def initialize(options = {})
+      @options = options
+      @feature_builders = []
+      @features = []
+      @data = []
+    end
+    
+    def add_data(data)
+      clear_features
+      (@data << data).flatten!
+    end
+    
+    def clear_data
+      @data = []
+      clear_features
+    end
+    
+    def clear_features
+      @features = []
+    end
+    
+    def generate_features(opts = {})
+      wrapped_data_set = self.class.wrap_dataset(data)
+
+      feature_builders.each {|fb| fb.before_generate_features(wrapped_data_set) }
+      
+      @features = wrapped_data_set.map do |row|
+        output_row = {}
+        
+        row.each do |key, datum|
+          (output_row[:class] = datum) and next if key == :class
+          output_row[key] = datum.value if opts[:include_original]
+
+          feature_builders.each do |builder|
+            builder.generate_features(datum, key, row).each do |feature, value|
+              output_row["#{key}_#{feature}".to_sym] = value
+            end
+          end
+        end
+        
+        output_row
+      end
+    end
+    
+    def add_feature_builders(*builders)
+      builders = BUILTIN_FEATURE_BUILDERS.map(&:new) if [:all, "all"].include?(builders.first)
+      (@feature_builders << builders).flatten!
+    end
+    alias_method :add_feature_builder, :add_feature_builders
+    
+    def self.wrap_dataset(dataset)
+      dataset.map { |row| row.inject({}) { |m, (k, v)| m[k] = (k == :class ? v : Datum.new(v)) ; m } }
+    end
+  end
+end

data/lib/feature_set/data/cusswords.txt

@@ -0,0 +1,351 @@
+anus
+arse
+arsehole
+ass
+ass-hat
+asshat
+ass-jabber
+assjabber
+ass-pirate
+asspirate
+assbag
+assbandit
+assbanger
+assbite
+assclown
+asscock
+asscracker
+asses
+assface
+assfuck
+assfucker
+assgoblin
+asshead
+asshole
+asshopper
+assjacker
+asslick
+asslicker
+assmonkey
+assmunch
+assmuncher
+assnigger
+assshit
+assshole
+asssucker
+asswad
+asswipe
+bampot
+bastard
+beaner
+bitch
+bitchass
+bitches
+bitchtits
+bitchy
+blow job
+blowjob
+bollocks
+bollox
+boner
+brotherfucker
+bullshit
+bumblefuck
+butt plug
+buttplug
+butt-pirate
+buttpirate
+buttfucka
+buttfucker
+camel toe
+cameltoe
+carpetmuncher
+chinc
+chink
+choad
+chode
+clit
+clitface
+clitfuck
+clusterfuck
+cock
+cockass
+cockbite
+cockburger
+cockface
+cockfucker
+cockhead
+cockjockey
+cockknoker
+cockmaster
+cockmongler
+cockmongruel
+cockmonkey
+cockmuncher
+cocknose
+cocknugget
+cockshit
+cocksmith
+cocksmoke
+cocksmoker
+cocksniffer
+cocksucker
+cockwaffle
+coochie
+coochy
+coon
+cooter
+cracker
+cum
+cumbubble
+cumdumpster
+cumguzzler
+cumjockey
+cumslut
+cumtart
+cunnie
+cunnilingus
+cunt
+cuntass
+cuntface
+cunthole
+cuntlicker
+cuntrag
+cuntslut
+dago
+damn
+deggo
+dick
+dickbag
+dickbeaters
+dickface
+dickfuck
+dickfucker
+dickhead
+dickhole
+dickjuice
+dickmilk
+dickmonger
+dicks
+dickslap
+dicksucker
+dicksucking
+dickwad
+dickweasel
+dickweed
+dickwod
+dike
+dildo
+dipshit
+doochbag
+dookie
+douche
+douche-fag
+douchefag
+douchebag
+douchewaffle
+dumass
+dumb ass
+dumbass
+dumbfuck
+dumbshit
+dumshit
+dyke
+fag
+fagbag
+fagfucker
+faggit
+faggot
+faggotcock
+fagtard
+fatass
+fellatio
+feltch
+flamer
+fuck
+fuckass
+fuckbag
+fuckboy
+fuckbrain
+fuckbutt
+fucked
+fucker
+fuckersucker
+fuckface
+fuckhead
+fuckhole
+fuckin
+fucking
+fucknut
+fucknutt
+fuckoff
+fucks
+fuckstick
+fucktard
+fucktart
+fuckup
+fuckwad
+fuckwit
+fuckwitt
+fudgepacker
+gay
+gayass
+gaybob
+gaydo
+gayfuck
+gayfuckist
+gaylord
+gaytard
+gaywad
+goddamn
+goddamnit
+gooch
+gook
+gringo
+guido
+handjob
+hard on
+hardon
+heeb
+hell
+ho
+hoe
+homo
+homodumbshit
+honkey
+humping
+jackass
+jap
+jerk off
+jerkoff
+jigaboo
+jizz
+jungle bunny
+junglebunny
+kike
+kooch
+kootch
+kraut
+kunt
+kyke
+lameass
+lesbian
+lesbo
+lezzie
+mcfagget
+mick
+minge
+mothafucka
+mothafuckin\'
+mothafuckin
+motherfucker
+motherfucking
+muff
+muffdiver
+munging
+negro
+nigaboo
+nigga
+nigger
+niggers
+niglet
+nut sack
+nutsack
+paki
+panooch
+pecker
+peckerhead
+penis
+penisbanger
+penisfucker
+penispuffer
+piss
+pissed
+pissed off
+pissedoff
+pissflaps
+polesmoker
+pollock
+poon
+poonani
+poonany
+poontang
+porch monkey
+porchmonkey
+prick
+punanny
+punta
+pussies
+pussy
+pussylicking
+puto
+queef
+queer
+queerbait
+queerhole
+renob
+rimjob
+ruski
+sand nigger
+sandnigger
+schlong
+scrote
+shit
+shitass
+shitbag
+shitbagger
+shitbrains
+shitbreath
+shitcanned
+shitcunt
+shitdick
+shitface
+shitfaced
+shithead
+shithole
+shithouse
+shitspitter
+shitstain
+shitter
+shittiest
+shitting
+shitty
+shiz
+shiznit
+skank
+skeet
+skullfuck
+slut
+slutbag
+smeg
+snatch
+spic
+spick
+splooge
+spook
+suckass
+tard
+testicle
+thundercunt
+tit
+titfuck
+tits
+tittyfuck
+twat
+twatlips
+twats
+twatwaffle
+unclefucker
+va-j-j
+vajj
+vag
+vagina
+vajayjay
+vjayjay
+wank
+wankjob
+wetback
+whore
+whorebag
+whoreface
+wop

data/lib/feature_set/datum.rb

@@ -0,0 +1,24 @@
+module FeatureSet
+  class Datum
+    TOKEN_REGEX = /[\s\/]+/
+    NON_ASCII_REGEX = /[^a-zA-Z0-9_-]/
+    
+    attr_accessor :value
+    
+    def initialize(v)
+      self.value = v
+    end
+    
+    def tokens
+      @tokens ||= begin
+        value.strip.downcase.gsub(NON_ASCII_REGEX, ' ').split(TOKEN_REGEX)
+      end
+    end
+    
+    def token_counts
+      @token_counts ||= begin
+        tokens.inject({}) { |m, w| m[w] ||= 0; m[w] += 1; m }
+      end
+    end
+  end
+end

data/lib/feature_set/feature_builder/base.rb

@@ -0,0 +1,18 @@
+module FeatureSet
+  module FeatureBuilder
+    class Base
+      attr_accessor :options
+      
+      def initialize(options = {})
+        @options = options
+      end
+      
+      def generate_features(datum, key, row)
+        raise "Please implement 'generate_features' in your subclass of FeatureBuilder::Base."
+      end
+      
+      def before_generate_features(dataset)
+      end
+    end
+  end
+end

data/lib/feature_set/feature_builder/cuss.rb

@@ -0,0 +1,14 @@
+require "feature_set/feature_builder/base"
+
+module FeatureSet
+  module FeatureBuilder
+    class Cuss < Base
+      CUSS_WORDS = File.read(File.expand_path(File.join(File.dirname(__FILE__), '..', 'data', 'cusswords.txt'))).split("\n").map {|i| i.strip.downcase }
+      
+      def generate_features(datum, key, row)
+        return {} unless datum.value.is_a?(String)
+        { :cuss_count => (datum.tokens & CUSS_WORDS).length }
+      end
+    end
+  end
+end

data/lib/feature_set/feature_builder/word_vector.rb

@@ -0,0 +1,45 @@
+require "feature_set/feature_builder/base"
+
+module FeatureSet
+  module FeatureBuilder
+    class WordVector < Base
+      attr_accessor :idfs
+
+      def initialize(options = {})
+        super
+      end
+
+      def before_generate_features(dataset)
+        @idfs = {}
+        dataset.each do |row|
+          row.each do |key, datum|
+            next if key == :class
+            if datum.value.is_a?(String)
+              idfs[key] ||= {}
+              datum.token_counts.keys.each do |token|
+                idfs[key][token] ||= 0
+                idfs[key][token] += 1
+              end
+            end
+          end
+        end
+
+        num_docs = dataset.length
+        idfs.each do |feature, freqs|
+          freqs.each do |key, value|
+            idfs[feature][key] = Math.log(num_docs / value.to_f)
+          end
+        end
+        
+        def generate_features(datum, key, row)
+          return {} unless datum.value.is_a?(String)
+          num_words = datum.tokens.length.to_f
+          idfs[key].inject({}) do |memo, (word, idf)|
+            memo[word] = ((datum.token_counts[word] || 0) / num_words) * idf
+            memo
+          end
+        end
+      end
+    end
+  end
+end

data/lib/feature_set/version.rb

@@ -0,0 +1,3 @@
+module FeatureSet
+  VERSION = "0.0.1"
+end

data/lib/feature_set.rb

@@ -0,0 +1,4 @@
+require "rarff"
+
+require "feature_set/version"
+require "feature_set/builder"

data/spec/feature_set/builder_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+describe FeatureSet::Builder do
+  describe "adding feature builders" do
+    it "can add all known feature builders" do
+      builder = FeatureSet::Builder.new
+      builder.add_feature_builders :all
+      builder.feature_builders.map {|i| i.class}.should include(FeatureSet::FeatureBuilder::WordVector)
+      builder.feature_builders.length.should == Dir[File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib", "feature_set", "feature_builder", "*.rb"))].length - 1
+    end
+    
+    it "can add individual feature builders" do
+      builder = FeatureSet::Builder.new
+      builder.add_feature_builder FeatureSet::FeatureBuilder::WordVector.new
+      builder.feature_builders.length.should == 1
+    end
+
+    it "can add arrays of feature builders" do
+      builder = FeatureSet::Builder.new
+      builder.add_feature_builders [FeatureSet::FeatureBuilder::WordVector.new, FeatureSet::FeatureBuilder::Cuss.new]
+      builder.feature_builders.length.should == 2
+    end
+  end
+  
+  describe "adding data" do
+    it "should accept mappings between one or more strings and their classifications" do
+      builder = FeatureSet::Builder.new
+      builder.add_data [ { :status => "I am happy!", :class => :happy },
+                         { :status => "I am sad." , :class => :sad } ]
+      builder.data.should == [ { :status => "I am happy!", :class => :happy },
+                               { :status => "I am sad." , :class => :sad } ]
+      builder.add_data :status => "Something", :another_feature => "Something else", :class => :awesome
+      builder.data.should == [ { :status => "I am happy!", :class => :happy },
+                               { :status => "I am sad." , :class => :sad },
+                               { :status => "Something", :another_feature => "Something else", :class => :awesome } ]
+      builder.clear_data
+      builder.data.should == []
+      builder.data = [ { :status => "I am happy!", :class => :happy },
+                       { :status => "I am sad." , :class => :sad } ]
+      builder.data.should == [ { :status => "I am happy!", :class => :happy },
+                               { :status => "I am sad." , :class => :sad } ]
+    end
+  end
+  
+  describe "generating features" do
+    before do
+      @builder = FeatureSet::Builder.new
+      @builder.add_feature_builder FeatureSet::FeatureBuilder::Cuss.new
+      @builder.add_data :status => "this is some text", :class => :awesome
+      @builder.add_data :status => "this is some shitty text", :class => :less_awesome
+    end
+    
+    it "should output a row of features for every line of data" do
+      @builder.generate_features
+      @builder.features[0].should == { :status_cuss_count => 0, :class => :awesome }
+      @builder.features[1].should == { :status_cuss_count => 1, :class => :less_awesome }
+    end
+    
+    it "should make it easy to keep the original data" do
+      @builder.generate_features(:include_original => true)
+      @builder.features[0].should == { :status => "this is some text", :status_cuss_count => 0, :class => :awesome }
+      @builder.features[1].should == { :status => "this is some shitty text", :status_cuss_count => 1, :class => :less_awesome }
+    end
+
+    it "should generate features for every string" do
+      @builder.add_data :status => "text", :foo => "more shitty text", :class => :awesome
+      @builder.generate_features
+      @builder.features[1].should == { :status_cuss_count => 1, :class => :less_awesome }
+      @builder.features[2].should == { :status_cuss_count => 0, :foo_cuss_count => 1, :class => :awesome }
+    end
+  end
+end

data/spec/feature_set/datum_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe FeatureSet::Datum do
+  describe "tokenize" do
+    it "should return an array of tokens" do
+      FeatureSet::Datum.new("hello world sup?").tokens.should =~ ["hello", "world", "sup"]
+    end
+
+    it "should memoize" do
+      datum = FeatureSet::Datum.new("hello world sup?")
+      datum.tokens.should =~ ["hello", "world", "sup"]
+      datum.value = "hello"
+      datum.tokens.should =~ ["hello", "world", "sup"]
+    end
+  end
+
+  describe "#token_counts" do
+    it "should provide counts for each token" do
+      datum = FeatureSet::Datum.new("hello world sup?  hello!")
+      datum.token_counts.should == { "hello" => 2, "world" => 1, "sup" => 1}
+    end
+
+    it "should memoize" do
+      datum = FeatureSet::Datum.new("hello world sup?  hello!")
+      datum.token_counts.should == { "hello" => 2, "world" => 1, "sup" => 1}
+      datum.value = "hello"
+      datum.instance_variable_set(:@tokens, ["hello"])
+      datum.token_counts.should == { "hello" => 2, "world" => 1, "sup" => 1}
+    end
+  end
+end

data/spec/feature_set/feature/cuss_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe FeatureSet::FeatureBuilder::Cuss do
+  before do
+    @builder = FeatureSet::FeatureBuilder::Cuss.new
+  end
+
+  it "should output :cuss_count as the number of distinct cuss words found" do
+    @builder.generate_features(FeatureSet::Datum.new("this fucking shit"), nil, nil).should == { :cuss_count => 2 }
+    @builder.generate_features(FeatureSet::Datum.new("this fucking fucking fucking shit"), nil, nil).should == { :cuss_count => 2 }
+  end
+
+  it "should ignore non-string features" do
+    @builder.generate_features(FeatureSet::Datum.new(2), nil, nil).should == {}
+  end
+end

data/spec/feature_set/feature/word_vector_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe FeatureSet::FeatureBuilder::WordVector do
+  it "should output a named feature for every word in the dataset, after performing tfidf" do
+    builder = FeatureSet::FeatureBuilder::WordVector.new
+    dataset = [
+                { :m1 => "hello world.  hello!", :m2 => "how goes?", :class => :yes }, 
+                { :m1 => "foo world", :m2 => "how?", :class => :no }
+              ]
+    wrapped_dataset = FeatureSet::Builder.wrap_dataset(dataset)
+    builder.before_generate_features(wrapped_dataset)
+
+    builder.idfs.should == {
+                             :m1 => { "hello" => Math.log(2/1.0), "world" => Math.log(2/2.0), "foo" => Math.log(2/1.0) },
+                             :m2 => { "how" => Math.log(2/2.0), "goes" => Math.log(2/1.0) }
+                           }
+
+    builder.generate_features(wrapped_dataset.first[:m1], :m1, wrapped_dataset.first).should == { "hello" => (2/3.0) * Math.log(2/1.0), "world" => (1/3.0) * Math.log(2/2.0), "foo" => 0 }
+    builder.generate_features(wrapped_dataset.first[:m2], :m2, wrapped_dataset.first).should == { "how" => (1/2.0) * Math.log(2/2.0), "goes" => (1/2.0) * Math.log(2/1.0) }
+
+    builder.generate_features(wrapped_dataset.last[:m1], :m1, wrapped_dataset.last).should == { "hello" => 0, "world" => (1/2.0) * Math.log(2/2.0), "foo" => (1/2.0) * Math.log(2/1.0) }
+    builder.generate_features(wrapped_dataset.last[:m2], :m2, wrapped_dataset.last).should == { "how" => (1/1.0) * Math.log(2/2.0), "goes" => 0 }
+  end
+
+  it "should ignore non-string features" do
+    builder = FeatureSet::FeatureBuilder::WordVector.new
+    builder.before_generate_features([{ :something => FeatureSet::Datum.new(2), :class => false }, { :something => FeatureSet::Datum.new(1), :class => true }])
+    builder.generate_features(FeatureSet::Datum.new(2), :something, { :something => FeatureSet::Datum.new(2), :class => false }).should == {}
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,4 @@
+--colour
+--format s -c
+--loadby mtime
+--reverse

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'feature_set'
+
+RSpec.configure do |c|
+end

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: feature_set
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Andrew Cantino
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-17 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70284888584540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70284888584540
+- !ruby/object:Gem::Dependency
+  name: wwood-rarff
+  requirement: &70284888584120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70284888584120
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &70284888583700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70284888583700
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: &70284888583280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70284888583280
+description: FeatureSet is a Ruby library for generating feature vectors from textual
+  data.  It can output in ARFF format for experimentation with Weka.
+email:
+- andrew@iterationlabs.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- README.markdown
+- Rakefile
+- feature_set.gemspec
+- lib/feature_set.rb
+- lib/feature_set/builder.rb
+- lib/feature_set/data/cusswords.txt
+- lib/feature_set/datum.rb
+- lib/feature_set/feature_builder/base.rb
+- lib/feature_set/feature_builder/cuss.rb
+- lib/feature_set/feature_builder/word_vector.rb
+- lib/feature_set/version.rb
+- spec/feature_set/builder_spec.rb
+- spec/feature_set/datum_spec.rb
+- spec/feature_set/feature/cuss_spec.rb
+- spec/feature_set/feature/word_vector_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+homepage: https://github.com/iterationlabs/feature_set
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: feature_set
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Generate feature vectors from textual data
+test_files:
+- spec/feature_set/builder_spec.rb
+- spec/feature_set/datum_spec.rb
+- spec/feature_set/feature/cuss_spec.rb
+- spec/feature_set/feature/word_vector_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb