decisiontree 0.3.0 → 0.4.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ..gemspec
+gemspec

data/README.md

@@ -0,0 +1,48 @@
+# Decision Tree
+
+A ruby library which implements ID3 (information gain) algorithm for decision tree learning. Currently, continuous and discrete datasets can be learned.
+
+- Discrete model assumes unique labels & can be graphed and converted into a png for visual analysis
+- Continuous looks at all possible values for a variable and iteratively chooses the best threshold between all possible assignments. This results in a binary tree which is partitioned by the threshold at every step. (e.g. temperate > 20C)
+
+## Features
+- ID3 algorithms for continuous and discrete cases, with support for incosistent datasets.
+- Graphviz component to visualize the learned tree (http://rockit.sourceforge.net/subprojects/graphr/)
+- Support for multiple, and symbolic outputs and graphing of continuos trees.
+- Returns default value when no branches are suitable for input
+
+## Implementation
+
+- Ruleset is a class that trains an ID3Tree with 2/3 of the training data, converts it into a set of rules and prunes the rules with the remaining 1/3 of the training data (in a C4.5 way).
+- Bagging is a bagging-based trainer (quite obvious), which trains 10 Ruleset trainers and when predicting chooses the best output based on voting.
+
+Blog post with explanation & examples: http://www.igvita.com/2007/04/16/decision-tree-learning-in-ruby/
+
+## Example
+
+```ruby
+require 'decisiontree'
+
+attributes = ['Temperature']
+training = [
+  [36.6, 'healthy'],
+  [37, 'sick'],
+  [38, 'sick'],
+  [36.7, 'healthy'],
+  [40, 'sick'],
+  [50, 'really sick'],
+]
+
+# Instantiate the tree, and train it based on the data (set default to '1')
+dec_tree = DecisionTree::ID3Tree.new(attributes, training, 'sick', :continuous)
+dec_tree.train
+
+decision = dec_tree.predict([37, 'sick'])
+puts "Predicted: #{decision} ... True decision: #{test.last}";
+
+# => Predicted: sick ... True decision: sick
+```
+
+## License
+
+The MIT License - Copyright (c) 2006 Ilya Grigorik

data/Rakefile

@@ -1,123 +1,7 @@
-require 'rubygems'
-require 'rake'
-require 'rake/clean'
-require 'rake/testtask'
-require 'rake/packagetask'
-require 'rake/gempackagetask'
-require 'rake/rdoctask'
-require 'rake/contrib/rubyforgepublisher'
-require 'fileutils'
-require 'hoe'
-
-include FileUtils
-require File.join(File.dirname(__FILE__), 'lib', 'decisiontree', 'version')
-
-AUTHOR = 'Ilya Grigorik'  # can also be an array of Authors
-EMAIL = "ilya <at> fortehost.com"
-DESCRIPTION = "ID3-based implementation of the M.L. Decision Tree algorithm"
-GEM_NAME = 'decisiontree' # what ppl will type to install your gem
-
-@config_file = "~/.rubyforge/user-config.yml"
-@config = nil
-def rubyforge_username
-  unless @config
-    begin
-      @config = YAML.load(File.read(File.expand_path(@config_file)))
-    rescue
-      puts <<-EOS
-ERROR: No rubyforge config file found: #{@config_file}"
-Run 'rubyforge setup' to prepare your env for access to Rubyforge
- - See http://newgem.rubyforge.org/rubyforge.html for more details
-      EOS
-      exit
-    end
-  end
-  @rubyforge_username ||= @config["username"]
-end
-
-RUBYFORGE_PROJECT = 'decisiontree' # The unix name for your project
-HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
-DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
-
-NAME = "decisiontree"
-REV = nil 
-# UNCOMMENT IF REQUIRED: 
-# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
-VERS = DecisionTree::VERSION::STRING + (REV ? ".#{REV}" : "")
-CLEAN.include ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']
-RDOC_OPTS = ['--quiet', '--title', 'decisiontree documentation',
-    "--opname", "index.html",
-    "--line-numbers", 
-    "--main", "README",
-    "--inline-source"]
-
-class Hoe
-  def extra_deps 
-    @extra_deps.reject { |x| Array(x).first == 'hoe' } 
-  end 
-end
-
-# Generate all the Rake tasks
-# Run 'rake -T' to see list of generated tasks (from gem root directory)
-hoe = Hoe.new(GEM_NAME, VERS) do |p|
-  p.author = AUTHOR 
-  p.description = DESCRIPTION
-  p.email = EMAIL
-  p.summary = DESCRIPTION
-  p.url = HOMEPATH
-  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
-  p.test_globs = ["test/**/test_*.rb"]
-  p.clean_globs |= CLEAN  #An array of file patterns to delete on clean.
-  
-  # == Optional
-  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
-  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
-  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
-end
-
-CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\n\n")
-PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
-hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
-
-desc 'Generate website files'
-task :website_generate do
-  Dir['website/**/*.txt'].each do |txt|
-    sh %{ ruby scripts/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
-  end
-end
-
-desc 'Upload website files to rubyforge'
-task :website_upload do
-  host = "#{rubyforge_username}@rubyforge.org"
-  remote_dir = "/var/www/gforge-projects/#{PATH}/"
-  local_dir = 'website'
-  sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
-end
-
-desc 'Generate and upload website files'
-task :website => [:website_generate, :website_upload, :publish_docs]
-
-desc 'Release the website and new gem version'
-task :deploy => [:check_version, :website, :release] do
-  puts "Remember to create SVN tag:"
-  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
-    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
-  puts "Suggested comment:"
-  puts "Tagging release #{CHANGES}"
-end
-
-desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
-task :local_deploy => [:website_generate, :install_gem]
-
-task :check_version do
-  unless ENV['VERSION']
-    puts 'Must pass a VERSION=x.y.z release version'
-    exit
-  end
-  unless ENV['VERSION'] == VERS
-    puts "Please update your version.rb to match the release version, currently #{VERS}"
-    exit
-  end
-end
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
 
+task :default => :spec

data/decisiontree.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "decisiontree"
+  s.version     = "0.4.0"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ilya Grigorik"]
+  s.email       = ["ilya@igvita.com"]
+  s.homepage    = "https://github.com/igrigorik/decisiontree"
+  s.summary     = %q{ID3-based implementation of the M.L. Decision Tree algorithm}
+  s.description = s.summary
+
+  s.rubyforge_project = "decisiontree"
+
+  s.add_development_dependency "graphr"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rspec-given"
+  s.add_development_dependency "pry"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/decisiontree.rb

@@ -1 +1 @@
-Dir[File.join(File.dirname(__FILE__), 'decisiontree/**/*.rb')].sort.each { |lib| require lib }
+require File.dirname(__FILE__) + '/decisiontree/id3_tree.rb'

data/lib/decisiontree/id3_tree.rb

@@ -1,13 +1,9 @@
-#The MIT License
-
-###Copyright (c) 2007 Ilya Grigorik <ilya AT fortehost DOT com>
-###Modifed at 2007 by José Ignacio Fernández <joseignacio.fernandez AT gmail DOT com>
-
-begin; 
-  require 'graph/graphviz_dot'
-rescue LoadError
-  STDERR.puts "graph/graphviz_dot not installed, graphing functionality not included."
-end
+# The MIT License
+#
+### Copyright (c) 2007 Ilya Grigorik <ilya AT igvita DOT com>
+### Modifed at 2007 by José Ignacio Fernández <joseignacio.fernandez AT gmail DOT com>
+
+require 'graphr'
 
 class Object
   def save_to_file(filename)
@@ -19,9 +15,9 @@ class Object
   end
 end
 
-class Array 
-  def classification; collect { |v| v.last }; end 
-    
+class Array
+  def classification; collect { |v| v.last }; end
+
   # calculate information entropy
   def entropy
     return 0 if empty?
@@ -55,28 +51,34 @@ module DecisionTree
 
       @tree = id3_train(data2, attributes, default)
     end
-    
-    def id3_train(data, attributes, default, used={})
-      # Choose a fitness algorithm   
-      case @type
-        when :discrete; fitness = proc{|a,b,c| id3_discrete(a,b,c)} 
+
+    def type(attribute)
+      @type.is_a?(Hash) ? @type[attribute.to_sym] : @type
+    end
+
+    def fitness_for(attribute)
+      case type(attribute)
+        when :discrete; fitness = proc{|a,b,c| id3_discrete(a,b,c)}
         when :continuous; fitness = proc{|a,b,c| id3_continuous(a,b,c)}
       end
-      
-      return default if data.empty?                                               
+    end
+
+    def id3_train(data, attributes, default, used={})
+      return default if data.empty?
 
       # return classification if all examples have the same classification
       return data.first.last if data.classification.uniq.size == 1
 
       # Choose best attribute (1. enumerate all attributes / 2. Pick best attribute)
-      performance = attributes.collect { |attribute| fitness.call(data, attributes, attribute) }
+      performance = attributes.collect { |attribute| fitness_for(attribute).call(data, attributes, attribute) }
       max = performance.max { |a,b| a[0] <=> b[0] }
       best = Node.new(attributes[performance.index(max)], max[1], max[0])
       best.threshold = nil if @type == :discrete
-      @used.has_key?(best.attribute) ? @used[best.attribute] += [best.threshold] : @used[best.attribute] = [best.threshold]  
+      @used.has_key?(best.attribute) ? @used[best.attribute] += [best.threshold] : @used[best.attribute] = [best.threshold]
       tree, l = {best => {}}, ['>=', '<']
-      
-      case @type
+
+      fitness = fitness_for(best.attribute)
+      case type(best.attribute)
         when :continuous
           data.partition { |d| d[attributes.index(best.attribute)] >= best.threshold }.each_with_index  { |examples, i|
             tree[best][String.new(l[i])] = id3_train(examples, attributes, (data.classification.mode rescue 0), &fitness)
@@ -86,7 +88,7 @@ module DecisionTree
           partitions = values.collect { |val| data.select { |d| d[attributes.index(best.attribute)] == val } }
           partitions.each_with_index  { |examples, i|
             tree[best][values[i]] = id3_train(examples, attributes-[values[i]], (data.classification.mode rescue 0), &fitness)
-          }  
+          }
         end
 
       tree
@@ -100,32 +102,32 @@ module DecisionTree
       thresholds.pop
       #thresholds -= used[attribute] if used.has_key? attribute
 
-      gain = thresholds.collect { |threshold|   
+      gain = thresholds.collect { |threshold|
         sp = data.partition { |d| d[attributes.index(attribute)] >= threshold }
         pos = (sp[0].size).to_f / data.size
         neg = (sp[1].size).to_f / data.size
-       
+
         [data.classification.entropy - pos*sp[0].classification.entropy - neg*sp[1].classification.entropy, threshold]
       }.max { |a,b| a[0] <=> b[0] }
 
       return [-1, -1] if gain.size == 0
       gain
     end
-    
+
     # ID3 for discrete label cases
     def id3_discrete(data, attributes, attribute)
       values = data.collect { |d| d[attributes.index(attribute)] }.uniq.sort
       partitions = values.collect { |val| data.select { |d| d[attributes.index(attribute)] == val } }
       remainder = partitions.collect {|p| (p.size.to_f / data.size) * p.classification.entropy}.inject(0) {|i,s| s+=i }
-      
+
       [data.classification.entropy - remainder, attributes.index(attribute)]
     end
 
     def predict(test)
-       return (@type == :discrete ? descend_discrete(@tree, test) : descend_continuous(@tree, test)), 1
+      descend(@tree, test)
     end
 
-    def graph(filename) 
+    def graph(filename)
       dgp = DotGraphPrinter.new(build_tree)
       dgp.write_to_file("#{filename}.png", "png")
     end
@@ -155,22 +157,20 @@ module DecisionTree
     end
 
   private
-    def descend_continuous(tree, test)
+    def descend(tree, test)
       attr = tree.to_a.first
       return @default if !attr
-      return attr[1]['>='] if !attr[1]['>='].is_a?(Hash) and test[@attributes.index(attr.first.attribute)] >= attr.first.threshold
-      return attr[1]['<'] if !attr[1]['<'].is_a?(Hash) and test[@attributes.index(attr.first.attribute)] < attr.first.threshold
-      return descend_continuous(attr[1]['>='],test) if test[@attributes.index(attr.first.attribute)] >= attr.first.threshold
-      return descend_continuous(attr[1]['<'],test) if test[@attributes.index(attr.first.attribute)] < attr.first.threshold 
-    end 
-    
-    def descend_discrete(tree, test)
-      attr = tree.to_a.first
-      return @default if !attr
-      return attr[1][test[@attributes.index(attr[0].attribute)]] if !attr[1][test[@attributes.index(attr[0].attribute)]].is_a?(Hash)
-      return descend_discrete(attr[1][test[@attributes.index(attr[0].attribute)]],test)
+      if type(attr.first.attribute) == :continuous
+        return attr[1]['>='] if !attr[1]['>='].is_a?(Hash) and test[@attributes.index(attr.first.attribute)] >= attr.first.threshold
+        return attr[1]['<'] if !attr[1]['<'].is_a?(Hash) and test[@attributes.index(attr.first.attribute)] < attr.first.threshold
+        return descend(attr[1]['>='],test) if test[@attributes.index(attr.first.attribute)] >= attr.first.threshold
+        return descend(attr[1]['<'],test) if test[@attributes.index(attr.first.attribute)] < attr.first.threshold
+      else
+        return attr[1][test[@attributes.index(attr[0].attribute)]] if !attr[1][test[@attributes.index(attr[0].attribute)]].is_a?(Hash)
+        return descend(attr[1][test[@attributes.index(attr[0].attribute)]],test)
+      end
     end
-    
+
     def build_tree(tree = @tree)
       return [] unless tree.is_a?(Hash)
       return [["Always", @default]] if tree.empty?
@@ -186,7 +186,7 @@ module DecisionTree
           child = attr[1][key]
           child_text = "#{child}\n(#{child.to_s.clone.object_id})"
         end
-        label_text = "#{key} #{@type == :continuous ? attr[0].threshold : ""}"
+        label_text = "#{key} #{type(attr[0].attribute) == :continuous ? attr[0].threshold : ""}"
 
         [parent_text, child_text, label_text]
       end
@@ -286,7 +286,7 @@ module DecisionTree
 
     def predict(test)
       @rules.each do |r|
-        prediction = r.predict(test) 
+        prediction = r.predict(test)
         return prediction, r.accuracy unless prediction.nil?
       end
       return @default, 0.0

data/spec/id3_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+describe describe DecisionTree::ID3Tree do
+
+  describe "simple discrete case" do
+    Given(:labels) { ["sun", "rain"]}
+    Given(:data) do
+      [
+        [1,0,1],
+        [0,1,0]
+      ]
+    end
+    Given(:tree) { DecisionTree::ID3Tree.new(labels, data, 1, :discrete) }
+    When { tree.train }
+    Then { tree.predict([1,0]).should == 1 }
+    Then { tree.predict([0,1]).should == 0 }
+  end
+
+  describe "discrete attributes" do
+    Given(:labels) { ["hungry", "color"] }
+    Given(:data) do
+      [
+        ["yes", "red", "angry"],
+        ["no", "blue", "not angry"],
+        ["yes", "blue", "not angry"],
+        ["no", "red", "not angry"]
+      ]
+    end
+    Given(:tree) { DecisionTree::ID3Tree.new(labels, data, "not angry", :discrete) }
+    When { tree.train }
+    Then { tree.predict(["yes", "red"]).should == "angry" }
+    Then { tree.predict(["no", "red"]).should == "not angry" }
+  end
+
+  describe "discrete attributes" do
+    Given(:labels) { ["hunger", "happiness"] }
+    Given(:data) do
+      [
+        [8, 7, "angry"],
+        [6, 7, "angry"],
+        [7, 9, "angry"],
+        [7, 1, "not angry"],
+        [2, 9, "not angry"],
+        [3, 2, "not angry"],
+        [2, 3, "not angry"],
+        [1, 4, "not angry"]
+      ]
+    end
+    Given(:tree) { DecisionTree::ID3Tree.new(labels, data, "not angry", :continuous) }
+    When { tree.train }
+    Then { tree.graph("continuous") }
+    Then { tree.predict([7, 7]).should == "angry" }
+    Then { tree.predict([2, 3]).should == "not angry" }
+  end
+
+  describe "a mixture" do
+    Given(:labels) { ["hunger", "color"] }
+    Given(:data) do
+      [
+        [8, "red", "angry"],
+        [6, "red", "angry"],
+        [7, "red", "angry"],
+        [7, "blue", "not angry"],
+        [2, "red", "not angry"],
+        [3, "blue", "not angry"],
+        [2, "blue", "not angry"],
+        [1, "red", "not angry"]
+      ]
+    end
+    Given(:tree) { DecisionTree::ID3Tree.new(labels, data, "not angry", color: :discrete, hunger: :continuous) }
+    When { tree.train }
+    Then { tree.graph("continuous") }
+    Then { tree.predict([7, "red"]).should == "angry" }
+    Then { tree.predict([2, "blue"]).should == "not angry" }
+  end
+
+end