baobab 0.1.0

data/lib/baobab.rb

@@ -0,0 +1,4 @@
+require 'baobab/dataset'
+require 'baobab/node'
+require 'baobab/shannon'
+require 'baobab/tree'

data/lib/baobab/dataset.rb

@@ -0,0 +1,74 @@
+
+# Represents a dataset or subset thereof.
+# Is an array of hashes where all hashes contain the same keys.
+class Dataset < Array
+
+    # Receives an array of hashes. All hashes must contain the same keys.
+    def initialize data
+        data.each do |row|
+            self << row
+        end
+    end
+
+    def self.from_json filename
+        text = File.read(filename)
+        self.new JSON.parse(text)
+    end
+
+    def attribute_names class_var
+        self.column_names.reject{|name| name == class_var}
+    end
+
+    # Returns an array of the attribute names in the dataset
+    # Careful: it's empty on an empty set.
+    def column_names
+        self[0].keys
+    end
+
+    # Returns an array of the values of an attribute in the dataset.
+    # Careful: it's empty on an empty set.
+    def column_values attribute
+        self.map{|row| row[attribute]}.to_a.uniq
+    end
+
+    # Gets a subset with given conditions. Keys must be of the same type as
+    # in the dataset (be careful with symbols).
+    def subset conditions
+        rows = self.select do |row|
+            conditions.reduce(true) do |memo, (var, val)|
+                memo and row[var] == val
+            end
+        end
+        Dataset.new rows
+    end
+
+    def entropy class_var
+        class_vals = self.column_values(class_var)
+        probabilities = class_vals.map do |class_val|
+            self.probability(class_var, class_val)
+        end
+        Shannon::entropy *probabilities
+    end
+
+    # Evaluates the probability that var be val in this dataset.
+    # Can also be used for subsets.
+    def probability var, val
+        unless self.count.zero?
+            self.count{|r| r[var] == val}.fdiv(self.count)
+        else
+            0
+        end
+    end
+
+    def validate
+        raise 'Dataset is empty' if self.empty?
+        self.reduce(self[0].keys) do |memo, row|
+            if memo == row.keys then
+                memo
+            else
+                raise 'Dataset is inconsistent'
+            end
+        end
+        return nil
+    end
+end

data/lib/baobab/node.rb

@@ -0,0 +1,189 @@
+
+class DecisionTreeNode
+
+    # A list of nodes
+    attr_accessor :children
+
+    # The variable on which this operates, a string
+    attr_reader :variable
+
+    # The value for the variable, a string
+    attr_reader :value
+
+    # The decision tree
+    attr_reader :tree
+
+    attr_reader :entropy
+
+    attr_accessor :next_attribute
+
+    # Accumulates the conditions down the tree
+    attr_reader :conditions
+
+    # The conditional probability that variable=value
+    attr_reader :probability
+
+    def initialize tree, parent, variable, value, entropy, conditions=nil
+        @tree = tree
+        @parent = parent
+        @variable = variable
+        @value = value
+        @entropy = entropy
+        @conditions = conditions ? conditions : {}
+        @children = Array.new
+        @subset = nil
+    end
+
+    def to_s
+        s = @variable ? "#{@variable} => #{@value}" : "ROOT"
+        s += " (#{@entropy.round(3)})"
+    end
+
+    def subset
+        unless @subset.nil?
+           @subset
+        else
+            @subset = @tree.dataset.subset(self.full_conditions)
+        end
+    end
+
+    def clear
+        @subset = nil
+    end
+
+    def pending_attrs
+        @tree.dataset.column_names.reject do |name|
+            @tree.class_var == name or @conditions.include? name
+        end
+    end
+
+    # Returns a subset selected on all the parent node's conditions plus this
+    # node's attribute and its value.
+    def full_conditions
+        if @variable
+            conditions.merge({@variable => @value})
+        else
+            conditions
+        end
+    end
+
+    def build_subtree recursive=true
+        if self.try_finish
+            return
+        end 
+        subset = self.subset
+        subset_count = self.subset.count
+        entropies = self.entropies
+        inf_gain = entropies.each.with_object({}) do |(a, e), o|
+            o[a] = @entropy - e
+        end
+        max_attr, max_gain = inf_gain.max_by{|v, g| g}
+        self.next_attribute = max_attr
+        @children += self.tree.dataset.column_values(max_attr).map do |value|
+            conditions = self.full_conditions
+            DecisionTreeNode.new(
+                tree=@tree, parent=self,
+                attribute=max_attr, value=value,
+                entropy=entropies[max_attr], conditions=conditions
+            )
+        end
+        if recursive
+            @children.each do |c|
+                c.build_subtree
+            end
+        end
+    end
+
+    # Checks whether any of three is true:
+    # - All elements in the subset belong to the same class value: a leaf with
+    # this value is created.
+    # - There are no more attributes to be selected: a leaf with the most common
+    # class value is selected
+    # - There are no more rows in the dataset: a leaf with the most common class
+    # value in the parent node is created.
+    def try_finish
+        var = self.tree.class_var
+        val = (
+            self.try_finish_single_value_class or
+            self.try_finish_empty_subset or
+            self.try_finish_no_more_attributes
+            )
+        if val
+            @next_attribute = @tree.class_var
+            self.children << DecisionTreeNode.new(
+                tree=@tree, parent=self,
+                variable=@tree.class_var, value=val,
+                entropy=0, conditions=self.full_conditions
+                )
+        else
+            false
+        end
+    end
+
+    # If all class values are the same, returns that value; else, nil.
+    def try_finish_single_value_class
+        if self.subset.any?
+            v0 = self.subset[0][@tree.class_var]
+            self.subset.slice(1...-1).reduce(v0) do |memo, row|
+                if memo != row[@tree.class_var]
+                    return nil
+                else
+                    memo
+                end
+            end
+        else
+            nil
+        end
+
+    end
+
+    # If there are not more attributes, returns the most common class value;
+    # else, nil.
+    def try_finish_no_more_attributes
+        if self.pending_attrs.empty? then
+            self.most_common_value
+        else
+            nil
+        end
+    end
+
+    # If the subset is empty, returns the most common value in the parent
+    # node's subset.
+    def try_finish_empty_subset
+        if self.subset.empty?
+            @parent.most_common_value
+        else
+            nil
+        end
+    end
+
+    # Returns the most common class value in the dataset.
+    def most_common_value
+        class_var = @tree.class_var
+        class_values = @tree.dataset.column_values(class_var)
+        count = class_values.each.with_object({}) do |val, o|
+            o[val] = 0
+        end
+        self.subset.each.with_object(count) do |row, o|
+            count[row[class_var]] += 1
+        end
+        count.max_by{|v, c| c}[0]
+    end
+
+    # Returns a hash of {attribute, entropy} given that we divide the dataset
+    # on attribute.
+    def entropies
+        self.pending_attrs.each.with_object({}) do |a, o|
+            values = @tree.dataset.column_values(a)
+            val_probabilities = values.each.with_object({}) do |v, o|
+                o[v] = subset.probability a, v
+            end
+            val_entropies = values.each.with_object({}) do |v, o|
+                o[v] = subset.subset({a => v}).entropy(self.tree.class_var)
+            end
+            o[a] = values.reduce(0) do |memo, v|
+                memo + val_entropies[v] * val_probabilities[v]
+            end
+        end
+    end
+end

data/lib/baobab/shannon.rb

@@ -0,0 +1,13 @@
+
+module Shannon
+
+    def self.entropy *probabilities
+        probabilities.reduce(0) do |memo, p|
+            if p.zero? then 0 else memo + self.entropy_term(p) end
+        end
+    end
+
+    def self.entropy_term probability
+        - probability * Math::log2(probability)
+    end
+end

data/lib/baobab/tree.rb

@@ -0,0 +1,64 @@
+require 'json'
+require 'set'
+
+class DecisionTree
+
+    #  The first decision tree node
+    attr_reader :root
+
+    # The class variable of interest in this decision tree.
+    attr_reader :class_var
+
+    # The underlying dataset
+    attr_reader :dataset
+
+    def initialize dataset, class_var
+        @dataset = dataset
+        @class_var = class_var
+        entropy = dataset.entropy class_var
+        @root = DecisionTreeNode.new(
+            self, parent=nil,
+            attribute=nil, value=nil,
+            entropy
+            )
+        @root.build_subtree
+    end
+
+    # Prints the decision depth-first with the respective entropy values.
+    def to_s
+        s = ""
+        nodes = [[0, @root]]
+        while nodes.any?
+            l, n = nodes.last
+            nodes = nodes.slice(0...-1)
+            n.children.each do |c|
+                nodes << [l + 1, c]
+            end
+            s = s + "#{' ' * (l * 2)}#{n.to_s}\n"
+        end
+        return s
+    end
+
+    # Receives attributes and their values (they must be all defined).
+    # Returns the value of the predicted class value.
+    def query values
+        if values.keys != @dataset.attribute_names(@class_var)
+            raise "Query does not fit all variables"
+        end
+        node = @root
+        while node.variable != @class_var
+            if node.next_attribute
+                if node.children.count > 1
+                    val = values[node.next_attribute]
+                    node = node.children.select do |child|
+                        child.value == val
+                    end[0]
+                else
+                    node = node.children[0]
+                end
+            end
+        end
+        node.value
+    end
+
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: baobab
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Johnny E. Lee Othon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-22 00:00:00.000000000 Z
+dependencies: []
+description: ! "# baobab\n\nAn implementation of the ID3 (Iterative Dichotomiser 3)
+  in Ruby\n\n## How to run the tests\n\n```\nrake tests\n```\n\n## Coming soon\n\nI
+  promise I'll make this an installable gem. One of these days.\n\n## Sources of the
+  datasets\n\nThe weather dataset has been adapted from the `weather.nominal.arff`
+  that comes shipped with [Weka](http://www.cs.waikato.ac.nz/ml/weka/).\n\nThe transportation
+  dataset was taken from the example data in [https://www.youtube.com/watch?v=wL9aogTuZw8](https://www.youtube.com/watch?v=wL9aogTuZw8).\n\nThe
+  breast cancer dataset is adapted from the `breast-cancer.arff` file that comes shipped
+  with Weka. It should be attributed to:\n\nMatjaz Zwitter & Milan Soklic (physicians)\nInstitute
+  of Oncology \nUniversity Medical Center\nLjubljana, Yugoslavia\nDonors: Ming Tan
+  and Jeff Schlimmer (Jeffrey.Schlimmer@a.gp.cs.cmu.edu)\nDate: 11 July 1988\n"
+email: jleeothon@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/baobab.rb
+- lib/baobab/dataset.rb
+- lib/baobab/node.rb
+- lib/baobab/shannon.rb
+- lib/baobab/tree.rb
+homepage: https://github.com/jleeothon/baobab
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: ID3 decision trees for machine learning in Ruby
+test_files: []