ruby-em_algorithm 0.0.2

data/example/tools/xmeans.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'rsruby'
+require 'gsl'
+
+include GSL
+
+r = RSRuby.instance
+c = r.eval_R(<<-RCOMMAND)
+a <- read.csv('#{ARGV[0]}.csv')
+source('./xmeans.R')
+xmeans(a,2,20)
+RCOMMAND
+
+data_array = YAML.load_file("#{ARGV[0]}.txt").map {|v| Vector[v] }
+cluster = Array.new(c["size"].size).map { {"mu_sum" => 0.0, "sigma_sum" => Matrix.alloc(data_array[0].size, data_array[0].size)} }
+c["cluster"].each_with_index do |num, di|
+  cluster[num - 1]["mu_sum"] += data_array[di]
+  cluster[num - 1]["sigma_sum"] += data_array[di].trans * data_array[di]
+end
+c["size"].each_with_index do |size, num|
+  cluster[num]["mu"] = cluster[num]["mu_sum"] / size
+  cluster[num]["sigma"] = cluster[num]["sigma_sum"] / size
+end
+
+c["centers"].each_with_index do |cen, ci|
+  puts "### cluster #{ci}"
+  p cen
+end
+cluster.each_with_index do |clu, ci|
+  puts "### cluster #{ci}"
+  p clu["mu"].to_a
+  p clu["sigma"].to_a
+end

data/lib/em_algorithm/base.rb

@@ -0,0 +1,116 @@
+require 'em_algorithm/models/model'
+require 'em_algorithm/models/gaussian'
+require 'em_algorithm/models/md_gaussian'
+require 'em_algorithm/models/mixture'
+require 'em_algorithm/convergence/check_method'
+require 'em_algorithm/convergence/likelihood'
+require 'em_algorithm/convergence/chi_square'
+
+module EMAlgorithm
+  include Math
+  include GSL
+
+  MAX_ITERATION = 10000
+
+  class Base
+
+    attr_accessor :model, :original_data_array, :data_array, :likelihood, :num_step, :const
+
+    # * Model limitation
+    # currently only Gaussian Mixture model is supported.
+    # if you want to use simple Gaussian, you must use a mixture model
+    # which has only one Gaussian model entry with weight 1.0.
+    #
+    # * Input data format
+    # You can estimate the probability distribution and the target value distribution.
+    # If you want to estimate the target value distribution, you must specify the target
+    # value and its correspondence area size into the input vector x.
+    # x[-2]: target value
+    # x[-1]: correspondent area size
+    def initialize(options)
+      opts = {
+        :model => Mixture.new(:models => [Gaussian.new(0.0, 9.0)], :weights => [1.0]),
+        :data_array => [],
+        :value_distribution_estimation => false,
+        :debug => true
+      }.merge(options)
+      @model = opts[:model]
+      @original_data_array = opts[:data_array]
+      @value_distribution_estimation = opts[:value_distribution_estimation]
+      if @value_distribution_estimation
+        @data_array = value_to_frequency(@original_data_array)
+      else
+        @data_array = @original_data_array
+      end
+      @likelihood = Likelihood.new(@data_array)
+      @debug = opts[:debug]
+      @const = 1.0
+    end
+
+    # calculate @posterior_data_array
+    def estep
+      @model.clear_temp_weight_per_datum!
+      @posterior_data_array = @model.calculate_posterior_data_array(@data_array)
+      @model.update_temp_weights!(@data_array, @posterior_data_array)
+    end
+
+    # calculate posterior model parameters
+    def mstep
+      if @debug
+        $stderr.puts @model.inspect
+      end
+      @model.update_parameters!(@data_array)
+    end
+
+    def run!
+      MAX_ITERATION.times do |i|
+        if @debug
+          $stderr.puts "step#{i}"
+        end
+        # check convergence
+        @likelihood.calculate(@model)
+        if @debug
+          $stderr.puts @likelihood.debug_output
+        end
+        if @likelihood.converged?
+          @num_step = i
+          if @value_distribution_estimation
+            @const = distribution_to_value_ratio
+          end
+          break
+        end
+        if @debug
+          $stderr.puts @model.debug_output
+          $stderr.puts ""
+        end
+        estep
+        mstep
+      end
+      @model
+    end
+
+    def value_to_frequency(data_array)
+      modified_data_array = []
+      data_array.each do |x|
+        x[x.size-2].round.times do
+          x_without_value = x[0..(x.size-3)]
+          x_without_value = x_without_value.first if x_without_value.size == 1
+          modified_data_array << x_without_value
+        end
+      end
+      modified_data_array
+    end
+
+    # the integration of the distribution equal 1.0
+    # so thus, the integration of the target value means the ratio of
+    # probability distribution and the target value distribution
+    def distribution_to_value_ratio
+      integrated_value = 0.0
+      @original_data_array.each do |x|
+        value = x[x.size-2]
+        integrated_value += value * x[x.size-1]
+      end
+      integrated_value
+    end
+  end
+end

data/lib/em_algorithm/base.rb~

@@ -0,0 +1,116 @@
+require 'em_algorithm/models/model'
+require 'em_algorithm/models/gaussian'
+require 'em_algorithm/models/md_gaussian'
+require 'em_algorithm/models/mixture'
+require 'em_algorithm/convergence/check_method'
+require 'em_algorithm/convergence/likelihood'
+require 'em_algorithm/convergence/chi_square'
+
+module EMAlgorithm
+  include Math
+  include GSL
+
+  MAX_ITERATION = 10000
+
+  class Base
+
+    attr_accessor :model, :original_data_array, :data_array, :likelihood, :num_step, :const
+
+    # * Model limitation
+    # currently only Gaussian Mixture model is supported.
+    # if you want to use simple Gaussian, you must use a mixture model
+    # which has only one Gaussian model entry with weight 1.0.
+    #
+    # * Input data format
+    # You can estimate the probability distribution and the target value distribution.
+    # If you want to estimate the target value distribution, you must specify the target
+    # value and its correspondence area size into the input vector x.
+    # x[-2]: target value
+    # x[-1]: correspondent area size
+    def initialize(options)
+      opts = {
+        :model => Mixture.new(:models => [Gaussian.new(0.0, 9.0)], :weights => [1.0]),
+        :data_array => [],
+        :value_distribution_estimation => false,
+        :debug => true
+      }.merge(options)
+      @model = opts[:model]
+      @original_data_array = opts[:data_array]
+      @value_distribution_estimation = opts[:value_distribution_estimation]
+      if @value_distribution_estimation
+        @data_array = value_to_frequency(@original_data_array)
+      else
+        @data_array = @original_data_array
+      end
+      @likelihood = Likelihood.new(@data_array)
+      @debug = opts[:debug]
+      @const = 1.0
+    end
+
+    # calculate @posterior_data_array
+    def estep
+      @model.clear_temp_weight_per_datum!
+      @posterior_data_array = @model.calculate_posterior_data_array(@data_array)
+      @model.update_temp_weights!(@data_array, @posterior_data_array)
+    end
+
+    # calculate posterior model parameters
+    def mstep
+      if @debug
+        $stderr.puts @model.inspect
+      end
+      @model.update_parameters!(@data_array)
+    end
+
+    def run!
+      MAX_ITERATION.times do |i|
+        if @debug
+          $stderr.puts "step#{i}"
+        end
+        # check convergence
+        @likelihood.calculate(@model)
+        if @debug
+          $stderr.puts @likelihood.debug_output
+        end
+        if @likelihood.converged?
+          @num_step = i
+          if @value_distribution_estimation
+            @const = distribution_to_value_ratio
+          end
+          break
+        end
+        if @debug
+          $stderr.puts @model.debug_output
+          $stderr.puts ""
+        end
+        estep
+        mstep
+      end
+      @model
+    end
+
+    def value_to_frequency(data_array)
+      modified_data_array = []
+      data_array.each do |x|
+        x[x.size-2].round.times do
+          x_without_value = x[0..(x.size-3)]
+          x_without_value = x_without_value.first if x_without_value.size == 1
+          modified_data_array << x_without_value
+        end
+      end
+      modified_data_array
+    end
+
+    # the integration of the distribution equal 1.0
+    # so thus, the integration of the target value means the ratio of
+    # probability distribution and the target value distribution
+    def distribution_to_value_ratio
+      integrated_value = 0.0
+      @original_data_array.each do |x|
+        value = x[x.size-2]
+        integrated_value += value * x[x.size-1]
+      end
+      integrated_value
+    end
+  end
+end

data/lib/em_algorithm/convergence/check_method.rb

@@ -0,0 +1,4 @@
+module EMAlgorithm
+  class CheckMethod
+  end
+end

data/lib/em_algorithm/convergence/chi_square.rb

@@ -0,0 +1,40 @@
+module EMAlgorithm
+  class ChiSquare < CheckMethod
+    STAT_THRESHOLD = 0.05
+    CONV_THRESHOLD = 0.01
+
+    attr_accessor :history
+
+    def initialize(data_array)
+      @data_array = data_array
+      @history = []
+    end
+
+    # calculate chi square
+    def calculate(model, const)
+      chi_square = 0.0
+      @data_array.each do |x|
+        value = x[x.size-1]
+        pdf = model.pdf(x[0..(x.size-2)])
+        next if value <= 1.0
+        estimated = const * pdf
+        chi_square += (value - estimated)**2 / estimated
+      end
+      @history << chi_square
+      chi_square
+    end
+
+    def value
+      @history.last
+    end
+
+    def converged?
+      return false if @history.length == 1
+      (@history[-1] < STAT_THRESHOLD) || ((@history[-1] - @history[-2]).abs < CONV_THRESHOLD)
+    end
+
+    def debug_output
+      $stderr.puts "ChiSquare: #{value}"
+    end
+  end
+end

data/lib/em_algorithm/convergence/chi_square.rb~

@@ -0,0 +1,40 @@
+module EMAlgorithm
+  class ChiSquare < CheckMethod
+    STAT_THRESHOLD = 0.05
+    CONV_THRESHOLD = 0.01
+
+    attr_accessor :history
+
+    def initialize(data_array)
+      @data_array = data_array
+      @history = []
+    end
+
+    # calculate chi square
+    def calculate(model, const)
+      chi_square = 0.0
+      @data_array.each do |x|
+        value = x[x.size-1]
+        pdf = model.pdf(x[0..(x.size-2)])
+        next if value <= 1.0
+        estimated = const * pdf
+        chi_square += (value - estimated)**2 / estimated
+      end
+      @history << chi_square
+      chi_square
+    end
+
+    def value
+      @history.last
+    end
+
+    def converged?
+      return false if @history.length == 1
+      (@history[-1] < STAT_THRESHOLD) || ((@history[-1] - @history[-2]).abs < CONV_THRESHOLD)
+    end
+
+    def debug_output
+      $stderr.puts "ChiSquare: #{value}"
+    end
+  end
+end

data/lib/em_algorithm/convergence/likelihood.rb

@@ -0,0 +1,35 @@
+module EMAlgorithm
+  class Likelihood < CheckMethod
+    #THRESHOLD = 0.0001
+    THRESHOLD = 0.01
+
+    attr_accessor :history
+
+    def initialize(data_array)
+      @data_array = data_array
+      @history = []
+    end
+
+    # calculate log likelihood
+    def calculate(model)
+      likelihood = @data_array.inject(0.0) do |likelihood, x|
+        likelihood + log(model.pdf(x))
+      end
+      @history << likelihood
+      likelihood
+    end
+
+    def value
+      @history.last
+    end
+
+    def converged?
+      return false if @history.length == 1
+      (@history[-1] - @history[-2]).abs < THRESHOLD
+    end
+
+    def debug_output
+      "Likelihood: #{value}"
+    end
+  end
+end

data/lib/em_algorithm/convergence/likelihood.rb~

@@ -0,0 +1,35 @@
+module EMAlgorithm
+  class Likelihood < CheckMethod
+    #THRESHOLD = 0.0001
+    THRESHOLD = 0.01
+
+    attr_accessor :history
+
+    def initialize(data_array)
+      @data_array = data_array
+      @history = []
+    end
+
+    # calculate log likelihood
+    def calculate(model)
+      likelihood = @data_array.inject(0.0) do |likelihood, x|
+        likelihood + log(model.pdf(x))
+      end
+      @history << likelihood
+      likelihood
+    end
+
+    def value
+      @history.last
+    end
+
+    def converged?
+      return false if @history.length == 1
+      (@history[-1] - @history[-2]).abs < THRESHOLD
+    end
+
+    def debug_output
+      "Likelihood: #{value}"
+    end
+  end
+end