rumale 0.8.0

data/lib/rumale/evaluation_measure/accuracy.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  # This module consists of the classes for model evaluation.
+  module EvaluationMeasure
+    # Accuracy is a class that calculates the accuracy of classifier from the predicted labels.
+    #
+    # @example
+    #   evaluator = Rumale::EvaluationMeasure::Accuracy.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class Accuracy
+      include Base::Evaluator
+
+      # Calculate mean accuracy.
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::Int32] (shape: [n_samples]) Predicted labels.
+      # @return [Float] Mean accuracy
+      def score(y_true, y_pred)
+        check_label_array(y_true)
+        check_label_array(y_pred)
+
+        (y_true.to_a.map.with_index { |label, n| label == y_pred[n] ? 1 : 0 }).inject(:+) / y_true.size.to_f
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/f_score.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+require 'rumale/evaluation_measure/precision_recall'
+
+module Rumale
+  # This module consists of the classes for model evaluation.
+  module EvaluationMeasure
+    # FScore is a class that calculates the F1-score of the predicted labels.
+    #
+    # @example
+    #   evaluator = Rumale::EvaluationMeasure::FScore.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class FScore
+      include Base::Evaluator
+      include EvaluationMeasure::PrecisionRecall
+
+      # Return the average type for calculation of F1-score.
+      # @return [String] ('binary', 'micro', 'macro')
+      attr_reader :average
+
+      # Create a new evaluation measure calculater for F1-score.
+      #
+      # @param average [String] The average type ('binary', 'micro', 'macro')
+      def initialize(average: 'binary')
+        check_params_string(average: average)
+        @average = average
+      end
+
+      # Calculate average F1-score
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::Int32] (shape: [n_samples]) Predicted labels.
+      # @return [Float] Average F1-score
+      def score(y_true, y_pred)
+        check_label_array(y_true)
+        check_label_array(y_pred)
+
+        case @average
+        when 'binary'
+          f_score_each_class(y_true, y_pred).last
+        when 'micro'
+          micro_average_f_score(y_true, y_pred)
+        when 'macro'
+          macro_average_f_score(y_true, y_pred)
+        end
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/log_loss.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+require 'rumale/preprocessing/one_hot_encoder'
+
+module Rumale
+  module EvaluationMeasure
+    # LogLoss is a class that calculates the logarithmic loss of predicted class probability.
+    #
+    # @example
+    #   evaluator = Rumale::EvaluationMeasure::LogLoss.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class LogLoss
+      include Base::Evaluator
+
+      # Calculate mean logarithmic loss.
+      # If both y_true and y_pred are array (both shapes are [n_samples]), this method calculates
+      # mean logarithmic loss for binary classification.
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted class probability.
+      # @param eps [Float] A small value close to zero to avoid outputting infinity in logarithmic calcuation.
+      # @return [Float] mean logarithmic loss
+      def score(y_true, y_pred, eps = 1e-15)
+        check_params_type(Numo::Int32, y_true: y_true)
+        check_params_type(Numo::DFloat, y_pred: y_pred)
+
+        n_samples, n_classes = y_pred.shape
+        clipped_p = y_pred.clip(eps, 1 - eps)
+
+        log_loss = if n_classes.nil?
+                     negative_label = y_true.to_a.uniq.min
+                     bin_y_true = Numo::DFloat.cast(y_true.ne(negative_label))
+                     -(bin_y_true * Numo::NMath.log(clipped_p) + (1 - bin_y_true) * Numo::NMath.log(1 - clipped_p))
+                   else
+                     encoder = Rumale::Preprocessing::OneHotEncoder.new
+                     encoded_y_true = encoder.fit_transform(y_true)
+                     clipped_p /= clipped_p.sum(1).expand_dims(1)
+                     -(encoded_y_true * Numo::NMath.log(clipped_p)).sum(1)
+                   end
+        log_loss.sum / n_samples
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/mean_absolute_error.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  module EvaluationMeasure
+    # MeanAbsoluteError is a class that calculates the mean absolute error.
+    #
+    # @example
+    #   evaluator = Rumale::EvaluationMeasure::MeanAbsoluteError.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class MeanAbsoluteError
+      include Base::Evaluator
+
+      # Calculate mean absolute error.
+      #
+      # @param y_true [Numo::DFloat] (shape: [n_samples, n_outputs]) Ground truth target values.
+      # @param y_pred [Numo::DFloat] (shape: [n_samples, n_outputs]) Estimated target values.
+      # @return [Float] Mean absolute error
+      def score(y_true, y_pred)
+        check_tvalue_array(y_true)
+        check_tvalue_array(y_pred)
+        raise ArgumentError, 'Expect to have the same size both y_true and y_pred.' unless y_true.shape == y_pred.shape
+
+        (y_true - y_pred).abs.mean
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/mean_squared_error.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  module EvaluationMeasure
+    # MeanSquaredError is a class that calculates the mean squared error.
+    #
+    # @example
+    #   evaluator = Rumale::EvaluationMeasure::MeanSquaredError.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class MeanSquaredError
+      include Base::Evaluator
+
+      # Calculate mean squared error.
+      #
+      # @param y_true [Numo::DFloat] (shape: [n_samples, n_outputs]) Ground truth target values.
+      # @param y_pred [Numo::DFloat] (shape: [n_samples, n_outputs]) Estimated target values.
+      # @return [Float] Mean squared error
+      def score(y_true, y_pred)
+        check_tvalue_array(y_true)
+        check_tvalue_array(y_pred)
+        raise ArgumentError, 'Expect to have the same size both y_true and y_pred.' unless y_true.shape == y_pred.shape
+
+        ((y_true - y_pred)**2).mean
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/normalized_mutual_information.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  module EvaluationMeasure
+    # NormalizedMutualInformation is a class that calculates the normalized mutual information of cluatering results.
+    #
+    # @example
+    #   evaluator = Rumale::EvaluationMeasure::NormalizedMutualInformation.new
+    #   puts evaluator.score(ground_truth, predicted)
+    #
+    # *Reference*
+    # - C D. Manning, P. Raghavan, and H. Schutze, "Introduction to Information Retrieval," Cambridge University Press., 2008.
+    # - N X. Vinh, J. Epps, and J. Bailey, "Information Theoretic Measures for Clusterings Comparison: Variants, Properties, Normalization and Correction for Chance," J. Machine Learning Research, vol. 11, pp. 2837--1854, 2010.
+    class NormalizedMutualInformation
+      include Base::Evaluator
+
+      # Calculate noramlzied mutual information
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::Int32] (shape: [n_samples]) Predicted cluster labels.
+      # @return [Float] Normalized mutual information
+      def score(y_true, y_pred)
+        check_label_array(y_true)
+        check_label_array(y_pred)
+        # initiazlie some variables.
+        mutual_information = 0.0
+        n_samples = y_pred.size
+        class_ids = y_true.to_a.uniq
+        cluster_ids = y_pred.to_a.uniq
+        # calculate entropy.
+        class_entropy = -1.0 * class_ids.map do |k|
+          ratio = y_true.eq(k).count.fdiv(n_samples)
+          ratio * Math.log(ratio)
+        end.reduce(:+)
+        return 0.0 if class_entropy.zero?
+        cluster_entropy = -1.0 * cluster_ids.map do |k|
+          ratio = y_pred.eq(k).count.fdiv(n_samples)
+          ratio * Math.log(ratio)
+        end.reduce(:+)
+        return 0.0 if cluster_entropy.zero?
+        # calculate mutual information.
+        cluster_ids.map do |k|
+          pr_sample_ids = y_pred.eq(k).where.to_a
+          n_pr_samples = pr_sample_ids.size
+          class_ids.map do |j|
+            tr_sample_ids = y_true.eq(j).where.to_a
+            n_tr_samples = tr_sample_ids.size
+            n_intr_samples = (pr_sample_ids & tr_sample_ids).size
+            if n_intr_samples.positive?
+              mutual_information +=
+                n_intr_samples.fdiv(n_samples) * Math.log((n_samples * n_intr_samples).fdiv(n_pr_samples * n_tr_samples))
+            end
+          end
+        end
+        # return normalized mutual information.
+        mutual_information / Math.sqrt(class_entropy * cluster_entropy)
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/precision.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+require 'rumale/evaluation_measure/precision_recall'
+
+module Rumale
+  # This module consists of the classes for model evaluation.
+  module EvaluationMeasure
+    # Precision is a class that calculates the preicision of the predicted labels.
+    #
+    # @example
+    #   evaluator = Rumale::EvaluationMeasure::Precision.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class Precision
+      include Base::Evaluator
+      include EvaluationMeasure::PrecisionRecall
+
+      # Return the average type for calculation of precision.
+      # @return [String] ('binary', 'micro', 'macro')
+      attr_reader :average
+
+      # Create a new evaluation measure calculater for precision score.
+      #
+      # @param average [String] The average type ('binary', 'micro', 'macro')
+      def initialize(average: 'binary')
+        check_params_string(average: average)
+        @average = average
+      end
+
+      # Calculate average precision.
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::Int32] (shape: [n_samples]) Predicted labels.
+      # @return [Float] Average precision
+      def score(y_true, y_pred)
+        check_label_array(y_true)
+        check_label_array(y_pred)
+
+        case @average
+        when 'binary'
+          precision_each_class(y_true, y_pred).last
+        when 'micro'
+          micro_average_precision(y_true, y_pred)
+        when 'macro'
+          macro_average_precision(y_true, y_pred)
+        end
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/precision_recall.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  # This module consists of the classes for model evaluation.
+  module EvaluationMeasure
+    # @!visibility private
+    module PrecisionRecall
+      module_function
+
+      # @!visibility private
+      def precision_each_class(y_true, y_pred)
+        y_true.sort.to_a.uniq.map do |label|
+          target_positions = y_pred.eq(label)
+          next 0.0 if y_pred[target_positions].empty?
+          n_true_positives = Numo::Int32.cast(y_true[target_positions].eq(y_pred[target_positions])).sum.to_f
+          n_false_positives = Numo::Int32.cast(y_true[target_positions].ne(y_pred[target_positions])).sum.to_f
+          n_true_positives / (n_true_positives + n_false_positives)
+        end
+      end
+
+      # @!visibility private
+      def recall_each_class(y_true, y_pred)
+        y_true.sort.to_a.uniq.map do |label|
+          target_positions = y_true.eq(label)
+          next 0.0 if y_pred[target_positions].empty?
+          n_true_positives = Numo::Int32.cast(y_true[target_positions].eq(y_pred[target_positions])).sum.to_f
+          n_false_negatives = Numo::Int32.cast(y_true[target_positions].ne(y_pred[target_positions])).sum.to_f
+          n_true_positives / (n_true_positives + n_false_negatives)
+        end
+      end
+
+      # @!visibility private
+      def f_score_each_class(y_true, y_pred)
+        precision_each_class(y_true, y_pred).zip(recall_each_class(y_true, y_pred)).map do |p, r|
+          next 0.0 if p.zero? && r.zero?
+          (2.0 * p * r) / (p + r)
+        end
+      end
+
+      # @!visibility private
+      def micro_average_precision(y_true, y_pred)
+        evaluated_values = y_true.sort.to_a.uniq.map do |label|
+          target_positions = y_pred.eq(label)
+          next [0.0, 0.0] if y_pred[target_positions].empty?
+          n_true_positives = Numo::Int32.cast(y_true[target_positions].eq(y_pred[target_positions])).sum.to_f
+          n_false_positives = Numo::Int32.cast(y_true[target_positions].ne(y_pred[target_positions])).sum.to_f
+          [n_true_positives, n_true_positives + n_false_positives]
+        end
+        res = evaluated_values.transpose.map { |v| v.inject(:+) }
+        res.first / res.last
+      end
+
+      # @!visibility private
+      def micro_average_recall(y_true, y_pred)
+        evaluated_values = y_true.sort.to_a.uniq.map do |label|
+          target_positions = y_true.eq(label)
+          next 0.0 if y_pred[target_positions].empty?
+          n_true_positives = Numo::Int32.cast(y_true[target_positions].eq(y_pred[target_positions])).sum.to_f
+          n_false_negatives = Numo::Int32.cast(y_true[target_positions].ne(y_pred[target_positions])).sum.to_f
+          [n_true_positives, n_true_positives + n_false_negatives]
+        end
+        res = evaluated_values.transpose.map { |v| v.inject(:+) }
+        res.first / res.last
+      end
+
+      # @!visibility private
+      def micro_average_f_score(y_true, y_pred)
+        p = micro_average_precision(y_true, y_pred)
+        r = micro_average_recall(y_true, y_pred)
+        (2.0 * p * r) / (p + r)
+      end
+
+      # @!visibility private
+      def macro_average_precision(y_true, y_pred)
+        precision_each_class(y_true, y_pred).inject(:+) / y_true.to_a.uniq.size
+      end
+
+      # @!visibility private
+      def macro_average_recall(y_true, y_pred)
+        recall_each_class(y_true, y_pred).inject(:+) / y_true.to_a.uniq.size
+      end
+
+      # @!visibility private
+      def macro_average_f_score(y_true, y_pred)
+        f_score_each_class(y_true, y_pred).inject(:+) / y_true.to_a.uniq.size
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/purity.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  module EvaluationMeasure
+    # Purity is a class that calculates the purity of cluatering results.
+    #
+    # @example
+    #   evaluator = Rumale::EvaluationMeasure::Purity.new
+    #   puts evaluator.score(ground_truth, predicted)
+    #
+    # *Reference*
+    # - C D. Manning, P. Raghavan, and H. Schutze, "Introduction to Information Retrieval," Cambridge University Press., 2008.
+    class Purity
+      include Base::Evaluator
+
+      # Calculate purity
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::Int32] (shape: [n_samples]) Predicted cluster labels.
+      # @return [Float] Purity
+      def score(y_true, y_pred)
+        check_label_array(y_true)
+        check_label_array(y_pred)
+        # initiazlie some variables.
+        purity = 0
+        n_samples = y_pred.size
+        class_ids = y_true.to_a.uniq
+        cluster_ids = y_pred.to_a.uniq
+        # calculate purity.
+        cluster_ids.each do |k|
+          pr_sample_ids = y_pred.eq(k).where.to_a
+          purity += class_ids.map { |j| (pr_sample_ids & y_true.eq(j).where.to_a).size }.max
+        end
+        purity.fdiv(n_samples)
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/r2_score.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+require 'rumale/evaluation_measure/precision_recall'
+
+module Rumale
+  module EvaluationMeasure
+    # R2Score is a class that calculates the coefficient of determination for the predicted values.
+    #
+    # @example
+    #   evaluator = Rumale::EvaluationMeasure::R2Score.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class R2Score
+      include Base::Evaluator
+
+      # Create a new evaluation measure calculater for coefficient of determination.
+      def initialize; end
+
+      # Calculate the coefficient of determination.
+      #
+      # @param y_true [Numo::DFloat] (shape: [n_samples, n_outputs]) Ground truth target values.
+      # @param y_pred [Numo::DFloat] (shape: [n_samples, n_outputs]) Estimated taget values.
+      # @return [Float] Coefficient of determination
+      def score(y_true, y_pred)
+        check_tvalue_array(y_true)
+        check_tvalue_array(y_pred)
+        raise ArgumentError, 'Expect to have the same size both y_true and y_pred.' unless y_true.shape == y_pred.shape
+
+        n_samples, n_outputs = y_true.shape
+        numerator = ((y_true - y_pred)**2).sum(0)
+        yt_mean = y_true.sum(0) / n_samples
+        denominator = ((y_true - yt_mean)**2).sum(0)
+        if n_outputs.nil?
+          denominator.zero? ? 0.0 : 1.0 - numerator / denominator
+        else
+          scores = 1 - numerator / denominator
+          scores[denominator.eq(0)] = 0.0
+          scores.sum / scores.size
+        end
+      end
+    end
+  end
+end