rumale-evaluation_measure 0.24.0

data/lib/rumale/evaluation_measure/mutual_information.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  module EvaluationMeasure
+    # MutualInformation is a class that calculates the mutual information.
+    #
+    # @example
+    #   require 'rumale/evaluation_measure/mutual_information'
+    #
+    #   evaluator = Rumale::EvaluationMeasure::MutualInformation.new
+    #   puts evaluator.score(ground_truth, predicted)
+    #
+    # *Reference*
+    # - Vinh, N X., Epps, J., and Bailey, J., "Information Theoretic Measures for Clusterings Comparison: Variants, Properties, Normalization and Correction for Chance," J. Machine Learning Research, vol. 11, pp. 2837--1854, 2010.
+    class MutualInformation
+      include ::Rumale::Base::Evaluator
+
+      # Calculate mutual information
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::Int32] (shape: [n_samples]) Predicted cluster labels.
+      # @return [Float] Mutual information.
+      def score(y_true, y_pred)
+        # initiazlie some variables.
+        mutual_information = 0.0
+        n_samples = y_pred.size
+        class_ids = y_true.to_a.uniq
+        cluster_ids = y_pred.to_a.uniq
+        # calculate mutual information.
+        cluster_ids.map do |k|
+          pr_sample_ids = y_pred.eq(k).where.to_a
+          n_pr_samples = pr_sample_ids.size
+          class_ids.map do |j|
+            tr_sample_ids = y_true.eq(j).where.to_a
+            n_tr_samples = tr_sample_ids.size
+            n_intr_samples = (pr_sample_ids & tr_sample_ids).size
+            if n_intr_samples.positive?
+              mutual_information +=
+                n_intr_samples.fdiv(n_samples) * Math.log((n_samples * n_intr_samples).fdiv(n_pr_samples * n_tr_samples))
+            end
+          end
+        end
+        mutual_information
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/normalized_mutual_information.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+require 'rumale/evaluation_measure/mutual_information'
+
+module Rumale
+  module EvaluationMeasure
+    # NormalizedMutualInformation is a class that calculates the normalized mutual information.
+    #
+    # @example
+    #   require 'rumale/evaluation_measure/normalized_mutual_information'
+    #
+    #   evaluator = Rumale::EvaluationMeasure::NormalizedMutualInformation.new
+    #   puts evaluator.score(ground_truth, predicted)
+    #
+    # *Reference*
+    # - Manning, C D., Raghavan, P., and Schutze, H., "Introduction to Information Retrieval," Cambridge University Press., 2008.
+    # - Vinh, N X., Epps, J., and Bailey, J., "Information Theoretic Measures for Clusterings Comparison: Variants, Properties, Normalization and Correction for Chance," J. Machine Learning Research, vol. 11, pp. 2837--1854, 2010.
+    class NormalizedMutualInformation
+      include ::Rumale::Base::Evaluator
+
+      # Calculate noramlzied mutual information
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::Int32] (shape: [n_samples]) Predicted cluster labels.
+      # @return [Float] Normalized mutual information
+      def score(y_true, y_pred)
+        # calculate entropies.
+        class_entropy = entropy(y_true)
+        return 0.0 if class_entropy.zero?
+
+        cluster_entropy = entropy(y_pred)
+        return 0.0 if cluster_entropy.zero?
+
+        # calculate mutual information.
+        mi = MutualInformation.new
+        mi.score(y_true, y_pred) / Math.sqrt(class_entropy * cluster_entropy)
+      end
+
+      private
+
+      def entropy(y)
+        n_samples = y.size
+        indices = y.to_a.uniq
+        sum_log = indices.sum do |k|
+          ratio = y.eq(k).count.fdiv(n_samples)
+          ratio * Math.log(ratio)
+        end
+        -sum_log
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/precision.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+require 'rumale/evaluation_measure/precision_recall'
+
+module Rumale
+  # This module consists of the classes for model evaluation.
+  module EvaluationMeasure
+    # Precision is a class that calculates the preicision of the predicted labels.
+    #
+    # @example
+    #   require 'rumale/evaluation_measure/precision'
+    #
+    #   evaluator = Rumale::EvaluationMeasure::Precision.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class Precision
+      include ::Rumale::Base::Evaluator
+      include ::Rumale::EvaluationMeasure::PrecisionRecall
+
+      # Return the average type for calculation of precision.
+      # @return [String] ('binary', 'micro', 'macro')
+      attr_reader :average
+
+      # Create a new evaluation measure calculater for precision score.
+      #
+      # @param average [String] The average type ('binary', 'micro', 'macro')
+      def initialize(average: 'binary')
+        @average = average
+      end
+
+      # Calculate average precision.
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::Int32] (shape: [n_samples]) Predicted labels.
+      # @return [Float] Average precision
+      def score(y_true, y_pred)
+        case @average
+        when 'binary'
+          precision_each_class(y_true, y_pred).last
+        when 'micro'
+          micro_average_precision(y_true, y_pred)
+        when 'macro'
+          macro_average_precision(y_true, y_pred)
+        end
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/precision_recall.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'numo/narray'
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  # This module consists of the classes for model evaluation.
+  module EvaluationMeasure
+    # @!visibility private
+    module PrecisionRecall
+      module_function
+
+      # @!visibility private
+      def precision_each_class(y_true, y_pred)
+        y_true.sort.to_a.uniq.map do |label|
+          target_positions = y_pred.eq(label)
+          next 0.0 if y_pred[target_positions].empty?
+
+          n_true_positives = Numo::Int32.cast(y_true[target_positions].eq(y_pred[target_positions])).sum.to_f
+          n_false_positives = Numo::Int32.cast(y_true[target_positions].ne(y_pred[target_positions])).sum.to_f
+          n_true_positives / (n_true_positives + n_false_positives)
+        end
+      end
+
+      # @!visibility private
+      def recall_each_class(y_true, y_pred)
+        y_true.sort.to_a.uniq.map do |label|
+          target_positions = y_true.eq(label)
+          next 0.0 if y_pred[target_positions].empty?
+
+          n_true_positives = Numo::Int32.cast(y_true[target_positions].eq(y_pred[target_positions])).sum.to_f
+          n_false_negatives = Numo::Int32.cast(y_true[target_positions].ne(y_pred[target_positions])).sum.to_f
+          n_true_positives / (n_true_positives + n_false_negatives)
+        end
+      end
+
+      # @!visibility private
+      def f_score_each_class(y_true, y_pred)
+        precision_each_class(y_true, y_pred).zip(recall_each_class(y_true, y_pred)).map do |p, r|
+          next 0.0 if p.zero? && r.zero?
+
+          (2.0 * p * r) / (p + r)
+        end
+      end
+
+      # @!visibility private
+      def micro_average_precision(y_true, y_pred)
+        evaluated_values = y_true.sort.to_a.uniq.map do |label|
+          target_positions = y_pred.eq(label)
+          next [0.0, 0.0] if y_pred[target_positions].empty?
+
+          n_true_positives = Numo::Int32.cast(y_true[target_positions].eq(y_pred[target_positions])).sum.to_f
+          n_false_positives = Numo::Int32.cast(y_true[target_positions].ne(y_pred[target_positions])).sum.to_f
+          [n_true_positives, n_true_positives + n_false_positives]
+        end
+        res = evaluated_values.transpose.map(&:sum)
+        res.first / res.last
+      end
+
+      # @!visibility private
+      def micro_average_recall(y_true, y_pred)
+        evaluated_values = y_true.sort.to_a.uniq.map do |label|
+          target_positions = y_true.eq(label)
+          next 0.0 if y_pred[target_positions].empty?
+
+          n_true_positives = Numo::Int32.cast(y_true[target_positions].eq(y_pred[target_positions])).sum.to_f
+          n_false_negatives = Numo::Int32.cast(y_true[target_positions].ne(y_pred[target_positions])).sum.to_f
+          [n_true_positives, n_true_positives + n_false_negatives]
+        end
+        res = evaluated_values.transpose.map(&:sum)
+        res.first / res.last
+      end
+
+      # @!visibility private
+      def micro_average_f_score(y_true, y_pred)
+        prec = micro_average_precision(y_true, y_pred)
+        recl = micro_average_recall(y_true, y_pred)
+        (2.0 * prec * recl) / (prec + recl)
+      end
+
+      # @!visibility private
+      def macro_average_precision(y_true, y_pred)
+        precision_each_class(y_true, y_pred).sum / y_true.to_a.uniq.size
+      end
+
+      # @!visibility private
+      def macro_average_recall(y_true, y_pred)
+        recall_each_class(y_true, y_pred).sum / y_true.to_a.uniq.size
+      end
+
+      # @!visibility private
+      def macro_average_f_score(y_true, y_pred)
+        f_score_each_class(y_true, y_pred).sum / y_true.to_a.uniq.size
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/purity.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  module EvaluationMeasure
+    # Purity is a class that calculates the purity of cluatering results.
+    #
+    # @example
+    #   require 'rumale/evaluation_measure/purity'
+    #
+    #   evaluator = Rumale::EvaluationMeasure::Purity.new
+    #   puts evaluator.score(ground_truth, predicted)
+    #
+    # *Reference*
+    # - Manning, C D., Raghavan, P., and Schutze, H., "Introduction to Information Retrieval," Cambridge University Press., 2008.
+    class Purity
+      include ::Rumale::Base::Evaluator
+
+      # Calculate purity
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::Int32] (shape: [n_samples]) Predicted cluster labels.
+      # @return [Float] Purity
+      def score(y_true, y_pred)
+        # initiazlie some variables.
+        purity = 0
+        n_samples = y_pred.size
+        class_ids = y_true.to_a.uniq
+        cluster_ids = y_pred.to_a.uniq
+        # calculate purity.
+        cluster_ids.each do |k|
+          pr_sample_ids = y_pred.eq(k).where.to_a
+          purity += class_ids.map { |j| (pr_sample_ids & y_true.eq(j).where.to_a).size }.max
+        end
+        purity.fdiv(n_samples)
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/r2_score.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  module EvaluationMeasure
+    # R2Score is a class that calculates the coefficient of determination for the predicted values.
+    #
+    # @example
+    #   require 'rumale/evaluation_measure/r2_score'
+    #
+    #   evaluator = Rumale::EvaluationMeasure::R2Score.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class R2Score
+      include ::Rumale::Base::Evaluator
+
+      # Create a new evaluation measure calculater for coefficient of determination.
+      def initialize; end
+
+      # Calculate the coefficient of determination.
+      #
+      # @param y_true [Numo::DFloat] (shape: [n_samples, n_outputs]) Ground truth target values.
+      # @param y_pred [Numo::DFloat] (shape: [n_samples, n_outputs]) Estimated taget values.
+      # @return [Float] Coefficient of determination
+      def score(y_true, y_pred)
+        n_samples, n_outputs = y_true.shape
+        numerator = ((y_true - y_pred)**2).sum(axis: 0)
+        yt_mean = y_true.sum(axis: 0) / n_samples
+        denominator = ((y_true - yt_mean)**2).sum(axis: 0)
+        if n_outputs.nil?
+          denominator.zero? ? 0.0 : 1.0 - numerator / denominator
+        else
+          scores = 1 - numerator / denominator
+          scores[denominator.eq(0)] = 0.0
+          scores.sum / scores.size
+        end
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/recall.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+require 'rumale/evaluation_measure/precision_recall'
+
+module Rumale
+  # This module consists of the classes for model evaluation.
+  module EvaluationMeasure
+    # Recall is a class that calculates the recall of the predicted labels.
+    #
+    # @example
+    #   require 'rumale/evaluation_measure/recall'
+    #
+    #   evaluator = Rumale::EvaluationMeasure::Recall.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class Recall
+      include ::Rumale::Base::Evaluator
+      include ::Rumale::EvaluationMeasure::PrecisionRecall
+
+      # Return the average type for calculation of recall.
+      # @return [String] ('binary', 'micro', 'macro')
+      attr_reader :average
+
+      # Create a new evaluation measure calculater for recall score.
+      #
+      # @param average [String] The average type ('binary', 'micro', 'macro')
+      def initialize(average: 'binary')
+        @average = average
+      end
+
+      # Calculate average recall
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::Int32] (shape: [n_samples]) Predicted labels.
+      # @return [Float] Average recall
+      def score(y_true, y_pred)
+        case @average
+        when 'binary'
+          recall_each_class(y_true, y_pred).last
+        when 'micro'
+          micro_average_recall(y_true, y_pred)
+        when 'macro'
+          macro_average_recall(y_true, y_pred)
+        end
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/roc_auc.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+
+module Rumale
+  module EvaluationMeasure
+    # ROCAUC is a class that calculate area under the receiver operation characteristic curve from predicted scores.
+    #
+    # @example
+    #   require 'rumale/preprocessing'
+    #   require 'rumale/linear_model'
+    #   require 'rumale/evaluation_measure/roc_auc'
+    #
+    #   # Encode labels to integer array.
+    #   labels = %w[A B B C A A C C C A]
+    #   label_encoder = Rumale::Preprocessing::LabelEncoder.new
+    #   y = label_encoder.fit_transform(labels)
+    #   # Fit classifier.
+    #   classifier = Rumale::LinearModel::LogisticRegression.new
+    #   classifier.fit(x, y)
+    #   # Predict class probabilities.
+    #   y_score = classifier.predict_proba(x)
+    #   # Encode labels to one-hot vectors.
+    #   one_hot_encoder = Rumale::Preprocessing::OneHotEncoder.new
+    #   y_onehot = one_hot_encoder.fit_transform(y)
+    #   # Calculate ROC AUC.
+    #   evaluator = Rumale::EvaluationMeasure::ROCAUC.new
+    #   puts evaluator.score(y_onehot, y_score)
+    class ROCAUC
+      include ::Rumale::Base::Evaluator
+
+      # Calculate area under the receiver operation characteristic curve (ROC AUC).
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples] or [n_samples, n_classes])
+      #   Ground truth binary labels or one-hot encoded multi-labels.
+      # @param y_score [Numo::DFloat] (shape: [n_samples] or [n_samples, n_classes])
+      #   Predicted class probabilities or confidence scores.
+      # @return [Float] (macro-averaged) ROC AUC.
+      def score(y_true, y_score)
+        n_classes = y_score.shape[1]
+        if n_classes.nil?
+          fpr, tpr, = roc_curve(y_true, y_score)
+          return auc(fpr, tpr)
+        end
+
+        scores = Array.new(n_classes) do |c|
+          fpr, tpr, = roc_curve(y_true[true, c], y_score[true, c])
+          auc(fpr, tpr)
+        end
+
+        scores.sum.fdiv(n_classes)
+      end
+
+      # Calculate receiver operation characteristic curve.
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth binary labels.
+      # @param y_score [Numo::DFloat] (shape: [n_samples]) Predicted class probabilities or confidence scores.
+      # @param pos_label [Integer] Label to be a positive label when binarizing the given labels.
+      #   If nil is given, the method considers the maximum value of the label as a positive label.
+      # @return [Array] fpr (Numo::DFloat): false positive rates. tpr (Numo::DFloat): true positive rates.
+      #   thresholds (Numo::DFloat): thresholds on the decision function used to calculate fpr and tpr.
+      def roc_curve(y_true, y_score, pos_label = nil)
+        labels = y_true.to_a.uniq
+        if pos_label.nil?
+          unless labels.size == 2
+            raise ArgumentError,
+                  'y_true must be binary labels or pos_label must be specified if y_true is multi-label'
+          end
+        else
+          unless y_true.to_a.uniq.include?(pos_label)
+            raise ArgumentError,
+                  'y_true must have elements whose values are pos_label.'
+          end
+        end
+
+        false_pos, true_pos, thresholds = binary_roc_curve(y_true, y_score, pos_label)
+
+        if true_pos.size.zero? || false_pos[0] != 0 || true_pos[0] != 0
+          # NOTE: Numo::NArray#insert is not a destructive method.
+          # rubocop:disable Style/RedundantSelfAssignment
+          true_pos = true_pos.insert(0, 0)
+          false_pos = false_pos.insert(0, 0)
+          thresholds = thresholds.insert(0, thresholds[0] + 1)
+          # rubocop:enable Style/RedundantSelfAssignment
+        end
+
+        tpr = true_pos / true_pos[-1].to_f
+        fpr = false_pos / false_pos[-1].to_f
+
+        [fpr, tpr, thresholds]
+      end
+
+      # Calculate area under the curve using the trapezoidal rule.
+      #
+      # @param x [Numo::Int32/Numo::DFloat] (shape: [n_elements])
+      #   x coordinates. These are expected to monotonously increase or decrease.
+      # @param y [Numo::Int32/Numo::DFloat] (shape: [n_elements]) y coordinates.
+      # @return [Float] area under the curve.
+      def auc(x, y)
+        n_samples = [x.shape[0], y.shape[0]].min
+        raise ArgumentError, 'At least two points are required to calculate area under curve.' if n_samples < 2
+
+        (0...n_samples).to_a.each_cons(2).sum { |i, j| 0.5 * (x[i] - x[j]).abs * (y[i] + y[j]) }
+      end
+
+      private
+
+      def binary_roc_curve(y_true, y_score, pos_label = nil)
+        pos_label = y_true.to_a.uniq.max if pos_label.nil?
+
+        bin_y_true = y_true.eq(pos_label)
+        desc_pred_ids = y_score.sort_index.reverse
+
+        desc_y_true = Numo::Int32.cast(bin_y_true[desc_pred_ids])
+        desc_y_score = y_score[desc_pred_ids]
+
+        threshold_ids = Numo::Int32.cast(desc_y_score.diff.ne(0).where.to_a.append(desc_y_true.size - 1))
+
+        true_pos = desc_y_true.cumsum[threshold_ids]
+        false_pos = 1 + threshold_ids - true_pos
+
+        [false_pos, true_pos, desc_y_score[threshold_ids]]
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/silhouette_score.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'rumale/base/evaluator'
+require 'rumale/pairwise_metric'
+
+module Rumale
+  module EvaluationMeasure
+    # SilhouetteScore is a class that calculates the Silhouette Coefficient.
+    #
+    # @example
+    #   require 'rumale/evaluation_measure/silhouette_score'
+    #
+    #   evaluator = Rumale::EvaluationMeasure::SilhouetteScore.new
+    #   puts evaluator.score(x, predicted)
+    #
+    # *Reference*
+    # - Rousseuw, P J., "Silhouettes: A graphical aid to the interpretation and validation of cluster analysis," Journal of Computational and Applied Mathematics, Vol. 20, pp. 53--65, 1987.
+    class SilhouetteScore
+      include ::Rumale::Base::Evaluator
+
+      # Create a new evaluator that calculates the silhouette coefficient.
+      #
+      # @param metric [String] The metric to calculate the sihouette coefficient.
+      #   If metric is 'euclidean', Euclidean distance is used for dissimilarity between sample points.
+      #   If metric is 'precomputed', the score method expects to be given a distance matrix.
+      def initialize(metric: 'euclidean')
+        @metric = metric
+      end
+
+      # Calculates the silhouette coefficient.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to be used for calculating score.
+      # @param y [Numo::Int32] (shape: [n_samples]) The predicted labels for each sample.
+      # @return [Float] The mean of silhouette coefficient.
+      def score(x, y)
+        dist_mat = @metric == 'precomputed' ? x : ::Rumale::PairwiseMetric.euclidean_distance(x)
+
+        labels = y.to_a.uniq.sort
+        n_clusters = labels.size
+        n_samples = dist_mat.shape[0]
+
+        intra_dists = Numo::DFloat.zeros(n_samples)
+        n_clusters.times do |n|
+          cls_pos = y.eq(labels[n])
+          sz_cluster = cls_pos.count
+          next unless sz_cluster > 1
+
+          cls_dist_mat = dist_mat[cls_pos, cls_pos].dup
+          cls_dist_mat[cls_dist_mat.diag_indices] = 0.0
+          intra_dists[cls_pos] = cls_dist_mat.sum(axis: 0) / (sz_cluster - 1)
+        end
+
+        inter_dists = Numo::DFloat.zeros(n_samples) + Float::INFINITY
+        n_clusters.times do |m|
+          cls_pos = y.eq(labels[m])
+          n_clusters.times do |n|
+            next if m == n
+
+            not_cls_pos = y.eq(labels[n])
+            inter_dists[cls_pos] = Numo::DFloat.minimum(
+              inter_dists[cls_pos], dist_mat[cls_pos, not_cls_pos].mean(1)
+            )
+          end
+        end
+
+        mask = Numo::DFloat.ones(n_samples)
+        n_clusters.times do |n|
+          cls_pos = y.eq(labels[n])
+          mask[cls_pos] = 0 unless cls_pos.count > 1
+        end
+
+        silhouettes = mask * ((inter_dists - intra_dists) / Numo::DFloat.maximum(inter_dists, intra_dists))
+        silhouettes[silhouettes.isnan] = 0.0
+
+        silhouettes.mean
+      end
+    end
+  end
+end

data/lib/rumale/evaluation_measure/version.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Rumale is a machine learning library in Ruby.
+module Rumale
+  # This module consists of the classes for model evaluation.
+  module EvaluationMeasure
+    # @!visibility private
+    VERSION = '0.24.0'
+  end
+end

data/lib/rumale/evaluation_measure.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'numo/narray'
+
+require_relative 'evaluation_measure/version'
+
+require_relative 'evaluation_measure/accuracy'
+require_relative 'evaluation_measure/adjusted_rand_score'
+require_relative 'evaluation_measure/calinski_harabasz_score'
+require_relative 'evaluation_measure/davies_bouldin_score'
+require_relative 'evaluation_measure/explained_variance_score'
+require_relative 'evaluation_measure/f_score'
+require_relative 'evaluation_measure/function'
+require_relative 'evaluation_measure/log_loss'
+require_relative 'evaluation_measure/mean_absolute_error'
+require_relative 'evaluation_measure/mean_squared_error'
+require_relative 'evaluation_measure/mean_squared_log_error'
+require_relative 'evaluation_measure/median_absolute_error'
+require_relative 'evaluation_measure/mutual_information'
+require_relative 'evaluation_measure/normalized_mutual_information'
+require_relative 'evaluation_measure/precision'
+require_relative 'evaluation_measure/precision_recall'
+require_relative 'evaluation_measure/purity'
+require_relative 'evaluation_measure/r2_score'
+require_relative 'evaluation_measure/recall'
+require_relative 'evaluation_measure/roc_auc'
+require_relative 'evaluation_measure/silhouette_score'