rumale 0.13.4 → 0.13.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 522eaabfd67ced29bf275fb6f5cec019ff60e3d5
-  data.tar.gz: 0eb97f58c3764bdcbf448f9a392f8f5091ce418d
+  metadata.gz: b9c82fecc8a90ec9d4379249b749e9257b2738c3
+  data.tar.gz: 1b4ca375174ee9d1c16a35d50e68ee9b5bd471c5
 SHA512:
-  metadata.gz: bf5a3caf614b08813aa4b11673da758778191847ba6fe4c4144cae7da1dd8e4b3ec3eac1367d54b78a00a7afd5ae1ae047fa84c58954b0e7d0571a9442a10380
-  data.tar.gz: 8bdb25aaec7304f12595673d3fa915cc1739ef14fedd89210205f53325de5a96076b4cf718f8d7ca15fdec1f55d5f9c65cd256781ec2d60462a48221525ad068
+  metadata.gz: 3d9bc3c21b951f5738cbf8a2e947d94d6fd735ebbcdc41fdb450d8c4f4f28a9788100c595b1b29bbe81e7124f0ca2b5703c4d6107730fab6f9ab6fe67db4fa1d
+  data.tar.gz: 2f93f78169dd8f694ca65634f3e4619ad83df72a59d0e2eacd8c686b2fdc82011f53b6b49e7c18e1c9a9d86904c77709d9743689e39e8db697f469a77a336fe7

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.13.5
+- Add transformer class for [Factor Analysis](https://yoshoku.github.io/rumale/doc/Rumale/Decomposition/FactorAnalysis.html).
+- Add covariance_type parameter to [Rumale::Clustering::GaussianMixture](https://yoshoku.github.io/rumale/doc/Rumale/Clustering/GaussianMixture.html).
+
 # 0.13.4
 - Add cluster analysis class for [HDBSCAN](https://yoshoku.github.io/rumale/doc/Rumale/Clustering/HDBSCAN.html).
 - Add cluster analysis class for [spectral clustering](https://yoshoku.github.io/rumale/doc/Rumale/Clustering/SpectralClustering.html).

data/lib/rumale.rb

@@ -69,6 +69,7 @@ require 'rumale/clustering/spectral_clustering'
 require 'rumale/clustering/single_linkage'
 require 'rumale/decomposition/pca'
 require 'rumale/decomposition/nmf'
+require 'rumale/decomposition/factor_analysis'
 require 'rumale/manifold/tsne'
 require 'rumale/manifold/mds'
 require 'rumale/preprocessing/l2_normalizer'

data/lib/rumale/clustering/gaussian_mixture.rb

@@ -3,17 +3,20 @@
 require 'rumale/base/base_estimator'
 require 'rumale/base/cluster_analyzer'
 require 'rumale/preprocessing/label_binarizer'
-require 'rumale/pairwise_metric'
 
 module Rumale
   module Clustering
     # GaussianMixture is a class that implements cluster analysis with gaussian mixture model.
-    # The current implementation uses only the diagonal elements of covariance matrices to represent mixture parameters
-    # without using full elements.
     #
     # @example
     #   analyzer = Rumale::Clustering::GaussianMixture.new(n_clusters: 10, max_iter: 50)
     #   cluster_labels = analyzer.fit_predict(samples)
+    #
+    #   # If Numo::Linalg is installed, you can specify 'full' for the tyep of covariance option.
+    #   require 'numo/linalg/autoloader'
+    #   analyzer = Rumale::Clustering::GaussianMixture.new(n_clusters: 10, max_iter: 50, covariance_type: 'full')
+    #   cluster_labels = analyzer.fit_predict(samples)
+    #
     class GaussianMixture
       include Base::BaseEstimator
       include Base::ClusterAnalyzer
@@ -31,18 +34,19 @@ module Rumale
       attr_reader :means
 
       # Return the diagonal elements of covariance matrix of each cluster.
-      # @return [Numo::DFloat] (shape: [n_clusters, n_features])
+      # @return [Numo::DFloat] (shape: [n_clusters, n_features] if 'diag', [n_clusters, n_features, n_features] if 'full')
       attr_reader :covariances
 
       # Create a new cluster analyzer with gaussian mixture model.
       #
       # @param n_clusters [Integer] The number of clusters.
       # @param init [String] The initialization method for centroids ('random' or 'k-means++').
+      # @param covariance_type [String] The type of covariance parameter to be used ('diag' or 'full').
       # @param max_iter [Integer] The maximum number of iterations.
       # @param tol [Float] The tolerance of termination criterion.
       # @param reg_covar [Float] The non-negative regularization to the diagonal of covariance.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(n_clusters: 8, init: 'k-means++', max_iter: 50, tol: 1.0e-4, reg_covar: 1.0e-6, random_seed: nil)
+      def initialize(n_clusters: 8, init: 'k-means++', covariance_type: 'diag', max_iter: 50, tol: 1.0e-4, reg_covar: 1.0e-6, random_seed: nil)
         check_params_integer(n_clusters: n_clusters, max_iter: max_iter)
         check_params_float(tol: tol)
         check_params_string(init: init)
@@ -51,6 +55,7 @@ module Rumale
         @params = {}
         @params[:n_clusters] = n_clusters
         @params[:init] = init == 'random' ? 'random' : 'k-means++'
+        @params[:covariance_type] = covariance_type == 'full' ? 'full' : 'diag'
         @params[:max_iter] = max_iter
         @params[:tol] = tol
         @params[:reg_covar] = reg_covar
@@ -70,14 +75,16 @@ module Rumale
       # @return [GaussianMixture] The learned cluster analyzer itself.
       def fit(x, _y = nil)
         check_sample_array(x)
+        check_enable_linalg('fit')
+
         n_samples = x.shape[0]
         memberships = init_memberships(x)
         @params[:max_iter].times do |t|
           @n_iter = t
           @weights = calc_weights(n_samples, memberships)
           @means = calc_means(x, memberships)
-          @covariances = calc_diag_covariances(x, @means, memberships) + @params[:reg_covar]
-          new_memberships = calc_memberships(x, @weights, @means, @covariances)
+          @covariances = calc_covariances(x, @means, memberships, @params[:reg_covar], @params[:covariance_type])
+          new_memberships = calc_memberships(x, @weights, @means, @covariances, @params[:covariance_type])
           error = (memberships - new_memberships).abs.max
           break if error <= @params[:tol]
           memberships = new_memberships.dup
@@ -91,7 +98,9 @@ module Rumale
       # @return [Numo::Int32] (shape: [n_samples]) Predicted cluster label per sample.
       def predict(x)
         check_sample_array(x)
-        memberships = calc_memberships(x, @weights, @means, @covariances)
+        check_enable_linalg('predict')
+
+        memberships = calc_memberships(x, @weights, @means, @covariances, @params[:covariance_type])
         assign_cluster(memberships)
       end
 
@@ -101,6 +110,8 @@ module Rumale
       # @return [Numo::Int32] (shape: [n_samples]) Predicted cluster label per sample.
       def fit_predict(x)
         check_sample_array(x)
+        check_enable_linalg('fit_predict')
+
         fit(x).predict(x)
       end
 
@@ -141,15 +152,14 @@ module Rumale
         Numo::DFloat.cast(encoder.fit_transform(cluster_ids))
       end
 
-      def calc_memberships(x, weights, means, diag_cov)
+      def calc_memberships(x, weights, means, covars, covar_type)
         n_samples = x.shape[0]
         n_clusters = means.shape[0]
         memberships = Numo::DFloat.zeros(n_samples, n_clusters)
         n_clusters.times do |n|
           centered = x - means[n, true]
-          inv_cov = 1.0 / diag_cov[n, true]
-          sqrt_det_cov = 1.0 / Math.sqrt(diag_cov[n, true].prod)
-          memberships[true, n] = weights[n] * sqrt_det_cov * Numo::NMath.exp(-0.5 * (centered * inv_cov * centered).sum(1))
+          covar = covar_type == 'full' ? covars[n, true, true] : covars[n, true]
+          memberships[true, n] = calc_unnormalized_membership(centered, weights[n], covar, covar_type)
         end
         memberships / memberships.sum(1).expand_dims(1)
       end
@@ -162,13 +172,67 @@ module Rumale
         memberships.transpose.dot(x) / memberships.sum(0).expand_dims(1)
       end
 
-      def calc_diag_covariances(x, means, memberships)
+      def calc_covariances(x, means, memberships, reg_cover, covar_type)
+        if covar_type == 'full'
+          calc_full_covariances(x, means, reg_cover, memberships)
+        else
+          calc_diag_covariances(x, means, reg_cover, memberships)
+        end
+      end
+
+      def calc_diag_covariances(x, means, reg_cover, memberships)
         n_clusters = means.shape[0]
         diag_cov = Array.new(n_clusters) do |n|
           centered = x - means[n, true]
           memberships[true, n].dot(centered**2) / memberships[true, n].sum
         end
-        Numo::DFloat.asarray(diag_cov)
+        Numo::DFloat.asarray(diag_cov) + reg_cover
+      end
+
+      def calc_full_covariances(x, means, reg_cover, memberships)
+        n_features = x.shape[1]
+        n_clusters = means.shape[0]
+        cov_mats = Numo::DFloat.zeros(n_clusters, n_features, n_features)
+        reg_mat = Numo::DFloat.eye(n_features) * reg_cover
+        n_clusters.times do |n|
+          centered = x - means[n, true]
+          members = memberships[true, n]
+          cov_mats[n, true, true] = reg_mat + (centered.transpose * members).dot(centered) / members.sum
+        end
+        cov_mats
+      end
+
+      def calc_unnormalized_membership(centered, weight, covar, covar_type)
+        inv_covar = calc_inv_covariance(covar, covar_type)
+        inv_sqrt_det_covar = calc_inv_sqrt_det_covariance(covar, covar_type)
+        distances = if covar_type == 'full'
+                      (centered.dot(inv_covar) * centered).sum(1)
+                    else
+                      (centered * inv_covar * centered).sum(1)
+                    end
+        weight * inv_sqrt_det_covar * Numo::NMath.exp(-0.5 * distances)
+      end
+
+      def calc_inv_covariance(covar, covar_type)
+        if covar_type == 'full'
+          Numo::Linalg.inv(covar)
+        else
+          1.0 / covar
+        end
+      end
+
+      def calc_inv_sqrt_det_covariance(covar, covar_type)
+        if covar_type == 'full'
+          1.0 / Math.sqrt(Numo::Linalg.det(covar))
+        else
+          1.0 / Math.sqrt(covar.prod)
+        end
+      end
+
+      def check_enable_linalg(method_name)
+        if (@params[:covariance_type] == 'full') && !enable_linalg?
+          raise "GaussianMixture##{method_name} requires Numo::Linalg when covariance_type is 'full' but that is not loaded."
+        end
       end
     end
   end

data/lib/rumale/decomposition/factor_analysis.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+require 'rumale/base/transformer'
+require 'rumale/utils'
+
+module Rumale
+  module Decomposition
+    # FactorAnalysis is a class that implements fator analysis with EM algorithm.
+    #
+    # @example
+    #   require 'numo/linalg/autoloader'
+    #   decomposer = Rumale::Decomposition::FactorAnalysis.new(n_components: 2)
+    #   representaion = decomposer.fit_transform(samples)
+    #
+    # *Reference*
+    # - D. Barber, "Bayesian Reasoning and Machine Learning," Cambridge University Press, 2012.
+    class FactorAnalysis
+      include Base::BaseEstimator
+      include Base::Transformer
+
+      # Returns the mean vector.
+      # @return [Numo::DFloat] (shape: [n_features])
+      attr_reader :mean
+
+      # Returns the estimated noise variance for each feature.
+      # @return [Numo::DFloat] (shape: [n_features])
+      attr_reader :noise_variance
+
+      # Returns the components with maximum variance.
+      # @return [Numo::DFloat] (shape: [n_components, n_features])
+      attr_reader :components
+
+      # Returns the log likelihood at each iteration.
+      # @return [Numo::DFloat] (shape: [n_iter])
+      attr_reader :loglike
+
+      # Return the number of iterations run for optimization
+      # @return [Integer]
+      attr_reader :n_iter
+
+      # Create a new transformer with factor analysis.
+      #
+      # @param n_components [Integer] The number of components (dimensionality of latent space).
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param tol [Float/Nil] The tolerance of termination criterion for EM algorithm.
+      #   If nil is given, iterate EM steps up to the maximum number of iterations.
+      def initialize(n_components: 2, max_iter: 100, tol: 1e-8)
+        check_params_integer(n_components: n_components, max_iter: max_iter)
+        check_params_type_or_nil(Float, tol: tol)
+        check_params_positive(n_components: n_components, max_iter: max_iter)
+        @params = {}
+        @params[:n_components] = n_components
+        @params[:max_iter] = max_iter
+        @params[:tol] = tol
+        @mean = nil
+        @noise_variance = nil
+        @components = nil
+        @loglike = nil
+        @n_iter = nil
+      end
+
+      # Fit the model with given training data.
+      #
+      # @overload fit(x) -> FactorAnalysis
+      #   @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @return [FactorAnalysis] The learned transformer itself.
+      def fit(x, _y = nil)
+        raise 'FactorAnalysis#fit requires Numo::Linalg but that is not loaded.' unless enable_linalg?
+
+        # initialize some variables.
+        n_samples, n_features = x.shape
+        @mean = x.mean(0)
+        centered_x = x - @mean
+        cov_mat = centered_x.transpose.dot(centered_x) / n_samples
+        sample_vars = x.var(0)
+        sqrt_n_samples = Math.sqrt(n_samples)
+        @noise_variance = Numo::DFloat.ones(n_features)
+
+        # run optimization.
+        old_loglike = 0.0
+        @n_iter = 0
+        @loglike = [] unless @params[:tol].nil?
+        @params[:max_iter].times do |t|
+          @n_iter = t + 1
+          sqrt_noise_variance = Numo::NMath.sqrt(@noise_variance)
+          scaled_x = centered_x / (sqrt_noise_variance * sqrt_n_samples + 1e-12)
+          s, u = truncate_svd(scaled_x, @params[:n_components])
+          scaler = Numo::NMath.sqrt(Numo::DFloat.maximum(s**2 - 1.0, 0.0))
+          @components = (sqrt_noise_variance.diag.dot(u) * scaler).transpose.dup
+          @noise_variance = Numo::DFloat.maximum(sample_vars - @components.transpose.dot(@components).diagonal, 1e-12)
+          next if @params[:tol].nil?
+          new_loglike = log_likelihood(cov_mat, @components, @noise_variance)
+          @loglike.push(new_loglike)
+          break if (old_loglike - new_loglike).abs <= @params[:tol]
+          old_loglike = new_loglike
+        end
+
+        @loglike = Numo::DFloat.cast(@loglike) unless @params[:tol].nil?
+        @components = @components[0, true].dup if @params[:n_components] == 1
+        self
+      end
+
+      # Fit the model with training data, and then transform them with the learned model.
+      #
+      # @overload fit_transform(x) -> Numo::DFloat
+      #   @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed data
+      def fit_transform(x, _y = nil)
+        check_sample_array(x)
+        raise 'FactorAnalysis#fit_transform requires Numo::Linalg but that is not loaded.' unless enable_linalg?
+
+        fit(x).transform(x)
+      end
+
+      # Transform the given data with the learned model.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The data to be transformed with the learned model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed data.
+      def transform(x)
+        check_sample_array(x)
+        raise 'FactorAnalysis#transform requires Numo::Linalg but that is not loaded.' unless enable_linalg?
+
+        factors = @params[:n_components] == 1 ? @components.expand_dims(0) : @components
+        centered_x = x - @mean
+        beta = Numo::Linalg.inv(Numo::DFloat.eye(factors.shape[0]) + (factors / @noise_variance).dot(factors.transpose))
+        z = centered_x.dot((beta.dot(factors) / @noise_variance).transpose)
+        @params[:n_components] == 1 ? z[true, 0].dup : z
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data.
+      def marshal_dump
+        { params: @params,
+          mean: @mean,
+          noise_variance: @noise_variance,
+          components: @components,
+          loglike: @loglike,
+          n_iter: @n_iter }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @mean = obj[:mean]
+        @noise_variance = obj[:noise_variance]
+        @components = obj[:components]
+        @loglike = obj[:loglike]
+        @n_iter = obj[:n_iter]
+      end
+
+      private
+
+      def log_likelihood(cov_mat, factors, noise_vars)
+        n_samples = noise_vars.size
+        fact_cov_mat = factors.transpose.dot(factors) + noise_vars.diag
+        n_samples.fdiv(2) * Math.log(Numo::Linalg.det(fact_cov_mat)) + Numo::Linalg.inv(fact_cov_mat).dot(cov_mat).trace
+      end
+
+      def truncate_svd(x, k)
+        m = x.shape[1]
+        eig_vals, eig_vecs = Numo::Linalg.eigh(x.transpose.dot(x), vals_range: (m - k)...m)
+        s = Numo::NMath.sqrt(eig_vals.reverse.dup)
+        u = eig_vecs.reverse(1).dup
+        [s, u]
+      end
+    end
+  end
+end

data/lib/rumale/version.rb

@@ -3,5 +3,5 @@
 # Rumale is a machine learning library in Ruby.
 module Rumale
   # The version of Rumale you are using.
-  VERSION = '0.13.4'
+  VERSION = '0.13.5'
 end

data/rumale.gemspec

@@ -46,10 +46,10 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'numo-narray', '>= 0.9.1'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'coveralls', '>= 0.8.23'
+  spec.add_development_dependency 'coveralls', '~> 0.8'
   spec.add_development_dependency 'numo-linalg', '>= 0.1.4'
   spec.add_development_dependency 'parallel', '>= 1.17.0'
-  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rake-compiler', '~> 1.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rumale
 version: !ruby/object:Gem::Version
-  version: 0.13.4
+  version: 0.13.5
 platform: ruby
 authors:
 - yoshoku
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-09-22 00:00:00.000000000 Z
+date: 2019-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -42,16 +42,16 @@ dependencies:
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.23
+        version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.23
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: numo-linalg
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
@@ -171,6 +171,7 @@ files:
 - lib/rumale/clustering/snn.rb
 - lib/rumale/clustering/spectral_clustering.rb
 - lib/rumale/dataset.rb
+- lib/rumale/decomposition/factor_analysis.rb
 - lib/rumale/decomposition/nmf.rb
 - lib/rumale/decomposition/pca.rb
 - lib/rumale/ensemble/ada_boost_classifier.rb