machine_learning_workbench 0.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a0550319ef523cd49f7c09b635a4e21508cf730
-  data.tar.gz: b3fb9a716bfac1850bc5af0c8abf96f17f0292b6
+  metadata.gz: 743cc4c65bda521785e00dc563b20fcd5660a6af
+  data.tar.gz: 80924d34dc550df7b25d565022f50297b8bb72d5
 SHA512:
-  metadata.gz: 51b05034a2fffcc135388c5760b14067728d4fcf7210ff47d4a7f58f66fb174d3940fd538e3551a78d9df24a0c14b01e0f277aba2ceb898d7206302f8c30721b
-  data.tar.gz: 0dd1cf85fdb8577278882fe197e90032b1061a11fec9176b8d64ef38b97140876059ec964199007f52840eaab13ca2a25b2b01b8d0935c5f55c2ffc518f59124
+  metadata.gz: 2ef96c2884a6f43304f0dba63239a7469b80c7be46c97af7c8de283a448a8ee049b55cdc98b0aa5bfde82fc024fde891c982ec621bb4cafcd6070814e8366976
+  data.tar.gz: 0b5571eb26babf90015deec8425dc0a9c75bebd47ef50b4927a131e86f5f0c9e6e828e3ee1b212597222b9aa7209cd54f6b3db7a9c8ec6afcceafe94016165a1

data/bin/setup

@@ -4,7 +4,7 @@ IFS=$'\n\t'
 set -vx
 
 # ubuntu:
-sudo apt install libatlas-base-dev # for nmatrix
+sudo apt install libopenblas-base # for numo
 
 bundle install
 

data/examples/image_compression.rb

@@ -5,7 +5,8 @@ Img = MachineLearningWorkbench::Tools::Imaging
 Norm = MachineLearningWorkbench::Tools::Normalization
 
 ncentrs = 1
-image_files = Dir[ENV['HOME']+'/jaffe/KA.HA*.png']
+# image_files = Dir[ENV['HOME']+'/jaffe/KA.HA*.png']
+image_files = Dir[ENV['HOME']+'/jaffe/*.png']
 raise "Download the JAFFE dataset in your home dir" if image_files&.empty?
 # ... and convert the `.tiff` in `.png`: `mogrify -format png jaffe/*.tiff`
 centr_range = [-1, 1]
@@ -14,14 +15,13 @@ img_range = [0, 2**16-1]
 
 puts "Loading images"
 images = image_files.map do |fname|
-  nmat = Img.nmat_from_png fname, flat: true, dtype: :float64
-  ret = Norm.feature_scaling nmat, from: img_range, to: centr_range
+  ary = Img.narr_from_png fname, flat: true
+  ret = Norm.feature_scaling ary, from: img_range, to: centr_range
 end
 
 puts "Initializing VQ"
 vq = VectorQuantization.new ncentrs: ncentrs,
-  dims: images.first.shape, lrate: 0.3,
-  dtype: images.first.dtype, vrange: centr_range
+  dims: images.first.shape, lrate: 0.3, vrange: centr_range
 
 puts "Training"
 vq.train images, debug: true

data/lib/machine_learning_workbench.rb

@@ -1,4 +1,20 @@
-require 'nmatrix'
+
+gpu = false             # prepare for switching to GPUs
+if gpu
+  require 'cumo/narray'
+  require 'cumo/linalg'
+  Xumo = Cumo
+else
+  require 'numo/narray'
+  require 'numo/linalg'
+  # currently required for Ubuntu 16.04
+  # depends on openblas: `sudo apt install libopenblas-base`
+  Fiddle.dlopen("/usr/lib/libopenblas.so")
+  Numo::Linalg::Lapack.dlopen("/usr/lib/liblapacke.so.3")
+  Xumo = Numo
+end
+NArray = Xumo::DFloat   # set a single data type across the WB for now
+NMath = Xumo::NMath     # shorthand for extended math module
 
 module MachineLearningWorkbench
   module Compressor

data/lib/machine_learning_workbench/compressor/vector_quantization.rb

@@ -2,13 +2,12 @@ module MachineLearningWorkbench::Compressor
 
   # Standard Vector Quantization
   class VectorQuantization
-    attr_reader :ncentrs, :centrs, :dims, :vrange, :dtype, :lrate, :rng, :ntrains
+    attr_reader :ncentrs, :centrs, :dims, :vrange, :lrate, :rng, :ntrains
     Verification = MachineLearningWorkbench::Tools::Verification
 
-    def initialize ncentrs:, dims:, vrange:, dtype:, lrate:, rseed: Random.new_seed
+    def initialize ncentrs:, dims:, vrange:, lrate:, rseed: Random.new_seed
       @rng = Random.new rseed
       @ncentrs = ncentrs
-      @dtype = dtype
       @dims = Array(dims)
       check_lrate lrate # hack: so that we can overload it in online_vq
       @lrate = lrate
@@ -32,15 +31,13 @@ module MachineLearningWorkbench::Compressor
 
     # Creates a new (random) centroid
     def new_centr
-      # TODO: this is too slow, find another way to use the rng
-      # NMatrix.new(dims, dtype: dtype) { rng.rand Range.new *vrange }
-      NMatrix.random dims, dtype: dtype
+      NArray.new(*dims).rand(*vrange)
     end
 
     # Computes similarities between vector and all centroids
     def similarities vec
       raise NotImplementedError if vec.shape.size > 1
-      centrs.map { |c| c.dot(vec).first }
+      centrs.map { |c| c.dot(vec) }
       # require 'parallel'
       # Parallel.map(centrs) { |c| c.dot(vec).first }
     end
@@ -85,7 +82,7 @@ module MachineLearningWorkbench::Compressor
     end
 
     # Per-pixel errors in reconstructing vector
-    # @return [NMatrix] residuals
+    # @return [NArray] residuals
     def reconstr_error vec
       reconstruction(vec) - vec
     end

data/lib/machine_learning_workbench/monkey.rb

@@ -217,12 +217,112 @@ module MachineLearningWorkbench::Monkey
       raise NotImplementedError, "There's no setter for the data pointer!"
     end
   end
+
+  module ToNArrayConvertible
+    def to_na
+      NArray[*self]
+    end
+  end
+
+  module NArrayOuterFlattable
+    # Flat-output generalized outer relationship. Same as `#outer`, but the
+    # result is a 2-dim matrix of the interactions between all the elements
+    # in `self` (as rows) and all the elements in `other` (as columns)
+    # @param other [NArray] other matrix
+    # @return [NArray]
+    def outer_flat other
+      # TODO: Numo::NArray should be able to implement this with `#outer` and some other
+      # function to flatten the right layer -- much faster
+      raise ArgumentError, "Need to pass an operand block" unless block_given?
+      self.class.zeros([self.size, other.size]).tap do |ret|
+        self.size.times do |r|
+          other.size.times do |c|
+            ret[r,c] = yield self[r], other[c]
+          end
+        end
+      end
+    end
+  end
+
+  module NArrayApproximatable
+    # Verifies if `self` and `other` are withing `epsilon` of each other.
+    # @param other [NArray]
+    # @param epsilon [NArray]
+    # @return [Boolean]
+    def approximates? other, epsilon=1e-5
+      ((self - other).abs < epsilon).all?
+    end
+  end
+
+  module Invertable
+    # Inverses matrix
+    # @return [NArray]
+    def invert
+      Numo::Linalg.inv self
+    end
+  end
+
+  module Exponentiable
+    # Matrix exponential: `e**self` (not to be confused with `self**n`)
+    # @return [NArray]
+    def exponential
+      raise ArgumentError if ndim > 2
+      # special case: one-dimensional matrix: just exponentiate the values
+      return Numo::NMath.exp(self) if (ndim == 1) || shape.include?(1)
+      # at this point we need to validate it is a square matrix
+      raise ArgumentError unless shape.reduce(&:==)
+
+      # Eigenvalue decomposition method from `scipy/linalg/matfuncs.py#expm2` (deprecated)
+      # https://github.com/scipy/scipy/commit/236e0740ba951cb455ba8b6a306abb32740131cf
+      # s, vr = eig(A)
+      # vri = inv(vr)
+      # r = dot(dot(vr, diag(exp(s))), vri)
+
+      # TODO: this is a simple but outdated method, switch to Pade approximation
+      # https://github.com/scipy/scipy/blob/11509c4a98edded6c59423ac44ca1b7f28fba1fd/scipy/sparse/linalg/matfuncs.py#L557
+
+      # e_values, l_e_vectors, r_e_vectors_t = Numo::Linalg.svd self
+      evals, _wi, _vl, r_evecs = Numo::Linalg::Lapack.call(:geev, self, jobvl: false, jobvr: true)
+      r_evecs_t = r_evecs#.transpose
+      r_evecs_inv = r_evecs_t.invert
+      evals_exp_dmat = Numo::NMath.exp(evals).diag
+
+      # l_e_vectors.dot(e_vals_exp_dmat).dot(l_e_vectors.invert)#.transpose
+      r_evecs_t.dot(evals_exp_dmat).dot(r_evecs_inv)
+    end
+  end
+
+  module Mappable
+    # Maps along a NArray dimension, and returns NArray
+    # @return [NArray]
+    # NOTE: this indexing is not consistent with NArray, which uses 0 to indicate
+    #   columns rather than the 0th dimension (rows)
+    def map dim=0
+      raise ArgumentError unless dim.kind_of?(Integer) && dim.between?(0,ndim)
+      # TODO: return iterator instead of raise
+      raise NotImplementedError unless block_given?
+      indices = [true]*ndim
+      ret = []
+      shape[dim].times.each do |i|
+        indices[dim] = i
+        ret << yield(self[*indices])
+      end
+      self.class[*ret]
+    end
+  end
+
 end
 
 Array.include MachineLearningWorkbench::Monkey::Dimensionable
-NMatrix.extend MachineLearningWorkbench::Monkey::Buildable
-require 'nmatrix/lapack_plugin' # loads whichever is installed between atlas and lapacke
-NMatrix.include MachineLearningWorkbench::Monkey::AdvancelyOperationable
+# NMatrix.extend MachineLearningWorkbench::Monkey::Buildable
+# require 'nmatrix/lapack_plugin' # loads whichever is installed between atlas and lapacke
+# NMatrix.include MachineLearningWorkbench::Monkey::AdvancelyOperationable
 Numeric.include MachineLearningWorkbench::Monkey::NumericallyApproximatable
-NMatrix.include MachineLearningWorkbench::Monkey::MatrixApproximatable
-NMatrix.include MachineLearningWorkbench::Monkey::CPtrDumpable
+# NMatrix.include MachineLearningWorkbench::Monkey::MatrixApproximatable
+# NMatrix.include MachineLearningWorkbench::Monkey::CPtrDumpable
+Array.include MachineLearningWorkbench::Monkey::ToNArrayConvertible
+NArray.include MachineLearningWorkbench::Monkey::NArrayApproximatable
+NArray.include MachineLearningWorkbench::Monkey::NArrayOuterFlattable
+NArray.include MachineLearningWorkbench::Monkey::Exponentiable
+NArray.include MachineLearningWorkbench::Monkey::Invertable
+NArray.prepend MachineLearningWorkbench::Monkey::Mappable

data/lib/machine_learning_workbench/neural_network/base.rb

@@ -7,51 +7,52 @@ module MachineLearningWorkbench::NeuralNetwork
     #   List of matrices, each being the weights
     #   connecting a layer's inputs (rows) to a layer's neurons (columns),
     #   hence its shape is `[ninputs, nneurs]`
-    #   @return [Array<NMatrix>] list of weight matrices, each uniquely describing a layer
+    #   @return [Array<NArray>] list of weight matrices, each uniquely describing a layer
+    #   TODO: return a NArray after the usage of `#map` is figured out
     # @!attribute [r] state
     #   It's a list of one-dimensional matrices, each an input to a layer, plus the output layer's output. The first element is the input to the first layer of the network, which is composed of the network's input, possibly the first layer's activation on the last input (recursion), and a bias (fixed `1`). The second to but-last entries follow the same structure, but with the previous layer's output in place of the network's input. The last entry is the activation of the output layer, without additions since it's not used as an input by anyone.
-    #   @return [Array<NMatrix>] current state of the network.
+    #   TODO: return a NArray after the usage of `#map` is figured out
+    #   @return [Array<NArray>] current state of the network.
     # @!attribute [r] act_fn
     #   activation function, common to all neurons (for now)
     #   @return [#call] activation function
     # @!attribute [r] struct
     #   list of number of (inputs or) neurons in each layer
     #   @return [Array<Integer>] structure of the network
-    attr_reader :layers, :state, :act_fn, :struct, :dtype
+    attr_reader :layers, :state, :act_fn, :act_fn_name, :struct
 
 
     ## Initialization
 
     # @param struct [Array<Integer>] list of layer sizes
     # @param act_fn [Symbol] choice of activation function for the neurons
-    # @param dtype [NMatrix dtype] NMatrix dtype for weights and states
-    def initialize struct, act_fn: nil, dtype: :float32
+    def initialize struct, act_fn: nil
       @struct = struct
-      @act_fn = self.get_act_fn(act_fn || :sigmoid)
+      @act_fn_name = act_fn || :sigmoid
+      @act_fn = send(act_fn_name)
       # @state holds both inputs, possibly recurrency, and bias
       # it is a complete input for the next layer, hence size from layer sizes
       @state = layer_row_sizes.collect do |size|
-        NMatrix.zeros([1, size], dtype: dtype)
+        NArray.zeros [1, size]
       end
       # to this, append a matrix to hold the final network output
-      @state.push NMatrix.zeros([1, nneurs(-1)], dtype: dtype)
+      @state.push NArray.zeros [1, nneurs(-1)]
       reset_state
     end
 
     # Reset the network to the initial state
     def reset_state
-      @state.each do |m| # state has only single-row matrices
-        # reset all to zero
-        m[0,0..-1] = 0
-        # add bias to all but output
-        m[0,-1] = 1 unless m.object_id == @state.last.object_id
+      state.each do |s|
+        s.fill 0           # reset state to zero
+        s[0,-1] = 1        # add bias
       end
+      state[-1][0,-1] = 0  # last layer has no bias
     end
 
     # Initialize the network with random weights
     def init_random
-      # Will only be used for testing, no sense optimizing it (NMatrix#rand)
-      # Reusing #load_weights instead helps catching bugs
+      # Will only be used for testing, no sense optimizing it now (NArray#rand)
+      # Reusing `#load_weights` instead helps catching bugs
       load_weights nweights.times.collect { rand(-1.0..1.0) }
     end
 
@@ -90,7 +91,7 @@ module MachineLearningWorkbench::NeuralNetwork
     # @return [Array] three-dimensional Array of weights: a list of weight
     #   matrices, one for each layer.
     def weights
-      layers.collect(&:to_consistent_a)
+      layers.collect(&:to_a)
     end
 
     # Number of neurons per layer. Although this implementation includes inputs
@@ -126,10 +127,10 @@ module MachineLearningWorkbench::NeuralNetwork
     def load_weights weights
       raise ArgumentError unless weights.size == nweights
       weights_iter = weights.each
-      @layers ||= layer_shapes.collect { |shape| NMatrix.new shape, dtype: dtype }
-      layers.each do |nmat|
-        nmat.each_with_indices do |_val, *idxs|
-          nmat[*idxs] = weights_iter.next
+      @layers ||= layer_shapes.collect { |shape| NArray.zeros shape }
+      layers.each do |narr|
+        narr.each_with_index do |_val, *idxs|
+          narr[*idxs] = weights_iter.next
         end
       end
       reset_state
@@ -139,11 +140,6 @@ module MachineLearningWorkbench::NeuralNetwork
 
     ## Activation
 
-    # The "fixed `1`" used in the layer's input
-    def bias
-      @bias ||= NMatrix[[1], dtype: dtype]
-    end
-
     # Activate the network on a given input
     # @param input [Array<Float>] the given input
     # @return [Array] the activation of the output layer
@@ -153,9 +149,9 @@ module MachineLearningWorkbench::NeuralNetwork
       # load input in first state
       @state[0][0, 0..-2] = input
       # activate layers in sequence
-      (0...nlayers).each do |i|
+      nlayers.times.each do |i|
         act = activate_layer i
-        @state[i+1][0,0...act.size] = act
+        @state[i+1][0, 0...act.size] = act
       end
       return out
     end
@@ -163,49 +159,38 @@ module MachineLearningWorkbench::NeuralNetwork
     # Extract and convert the output layer's activation
     # @return [Array] the activation of the output layer as 1-dim Array
     def out
-      state.last.to_flat_a
+      state.last.to_a.flatten
     end
 
     # define #activate_layer in child class
 
     ## Activation functions
 
-    # Activation function caller. Allows to cleanly define the activation function as one-dimensional, by calling it over the inputs and building a NMatrix to return.
-    # @return [NMatrix] activations for one layer
-    def get_act_fn type, *args
-      fn = send(type,*args)
-      lambda do |inputs|
-        NMatrix.new([1, inputs.size], dtype: dtype) do |_,i|
-          # single-row matrix, indices are columns
-          fn.call inputs[i]
-        end
-      end
-    end
-
     # Traditional sigmoid with variable steepness
     def sigmoid k=0.5
       # k is steepness:  0<k<1 is flatter, 1<k is flatter
       # flatter makes activation less sensitive, better with large number of inputs
-      lambda { |x| 1.0 / (Math.exp(-k * x) + 1.0) }
+      lambda { |x| 1.0 / (Numo::NMath.exp(-k * x) + 1.0) }
     end
 
     # Traditional logistic
     def logistic
       lambda { |x|
-        exp = Math.exp(x)
-        exp.infinite? ? exp : exp / (1.0 + exp)
+        exp = Numo::NMath.exp(x)
+        # exp.infinite? ? exp : exp / (1.0 + exp)
+        exp / (1.0 + exp)
       }
     end
 
     # LeCun hyperbolic activation
     # @see http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf Section 4.4
     def lecun_hyperbolic
-      lambda { |x| 1.7159 * Math.tanh(2.0*x/3.0) + 1e-3*x }
+      lambda { |x| 1.7159 * Numo::NMath.tanh(2.0*x/3.0) + 1e-3*x }
     end
 
     # Rectified Linear Unit (ReLU)
     def relu
-      lambda { |x| x>0 && x || 0 }
+      lambda { |x| (x>0).all? && x || x.class.zeros(x.shape) }
     end
 
 

data/lib/machine_learning_workbench/neural_network/recurrent.rb

@@ -23,7 +23,7 @@ module MachineLearningWorkbench::NeuralNetwork
       previous = nlay     # index of previous layer (inputs)
       current = nlay + 1  # index of current layer (outputs)
       # Copy the level's last-time activation to the input (previous state)
-      # NOTE: ranges in NMatrix#[] not reliable! gotta loop :(
+      # TODO: ranges in `NArray#[]` should be reliable, get rid of loop
       nneurs(current).times do |i| # for each activations to copy
         # Copy output from last-time activation to recurrency in previous state
         @state[previous][0, nneurs(previous) + i] = state[current][0, i]

data/lib/machine_learning_workbench/optimizer/natural_evolution_strategies/base.rb

@@ -2,7 +2,7 @@
 module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
   # Natural Evolution Strategies base class
   class Base
-    attr_reader :ndims, :mu, :sigma, :opt_type, :obj_fn, :parallel_fit, :id, :rng, :last_fits, :best, :rescale_popsize, :rescale_lrate, :dtype
+    attr_reader :ndims, :mu, :sigma, :opt_type, :obj_fn, :parallel_fit, :eye, :rng, :last_fits, :best, :rescale_popsize, :rescale_lrate
 
     # NES object initialization
     # @param ndims [Integer] number of parameters to optimize
@@ -19,24 +19,23 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
     #   a single instance.
     # @param rescale_popsize [Float] scaling for the default population size
     # @param rescale_lrate [Float] scaling for the default learning rate
-    # @param dtype [NMatrix dtype] NMatrix dtype for all matrix computation
-    def initialize ndims, obj_fn, opt_type, rseed: nil, mu_init: 0, sigma_init: 1, parallel_fit: false, rescale_popsize: 1, rescale_lrate: 1, dtype: :float64
+    def initialize ndims, obj_fn, opt_type, rseed: nil, mu_init: 0, sigma_init: 1, parallel_fit: false, rescale_popsize: 1, rescale_lrate: 1
       raise ArgumentError unless [:min, :max].include? opt_type
       raise ArgumentError unless obj_fn.respond_to? :call
       @ndims, @opt_type, @obj_fn, @parallel_fit = ndims, opt_type, obj_fn, parallel_fit
       @rescale_popsize, @rescale_lrate = rescale_popsize, rescale_lrate
-      @id = NMatrix.identity(ndims, dtype: dtype)
+      @eye = NArray.eye(ndims)
       rseed ||= Random.new_seed
       # puts "NES rseed: #{s}"  # currently disabled
       @rng = Random.new rseed
       @best = [(opt_type==:max ? -1 : 1) * Float::INFINITY, nil]
       @last_fits = []
-      @dtype = dtype
       initialize_distribution mu_init: mu_init, sigma_init: sigma_init
     end
 
     # Box-Muller transform: generates standard (unit) normal distribution samples
     # @return [Float] a single sample from a standard normal distribution
+    # @note Numo::NArray implements this :) glad to have switched!
     def standard_normal_sample
       rho = Math.sqrt(-2.0 * Math.log(rng.rand))
       theta = 2 * Math::PI * rng.rand
@@ -53,7 +52,7 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
     def lrate;   @lrate     ||= cmaes_lrate * rescale_lrate end
 
     # Magic numbers from CMA-ES (TODO: add proper citation)
-    # @return [NMatrix] scale-invariant utilities
+    # @return [NArray] scale-invariant utilities
     def cmaes_utilities
       # Algorithm equations are meant for fitness maximization
       # Match utilities with individuals sorted by INCREASING fitness
@@ -63,7 +62,7 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
       total = log_range.reduce(:+)
       buf = 1.0/popsize
       vals = log_range.collect { |v| v / total - buf }.reverse
-      NMatrix[vals, dtype: dtype]
+      NArray[vals]
     end
 
     # (see #cmaes_utilities)
@@ -78,21 +77,26 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
       [5, 4 + (3*Math.log(ndims)).floor].max
     end
 
-    # Samples a standard normal distribution to construct a NMatrix of
+    # Samples a standard normal distribution to construct a NArray of
     #   popsize multivariate samples of length ndims
-    # @return [NMatrix] standard normal samples
+    # @return [NArray] standard normal samples
+    # @note Numo::NArray implements this :) glad to have switched!
     def standard_normal_samples
-      NMatrix.new([popsize, ndims], dtype: dtype) { standard_normal_sample }
+      NArray.zeros([popsize, ndims]).tap do |ret|
+        ret.each_with_index { |_,*i| ret[*i] = standard_normal_sample }
+      end
     end
 
     # Move standard normal samples to current distribution
-    # @return [NMatrix] individuals
+    # @return [NArray] individuals
     def move_inds inds
       # TODO: can we reduce the transpositions?
-      # sigma.dot(inds.transpose).map(&mu.method(:+)).transpose
-      multi_mu = NMatrix[*inds.rows.times.collect {mu.to_a}, dtype: dtype].transpose
-      (multi_mu + sigma.dot(inds.transpose)).transpose
-      # sigma.dot(inds.transpose).transpose + inds.rows.times.collect {mu.to_a}.to_nm
+
+      # multi_mu = NMatrix[*inds.rows.times.collect {mu.to_a}, dtype: dtype].transpose
+      # (multi_mu + sigma.dot(inds.transpose)).transpose
+
+      mu_tile = mu.tile(inds.shape.first, 1).transpose
+      (mu_tile + sigma.dot(inds.transpose)).transpose
     end
 
     # Sorted individuals
@@ -100,18 +104,27 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
     # matched with individuals sorted by INCREASING fitness. Then reverse order for minimization.
     # @return standard normal samples sorted by the respective individuals' fitnesses
     def sorted_inds
-      samples = standard_normal_samples
-      inds = move_inds(samples).to_a
+      # samples = standard_normal_samples # Numo::NArray implements the Box-Muller :)
+      samples = NArray.new([popsize, ndims]).rand_norm(0,1)
+      inds = move_inds(samples)
       fits = parallel_fit ? obj_fn.call(inds) : inds.map(&obj_fn)
       # Quick cure for NaN fitnesses
-      fits.map! { |x| x.nan? ? (opt_type==:max ? -1 : 1) * Float::INFINITY : x }
+      fits.map { |x| x.nan? ? (opt_type==:max ? -1 : 1) * Float::INFINITY : x }
       @last_fits = fits # allows checking for stagnation
-      sorted = [fits, inds, samples.to_a].transpose.sort_by(&:first)
-      sorted.reverse! if opt_type==:min
-      this_best = sorted.last.take(2)
+
+      # sorted = [fits.to_a, inds, samples.to_a].transpose.sort_by(&:first)
+      # sorted.reverse! if opt_type==:min
+      # this_best = sorted.last.take(2)
+      # NArray[*sorted.map(&:last)]
+
+      sort_idxs = fits.sort_index
+      sort_idxs = sort_idxs.reverse if opt_type == :min
+      this_best = [fits[sort_idxs[-1]], inds[sort_idxs[-1]]]
+
       opt_cmp_fn = opt_type==:min ? :< : :>
       @best = this_best if this_best.first.send(opt_cmp_fn, best.first)
-      NMatrix[*sorted.map(&:last), dtype: dtype]
+
+      samples[sort_idxs,true]
     end
 
     # @!method interface_methods

data/lib/machine_learning_workbench/optimizer/natural_evolution_strategies/bdnes.rb

@@ -46,8 +46,8 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
       end.transpose
 
       # Join the individuals for evaluation
-      full_inds = inds_lst.reduce(&:hconcat).to_a
-      # Need to fix samples dimensions for sorting
+      full_inds = inds_lst.reduce { |mem, var| mem.concatenate var, axis: 1 }
+      # Need to fix sample dimensions for sorting
       # - current dims: nblocks x ninds x [block sizes]
       # - for sorting: ninds x nblocks x [block sizes]
       full_samples = samples_lst.transpose
@@ -55,24 +55,30 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
       # Evaluate fitness of complete individuals
       fits = parallel_fit ? obj_fn.call(full_inds) : full_inds.map(&obj_fn)
       # Quick cure for NaN fitnesses
-      fits.map! { |x| x.nan? ? (opt_type==:max ? -1 : 1) * Float::INFINITY : x }
+      fits.map { |x| x.nan? ? (opt_type==:max ? -1 : 1) * Float::INFINITY : x }
       @last_fits = fits # allows checking for stagnation
 
       # Sort inds based on fit and opt_type, save best
-      sorted = [fits, full_inds, full_samples].transpose.sort_by(&:first)
-      sorted.reverse! if opt_type==:min
-      this_best = sorted.last.take(2)
+      # sorted = [fits, full_inds, full_samples].transpose.sort_by(&:first)
+      # sorted.reverse! if opt_type==:min
+      # this_best = sorted.last.take(2)
+      # opt_cmp_fn = opt_type==:min ? :< : :>
+      # @best = this_best if this_best.first.send(opt_cmp_fn, best.first)
+      # sorted_samples = sorted.map(&:last)
+      sort_idxs = fits.sort_index
+      sort_idxs = sort_idxs.reverse if opt_type == :min
+      this_best = [fits[sort_idxs[-1]], full_inds[sort_idxs[-1]]]
       opt_cmp_fn = opt_type==:min ? :< : :>
       @best = this_best if this_best.first.send(opt_cmp_fn, best.first)
-      sorted_samples = sorted.map(&:last)
+      sorted_samples = full_samples.values_at *sort_idxs
 
       # Need to bring back sample dimensions for each block
       # - current dims: ninds x nblocks x [block sizes]
       # - target blocks list: nblocks x ninds x [block sizes]
       block_samples = sorted_samples.transpose
 
-      # then back to NMatrix for usage in training
-      block_samples.map { |sample| NMatrix[*sample, dtype: dtype] }
+      # then back to NArray for usage in training
+      block_samples.map &:to_na
     end
 
     # duck-type the interface: [:train, :mu, :convergence, :save, :load]
@@ -84,7 +90,7 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
     end
 
     def mu
-      blocks.map(&:mu).reduce(&:hconcat)
+      blocks.map(&:mu).reduce { |mem, var| mem.concatenate var, axis: 1 }
     end
 
     def convergence

data/lib/machine_learning_workbench/optimizer/natural_evolution_strategies/rnes.rb

@@ -5,18 +5,34 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
     attr_reader :variance
 
     def initialize_distribution mu_init: 0, sigma_init: 1
-      @mu = NMatrix.new([1, ndims], mu_init, dtype: dtype)
-      raise ArgumentError unless sigma_init.kind_of? Numeric
+      @mu = case mu_init
+        when Array
+          raise ArgumentError unless mu_init.size == ndims
+          NArray[mu_init]
+        when Numeric
+          NArray.new([1,ndims]).fill mu_init
+        else
+          raise ArgumentError, "Something is wrong with mu_init: #{mu_init}"
+      end
       @variance = sigma_init
-      @sigma = id * variance
+      @sigma = case sigma_init
+      when Array
+        raise ArgumentError "RNES uses single global variance"
+      when Numeric
+        NArray.new([ndims]).fill(variance).diag
+      else
+        raise ArgumentError, "Something is wrong with sigma_init: #{sigma_init}"
+      end
     end
 
     def train picks: sorted_inds
       g_mu = utils.dot(picks)
-      g_sigma = utils.dot(picks.row_norms**2 - ndims).first # back to scalar
+      # g_sigma = utils.dot(picks.row_norms**2 - ndims).first # back to scalar
+      row_norms = Numo::Linalg.norm picks, 2, axis:1
+      g_sigma = utils.dot(row_norms**2 - ndims)[0] # back to scalar
       @mu += sigma.dot(g_mu.transpose).transpose * lrate
       @variance *= Math.exp(g_sigma * lrate / 2)
-      @sigma = id * variance
+      @sigma = NArray.new([ndims]).fill(variance).diag
     end
 
     # Estimate algorithm convergence based on variance
@@ -25,14 +41,14 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
     end
 
     def save
-      [mu.to_consistent_a, variance]
+      [mu.to_a, variance]
     end
 
     def load data
       raise ArgumentError unless data.size == 2
       mu_ary, @variance = data
-      @mu = NMatrix[*mu_ary, dtype: dtype]
-      @sigma = id * variance
+      @mu = mu_ary.to_na
+      @sigma = eye * variance
     end
   end
 end

data/lib/machine_learning_workbench/optimizer/natural_evolution_strategies/snes.rb

@@ -6,35 +6,48 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
     attr_reader :variances
 
     def initialize_distribution mu_init: 0, sigma_init: 1
-      @mu = NMatrix.new([1, ndims], mu_init, dtype: dtype)
-      sigma_init = [sigma_init]*ndims unless sigma_init.kind_of? Enumerable
-      @variances = NMatrix.new([1,ndims], sigma_init, dtype: dtype)
-      @sigma = NMatrix.diagonal(variances, dtype: dtype)
+      @mu = case mu_init
+        when Array
+          raise ArgumentError unless mu_init.size == ndims
+          NArray[mu_init]
+        when Numeric
+          NArray.new([1,ndims]).fill mu_init
+        else
+          raise ArgumentError, "Something is wrong with mu_init: #{mu_init}"
+      end
+      @variances = case sigma_init
+      when Array
+        raise ArgumentError unless sigma_init.size == ndims
+        NArray[*sigma_init]
+      when Numeric
+        NArray.new([ndims]).fill(sigma_init)
+      else
+        raise ArgumentError, "Something is wrong with sigma_init: #{sigma_init}"
+      end
+      @sigma = @variances.diag
     end
 
     def train picks: sorted_inds
       g_mu = utils.dot(picks)
       g_sigma = utils.dot(picks**2 - 1)
       @mu += sigma.dot(g_mu.transpose).transpose * lrate
-      @variances *= (g_sigma * lrate / 2).exponential
-      @sigma = NMatrix.diagonal(variances, dtype: dtype)
+      @variances *= (g_sigma * lrate / 2).exponential.flatten
+      @sigma = @variances.diag
     end
 
     # Estimate algorithm convergence as total variance
     def convergence
-      variances.reduce :+
+      variances.sum
     end
 
     def save
-      [mu.to_consistent_a, variances.to_consistent_a]
+      [mu.to_a, variances.to_a]
     end
 
     def load data
       raise ArgumentError unless data.size == 2
-      mu_ary, variances_ary = data
-      @mu = NMatrix[*mu_ary, dtype: dtype]
-      @variances = NMatrix[*variances_ary, dtype: dtype]
-      @sigma = NMatrix.diagonal(variances, dtype: dtype)
+      @mu, @variances = data.map &:to_na
+      @sigma = variances.diag
     end
   end
 end

data/lib/machine_learning_workbench/optimizer/natural_evolution_strategies/xnes.rb

@@ -5,21 +5,35 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
     attr_reader :log_sigma
 
     def initialize_distribution mu_init: 0, sigma_init: 1
-      @mu = NMatrix.new([1, ndims], mu_init, dtype: dtype)
-      sigma_init = [sigma_init]*ndims unless sigma_init.kind_of? Enumerable
-      @sigma = NMatrix.diag(sigma_init, dtype: dtype)
+      @mu = case mu_init
+        when Array
+          raise ArgumentError unless mu_init.size == ndims
+          NArray[mu_init]
+        when Numeric
+          NArray.new([1,ndims]).fill mu_init
+        else
+          raise ArgumentError, "Something is wrong with mu_init: #{mu_init}"
+      end
+      @sigma = case sigma_init
+      when Array
+        raise ArgumentError unless sigma_init.size == ndims
+        NArray[*sigma_init].diag
+      when Numeric
+        NArray.new([ndims]).fill(sigma_init).diag
+      else
+        raise ArgumentError, "Something is wrong with sigma_init: #{sigma_init}"
+      end
       # Works with the log of sigma to avoid continuous decompositions (thanks Sun Yi)
-      log_sigma_init = sigma_init.map &Math.method(:log)
-      @log_sigma = NMatrix.diag(log_sigma_init, dtype: dtype)
+      @log_sigma = Numo::NMath.log(sigma.diagonal).diag
     end
 
     def train picks: sorted_inds
       g_mu = utils.dot(picks)
-      g_log_sigma = popsize.times.inject(NMatrix.zeros_like sigma) do |sum, i|
+      g_log_sigma = popsize.times.inject(NArray.zeros sigma.shape) do |sum, i|
         u = utils[i]
-        ind = picks.row(i)
+        ind = picks[i, true]
         ind_sq = ind.outer_flat(ind, &:*)
-        sum + (ind_sq - id) * u
+        sum + (ind_sq - eye) * u
       end
       @mu += sigma.dot(g_mu.transpose).transpose * lrate
       @log_sigma += g_log_sigma * (lrate/2)
@@ -32,14 +46,12 @@ module MachineLearningWorkbench::Optimizer::NaturalEvolutionStrategies
     end
 
     def save
-      [mu.to_consistent_a, log_sigma.to_consistent_a]
+      [mu.to_a, log_sigma.to_a]
     end
 
     def load data
       raise ArgumentError unless data.size == 2
-      mu_ary, log_sigma_ary = data
-      @mu = NMatrix[*mu_ary, dtype: dtype]
-      @log_sigma = NMatrix[*log_sigma_ary, dtype: dtype]
+      @mu, @log_sigma = data.map &:to_na
       @sigma = log_sigma.exponential
     end
   end

data/lib/machine_learning_workbench/tools/imaging.rb

@@ -3,26 +3,32 @@ module MachineLearningWorkbench::Tools
     Forkable = MachineLearningWorkbench::Tools::Execution
     Norm = MachineLearningWorkbench::Tools::Normalization
 
-    # Create RMagick::Image from NMatrix data
-    def self.nmat_to_img nmat, shape: nil
-      shape ||= nmat.shape
+    # Create RMagick::Image from numeric matrix data
+    # @param narr [NArray] numeric matrix to display
+    # @param shape [Array<Integer>] optional reshaping
+    def self.narr_to_img narr, shape: nil
+      shape ||= narr.shape
       shape = [1, shape] if shape.kind_of?(Integer) || shape.size == 1
       # `Image::constitute` requires Float pixels to be in [0,1]
-      pixels = Norm.feature_scaling nmat.round(4), to: [0,1]
-      Magick::Image.constitute *shape, "I", pixels.to_flat_a
+      pixels = Norm.feature_scaling narr, to: [0,1]
+      Magick::Image.constitute *shape, "I", pixels.to_a.flatten
     end
 
-    # Create PNG file from NMatrix data
-    def self.nmat_to_png nmat, fname, shape: nil
-      nmat_to_img(nmat, shape: shape).write fname
+    # Create PNG file from numeric matrix data
+    # @param narr [NArray] numeric matrix to display
+    # @param fname [String] path to save PNG
+    # @param shape [Array<Integer>] optional reshaping before saving
+    def self.narr_to_png narr, fname, shape: nil
+      narr_to_img(narr, shape: shape).write fname
     end
 
-    # Show a NMatrix as image in a RMagick window
+    # Show a numeric matrix as image in a RMagick window
+    # @param narr [NArray] numeric matrix to display
     # @param disp_size [Array] the size of the image to display
-    # @param shape [Array] the true shape of the image (NMatrix could be flattened)
+    # @param shape [Array] the true shape of the image (numeric matrix could be flattened)
     # @param in_fork [bool] whether to execute the display in fork (and continue running)
-    def self.display nmat, disp_size: nil, shape: nil, in_fork: true
-      img = nmat_to_img nmat, shape: shape
+    def self.display narr, disp_size: nil, shape: nil, in_fork: true
+      img = narr_to_img narr, shape: shape
       img.resize!(*disp_size, Magick::TriangleFilter,0.51) if disp_size
       if in_fork
         MachineLearningWorkbench::Tools::Execution.in_fork { img.display }
@@ -31,19 +37,19 @@ module MachineLearningWorkbench::Tools
       end
     end
 
-    # Create NMatrix from png by filename.
+    # Create numeric matrix from png by filename.
     # @param fname the file name
     # @param scale optional rescaling of the image
     # @param flat [bool] whether to return a flat array
-    # @param dtype dtype for the NMatrix, leave `nil` for automatic detection
-    def self.nmat_from_png fname, scale: nil, flat: false, dtype: nil
+    # @param dtype dtype for the numeric matrix, leave `nil` for automatic detection
+    def self.narr_from_png fname, scale: nil, flat: false
       img = Magick::ImageList.new(fname).first
       img.scale!(scale) if scale
       shape = [img.columns, img.rows]
       pixels = img.export_pixels(0, 0, *shape, 'I') # 'I' for intensity
       raise "Sanity check" unless shape.reduce(:*)==pixels.size
-      return pixels.to_nm(nil, dtype) if flat
-      NMatrix.new shape, pixels, dtype: dtype
+      return pixels.to_na if flat
+      pixels.to_na.to_dimensions shape
     end
   end
 end

data/lib/machine_learning_workbench/tools/normalization.rb

@@ -1,22 +1,23 @@
 module MachineLearningWorkbench::Tools
   module Normalization
-    def self.feature_scaling nmat, from: nil, to: [0,1]
-      from ||= nmat.minmax
+    def self.feature_scaling narr, from: nil, to: [0,1]
+      from ||= narr.minmax
       old_min, old_max = from
       new_min, new_max = to
-      (nmat-old_min)*(new_max-new_min)/(old_max-old_min)+new_min
+      (narr-old_min)*(new_max-new_min)/(old_max-old_min)+new_min
     end
 
     # @param per_column [bool] wheather to compute stats per-column or matrix-wise
-    def self.z_score nmat, per_column: true
+    def self.z_score narr, per_column: true
       raise NotImplementedError unless per_column
-      means = nmat.mean
-      stddevs = nmat.std
+      raise "this would be a good time to test this implementation"
+      means = narr.mean
+      stddevs = narr.std
       # address edge case of zero variance
       stddevs.map! { |v| v.zero? ? 1 : v }
-      mean_mat = means.repeat nmat.rows, 0
-      stddev_mat = stddevs.repeat nmat.rows, 0
-      (nmat - mean_mat) / stddev_mat
+      mean_mat = means.repeat narr.rows, 0
+      stddev_mat = stddevs.repeat narr.rows, 0
+      (narr - mean_mat) / stddev_mat
     end
   end
 end

data/machine_learning_workbench.gemspec

@@ -9,8 +9,7 @@ Gem::Specification.new do |spec|
   spec.email         = "giuseppe.cuccu@gmail.com"
 
   spec.summary       = %q[Workbench for practical machine learning in Ruby.]
-  spec.description   = %q[\
-    This workbench holds a collection of machine learning
+  spec.description   = %q[This workbench holds a collection of machine learning
     methods in Ruby. Rather than specializing on a single task or method, this
     gem aims at providing an encompassing framework for any machine learning
     application.].gsub('  ', '')
@@ -44,7 +43,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-stack_explorer", "~> 0.4"
 
   # Run
-  spec.requirements << "libatlas-base-dev"  # library for following dependency
-  spec.add_dependency "nmatrix-atlas", "~> 0.2"
+  spec.requirements << "libopenblas-base"  # library for following dependency
+  spec.add_dependency "numo-linalg"
   spec.add_dependency "parallel", "~> 1.12"
+
+
+
+  # DELETEME
+  spec.add_dependency "nmatrix-atlas"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: machine_learning_workbench
 version: !ruby/object:Gem::Version
-  version: '0.3'
+  version: 0.4.0
 platform: ruby
 authors:
 - Giuseppe Cuccu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-19 00:00:00.000000000 Z
+date: 2018-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -123,19 +123,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.4'
 - !ruby/object:Gem::Dependency
-  name: nmatrix-atlas
+  name: numo-linalg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -150,8 +150,21 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: nmatrix-atlas
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |-
-  \
   This workbench holds a collection of machine learning
   methods in Ruby. Rather than specializing on a single task or method, this
   gem aims at providing an encompassing framework for any machine learning
@@ -219,7 +232,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- libatlas-base-dev
+- libopenblas-base
 rubyforge_project: 
 rubygems_version: 2.6.13
 signing_key: