ruby-spark 1.0.0

data/lib/spark/mllib/classification/naive_bayes.rb

@@ -0,0 +1,97 @@
+module Spark
+  module Mllib
+    ##
+    # NaiveBayesModel
+    #
+    # Model for Naive Bayes classifiers.
+    #
+    # Contains two parameters:
+    # pi:: vector of logs of class priors (dimension C)
+    # theta:: matrix of logs of class conditional probabilities (CxD)
+    #
+    # == Examples:
+    #
+    #   Spark::Mllib.import
+    #
+    #   # Dense vectors
+    #   data = [
+    #     LabeledPoint.new(0.0, [0.0, 0.0]),
+    #     LabeledPoint.new(0.0, [0.0, 1.0]),
+    #     LabeledPoint.new(1.0, [1.0, 0.0])
+    #   ]
+    #   model = NaiveBayes.train($sc.parallelize(data))
+    #
+    #   model.predict([0.0, 1.0])
+    #   # => 0.0
+    #   model.predict([1.0, 0.0])
+    #   # => 1.0
+    #
+    #
+    #   # Sparse vectors
+    #   data = [
+    #     LabeledPoint.new(0.0, SparseVector.new(2, {1 => 0.0})),
+    #     LabeledPoint.new(0.0, SparseVector.new(2, {1 => 1.0})),
+    #     LabeledPoint.new(1.0, SparseVector.new(2, {0 => 1.0}))
+    #   ]
+    #   model = NaiveBayes.train($sc.parallelize(data))
+    #
+    #   model.predict(SparseVector.new(2, {1 => 1.0}))
+    #   # => 0.0
+    #   model.predict(SparseVector.new(2, {0 => 1.0}))
+    #   # => 1.0
+    #
+    class NaiveBayesModel
+
+      attr_reader :labels, :pi, :theta
+
+      def initialize(labels, pi, theta)
+        @labels = labels
+        @pi = pi
+        @theta = theta
+      end
+
+      # Predict values for a single data point or an RDD of points using
+      # the model trained.
+      def predict(vector)
+        vector = Spark::Mllib::Vectors.to_vector(vector)
+        array = (vector.dot(theta) + pi).to_a
+        index = array.index(array.max)
+        labels[index]
+      end
+
+    end
+  end
+end
+
+
+module Spark
+  module Mllib
+    class NaiveBayes
+
+      # Trains a Naive Bayes model given an RDD of (label, features) pairs.
+      #
+      # This is the Multinomial NB (http://tinyurl.com/lsdw6p) which can handle all kinds of
+      # discrete data.  For example, by converting documents into TF-IDF vectors, it can be used for
+      # document classification.  By making every vector a 0-1 vector, it can also be used as
+      # Bernoulli NB (http://tinyurl.com/p7c96j6). The input feature values must be nonnegative.
+      #
+      # == Arguments:
+      # rdd:: RDD of LabeledPoint.
+      # lambda:: The smoothing parameter.
+      #
+      def self.train(rdd, lambda=1.0)
+        # Validation
+        first = rdd.first
+        unless first.is_a?(LabeledPoint)
+          raise Spark::MllibError, "RDD should contains LabeledPoint, got #{first.class}"
+        end
+
+        labels, pi, theta = Spark.jb.call(RubyMLLibAPI.new, 'trainNaiveBayes', rdd, lambda)
+        theta = Spark::Mllib::Matrices.dense(theta.size, theta.first.size, theta)
+
+        NaiveBayesModel.new(labels, pi, theta)
+      end
+
+    end
+  end
+end

data/lib/spark/mllib/classification/svm.rb

@@ -0,0 +1,135 @@
+module Spark
+  module Mllib
+    ##
+    # SVMModel
+    #
+    # A support vector machine.
+    #
+    # == Examples:
+    #
+    #   Spark::Mllib.import
+    #
+    #   # Dense vectors
+    #   data = [
+    #       LabeledPoint.new(0.0, [0.0]),
+    #       LabeledPoint.new(1.0, [1.0]),
+    #       LabeledPoint.new(1.0, [2.0]),
+    #       LabeledPoint.new(1.0, [3.0])
+    #   ]
+    #   svm = SVMWithSGD.train($sc.parallelize(data))
+    #
+    #   svm.predict([1.0])
+    #   # => 1
+    #   svm.clear_threshold
+    #   svm.predict([1.0])
+    #   # => 1.25...
+    #
+    #
+    #   # Sparse vectors
+    #   data = [
+    #       LabeledPoint.new(0.0, SparseVector.new(2, {0 => -1.0})),
+    #       LabeledPoint.new(1.0, SparseVector.new(2, {1 => 1.0})),
+    #       LabeledPoint.new(0.0, SparseVector.new(2, {0 => 0.0})),
+    #       LabeledPoint.new(1.0, SparseVector.new(2, {1 => 2.0}))
+    #   ]
+    #   svm = SVMWithSGD.train($sc.parallelize(data))
+    #
+    #   svm.predict(SparseVector.new(2, {1 => 1.0}))
+    #   # => 1
+    #   svm.predict(SparseVector.new(2, {0 => -1.0}))
+    #   # => 0
+    #
+    class SVMModel < ClassificationModel
+
+      def initialize(*args)
+        super
+        @threshold = 0.0
+      end
+
+      # Predict values for a single data point or an RDD of points using
+      # the model trained.
+      def predict(vector)
+        vector = Spark::Mllib::Vectors.to_vector(vector)
+        margin = weights.dot(vector) + intercept
+
+        if threshold.nil?
+          return margin
+        end
+
+        if margin > threshold
+          1
+        else
+          0
+        end
+      end
+
+    end
+  end
+end
+
+module Spark
+  module Mllib
+    class SVMWithSGD < ClassificationMethodBase
+
+      DEFAULT_OPTIONS = {
+        iterations: 100,
+        step: 1.0,
+        reg_param: 0.01,
+        mini_batch_fraction: 1.0,
+        initial_weights: nil,
+        reg_type: 'l2',
+        intercept: false
+      }
+
+      # Train a support vector machine on the given data.
+      #
+      # rdd::
+      #   The training data, an RDD of LabeledPoint.
+      #
+      # iterations::
+      #   The number of iterations (default: 100).
+      #
+      # step::
+      #   The step parameter used in SGD (default: 1.0).
+      #
+      # reg_param::
+      #   The regularizer parameter (default: 0.01).
+      #
+      # mini_batch_fraction::
+      #   Fraction of data to be used for each SGD iteration.
+      #
+      # initial_weights::
+      #   The initial weights (default: nil).
+      #
+      # reg_type::
+      #   The type of regularizer used for training our model (default: "l2").
+      #
+      #   Allowed values:
+      #   - "l1" for using L1 regularization
+      #   - "l2" for using L2 regularization
+      #   - nil for no regularization
+      #
+      # intercept::
+      #   Boolean parameter which indicates the use
+      #   or not of the augmented representation for
+      #   training data (i.e. whether bias features
+      #   are activated or not).
+      #
+      def self.train(rdd, options={})
+        super
+
+        weights, intercept = Spark.jb.call(RubyMLLibAPI.new, 'trainSVMModelWithSGD', rdd,
+                                           options[:iterations].to_i,
+                                           options[:step].to_f,
+                                           options[:reg_param].to_f,
+                                           options[:mini_batch_fraction].to_f,
+                                           options[:initial_weights],
+                                           options[:reg_type],
+                                           options[:intercept])
+
+        SVMModel.new(weights, intercept)
+      end
+
+    end
+  end
+end

data/lib/spark/mllib/clustering/gaussian_mixture.rb

@@ -0,0 +1,82 @@
+module Spark
+  module Mllib
+    ##
+    # GaussianMixtureModel
+    #
+    # A clustering model derived from the Gaussian Mixture Model method.
+    #
+    # == Examples:
+    #
+    #   Spark::Mllib.import
+    #
+    #   data = [
+    #     DenseVector.new([-0.1, -0.05]),
+    #     DenseVector.new([-0.01, -0.1]),
+    #     DenseVector.new([0.9, 0.8]),
+    #     DenseVector.new([0.75, 0.935]),
+    #     DenseVector.new([-0.83, -0.68]),
+    #     DenseVector.new([-0.91, -0.76])
+    #   ]
+    #
+    #   model = GaussianMixture.train($sc.parallelize(data), 3, convergence_tol: 0.0001, max_iterations: 50, seed: 10)
+    #
+    #   labels = model.predict($sc.parallelize(data)).collect
+    #
+    class GaussianMixtureModel
+
+      attr_reader :weights, :gaussians, :k
+
+      def initialize(weights, gaussians)
+        @weights = weights
+        @gaussians = gaussians
+        @k = weights.size
+      end
+
+      # Find the cluster to which the points in 'x' has maximum membership
+      # in this model.
+      def predict(rdd)
+        if rdd.is_a?(Spark::RDD)
+          predict_soft(rdd).map('lambda{|x| x.index(x.max)}')
+        else
+          raise ArgumentError, 'Argument must be a RDD.'
+        end
+      end
+
+      # Find the membership of each point in 'x' to all mixture components.
+      def predict_soft(rdd)
+        Spark.jb.call(RubyMLLibAPI.new, 'predictSoftGMM', rdd, weights, means, sigmas)
+      end
+
+      def means
+        @means ||= @gaussians.map(&:mu)
+      end
+
+      def sigmas
+        @sigmas ||= @gaussians.map(&:sigma)
+      end
+
+    end
+  end
+end
+
+module Spark
+  module Mllib
+    class GaussianMixture
+
+      def self.train(rdd, k, convergence_tol: 0.001, max_iterations: 100, seed: nil)
+        weights, means, sigmas = Spark.jb.call(RubyMLLibAPI.new, 'trainGaussianMixture', rdd,
+                                               k, convergence_tol, max_iterations, Spark.jb.to_long(seed))
+
+        means.map! {|mu|    Spark.jb.java_to_ruby(mu)}
+        sigmas.map!{|sigma| Spark.jb.java_to_ruby(sigma)}
+
+        mvgs = Array.new(k) do |i|
+          MultivariateGaussian.new(means[i], sigmas[i])
+        end
+
+        GaussianMixtureModel.new(weights, mvgs)
+      end
+
+    end
+  end
+end

data/lib/spark/mllib/clustering/kmeans.rb

@@ -0,0 +1,118 @@
+module Spark
+  module Mllib
+    ##
+    # KMeansModel
+    #
+    # A clustering model derived from the k-means method.
+    #
+    # == Examples:
+    #
+    #   Spark::Mllib.import
+    #
+    #   # Dense vectors
+    #   data = [
+    #     DenseVector.new([0.0,0.0]),
+    #     DenseVector.new([1.0,1.0]),
+    #     DenseVector.new([9.0,8.0]),
+    #     DenseVector.new([8.0,9.0])
+    #   ]
+    #
+    #   model = KMeans.train($sc.parallelize(data), 2, max_iterations: 10,
+    #                        runs: 30, initialization_mode: "random")
+    #
+    #   model.predict([0.0, 0.0]) == model.predict([1.0, 1.0])
+    #   # => true
+    #   model.predict([8.0, 9.0]) == model.predict([9.0, 8.0])
+    #   # => true
+    #
+    #
+    #   # Sparse vectors
+    #   data = [
+    #       SparseVector.new(3, {1 => 1.0}),
+    #       SparseVector.new(3, {1 => 1.1}),
+    #       SparseVector.new(3, {2 => 1.0}),
+    #       SparseVector.new(3, {2 => 1.1})
+    #   ]
+    #   model = KMeans.train($sc.parallelize(data), 2, initialization_mode: "k-means||")
+    #
+    #   model.predict([0.0, 1.0, 0.0]) == model.predict([0, 1.1, 0.0])
+    #   # => true
+    #   model.predict([0.0, 0.0, 1.0]) == model.predict([0, 0, 1.1])
+    #   # => true
+    #   model.predict(data[0]) == model.predict(data[1])
+    #   # => true
+    #   model.predict(data[2]) == model.predict(data[3])
+    #   # => true
+    #
+    class KMeansModel
+
+      attr_reader :centers
+
+      def initialize(centers)
+        @centers = centers
+      end
+
+      # Find the cluster to which x belongs in this model.
+      def predict(vector)
+        vector = Spark::Mllib::Vectors.to_vector(vector)
+        best = 0
+        best_distance = Float::INFINITY
+
+        @centers.each_with_index do |center, index|
+          distance = vector.squared_distance(center)
+          if distance < best_distance
+            best = index
+            best_distance = distance
+          end
+        end
+
+        best
+      end
+
+      def self.from_java(object)
+        centers = object.clusterCenters
+        centers.map! do |center|
+          Spark.jb.java_to_ruby(center)
+        end
+
+        KMeansModel.new(centers)
+      end
+
+    end
+  end
+end
+
+module Spark
+  module Mllib
+    class KMeans
+
+      # Trains a k-means model using the given set of parameters.
+      #
+      # == Arguments:
+      # rdd::
+      #   The training data, an RDD of Vectors.
+      #
+      # k::
+      #   Number of clusters.
+      #
+      # max_iterations::
+      #   Max number of iterations.
+      #
+      # runs::
+      #   Number of parallel runs, defaults to 1. The best model is returned.
+      #
+      # initialization_mode::
+      #   Initialization model, either "random" or "k-means||" (default).
+      #
+      # seed::
+      #   Random seed value for cluster initialization.
+      #
+      def self.train(rdd, k, max_iterations: 100, runs: 1, initialization_mode: 'k-means||', seed: nil)
+        # Call returns KMeansModel
+        Spark.jb.call(RubyMLLibAPI.new, 'trainKMeansModel', rdd,
+                      k, max_iterations, runs, initialization_mode, Spark.jb.to_long(seed))
+      end
+
+    end
+  end
+end

data/lib/spark/mllib/matrix.rb

@@ -0,0 +1,120 @@
+module Spark
+  module Mllib
+    module Matrices
+
+      def self.dense(*args)
+        DenseMatrix.new(*args)
+      end
+
+      def self.sparse(*args)
+        SparseMatrix.new(*args)
+      end
+
+      def self.to_matrix(data)
+        if data.is_a?(SparseMatrix) || data.is_a?(DenseMatrix)
+          data
+        elsif data.is_a?(Array)
+          DenseMatrix.new(data)
+        end
+      end
+
+    end
+  end
+end
+
+module Spark
+  module Mllib
+    # @abstract Parent for all type of matrices
+    class MatrixBase < MatrixAdapter
+    end
+  end
+end
+
+module Spark
+  module Mllib
+    ##
+    # DenseMatrix
+    #
+    #   DenseMatrix.new(2, 3, [[1,2,3], [4,5,6]]).values
+    #   # => [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
+    #
+    class DenseMatrix < MatrixBase
+
+      def initialize(rows, cols, values)
+        super(:dense, rows, cols, values.to_a)
+      end
+
+      def to_java
+        JDenseMatrix.new(shape[0], shape[1], values.flatten)
+      end
+
+      def self.from_java(object)
+        rows = object.numRows
+        cols = object.numCols
+        values = object.values
+
+        DenseMatrix.new(rows, cols, values)
+      end
+
+    end
+  end
+end
+
+module Spark
+  module Mllib
+    ##
+    # SparseMatrix
+    #
+    # == Arguments:
+    # rows::
+    #   Number of rows.
+    #
+    # cols::
+    #   Number of columns.
+    #
+    # col_pointers::
+    #   The index corresponding to the start of a new column.
+    #
+    # row_indices::
+    #   The row index of the entry. They must be in strictly
+    #   increasing order for each column.
+    #
+    # values::
+    #   Nonzero matrix entries in column major.
+    #
+    # == Examples:
+    #
+    #   SparseMatrix.new(3, 3, [0, 2, 3, 6], [0, 2, 1, 0, 1, 2], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).values
+    #
+    #   # => [
+    #   #      [1.0, 0.0, 4.0],
+    #   #      [0.0, 3.0, 5.0],
+    #   #      [2.0, 0.0, 6.0]
+    #   #    ]
+    #
+    class SparseMatrix < MatrixBase
+
+      attr_reader :col_pointers, :row_indices
+
+      def initialize(rows, cols, col_pointers, row_indices, values)
+        super(:sparse, rows, cols)
+
+        @col_pointers = col_pointers
+        @row_indices = row_indices
+        @values = values
+
+        j = 0
+        while j < cols
+          idx = col_pointers[j]
+          idx_end = col_pointers[j+1]
+          while idx < idx_end
+            self[row_indices[idx], j] = values[idx]
+            idx += 1
+          end
+          j += 1
+        end
+      end
+
+    end
+  end
+end