ruby-spark 1.0.0

data/lib/spark/java_bridge/rjb.rb

@@ -0,0 +1,41 @@
+if !ENV.has_key?('JAVA_HOME')
+  raise Spark::ConfigurationError, 'Environment variable JAVA_HOME is not set'
+end
+
+require 'rjb'
+
+module Spark
+  module JavaBridge
+    class RJB < Base
+
+      def initialize(*args)
+        super
+        Rjb.load(jars)
+        Rjb.primitive_conversion = true
+      end
+
+      def import(name, klass)
+        Object.const_set(name, silence_warnings { Rjb.import(klass) })
+      end
+
+      def java_object?(object)
+        object.is_a?(Rjb::Rjb_JavaProxy)
+      end
+
+      private
+
+        def jars
+          separator = windows? ? ';' : ':'
+          super.join(separator)
+        end
+
+        def silence_warnings
+          old_verbose, $VERBOSE = $VERBOSE, nil
+          yield
+        ensure
+          $VERBOSE = old_verbose
+        end
+
+    end
+  end
+end

data/lib/spark/logger.rb

@@ -0,0 +1,76 @@
+# Necessary libraries
+Spark.load_lib
+
+module Spark
+  class Logger
+
+    attr_reader :jlogger
+
+    def initialize
+      @jlogger = JLogger.getLogger('Ruby')
+    end
+
+    def level_off
+      JLevel.toLevel('OFF')
+    end
+
+    # Disable all Spark log
+    def disable
+      jlogger.setLevel(level_off)
+      JLogger.getLogger('org').setLevel(level_off)
+      JLogger.getLogger('akka').setLevel(level_off)
+      JLogger.getRootLogger.setLevel(level_off)
+    end
+
+    def enabled?
+      !disabled?
+    end
+
+    def info(message)
+      jlogger.info(message) if info?
+    end
+
+    def debug(message)
+      jlogger.debug(message) if debug?
+    end
+
+    def trace(message)
+      jlogger.trace(message) if trace?
+    end
+
+    def warning(message)
+      jlogger.warn(message) if warning?
+    end
+
+    def error(message)
+      jlogger.error(message) if error?
+    end
+
+    def info?
+      level_enabled?('info')
+    end
+
+    def debug?
+      level_enabled?('debug')
+    end
+
+    def trace?
+      level_enabled?('trace')
+    end
+
+    def warning?
+      level_enabled?('warn')
+    end
+
+    def error?
+      level_enabled?('error')
+    end
+
+    def level_enabled?(type)
+      jlogger.isEnabledFor(JPriority.toPriority(type.upcase))
+    end
+
+    alias_method :warn, :warning
+
+  end
+end

data/lib/spark/mllib.rb

@@ -0,0 +1,100 @@
+module Spark
+  # MLlib is Spark’s scalable machine learning library consisting of common learning algorithms and utilities,
+  # including classification, regression, clustering, collaborative filtering, dimensionality reduction,
+  # as well as underlying optimization primitives.
+  module Mllib
+
+    def self.autoload(klass, location, import=true)
+      if import
+        @for_importing ||= []
+        @for_importing << klass
+      end
+
+      super(klass, location)
+    end
+
+    def self.autoload_without_import(klass, location)
+      autoload(klass, location, false)
+    end
+
+    # Base classes
+    autoload_without_import :VectorBase, 'spark/mllib/vector'
+    autoload_without_import :MatrixBase, 'spark/mllib/matrix'
+    autoload_without_import :RegressionMethodBase,     'spark/mllib/regression/common'
+    autoload_without_import :ClassificationMethodBase, 'spark/mllib/classification/common'
+
+    # Linear algebra
+    autoload :Vectors,      'spark/mllib/vector'
+    autoload :DenseVector,  'spark/mllib/vector'
+    autoload :SparseVector, 'spark/mllib/vector'
+    autoload :Matrices,     'spark/mllib/matrix'
+    autoload :DenseMatrix,  'spark/mllib/matrix'
+    autoload :SparseMatrix, 'spark/mllib/matrix'
+
+    # Regression
+    autoload :LabeledPoint,            'spark/mllib/regression/labeled_point'
+    autoload :RegressionModel,         'spark/mllib/regression/common'
+    autoload :LinearRegressionModel,   'spark/mllib/regression/linear'
+    autoload :LinearRegressionWithSGD, 'spark/mllib/regression/linear'
+    autoload :LassoModel,              'spark/mllib/regression/lasso'
+    autoload :LassoWithSGD,            'spark/mllib/regression/lasso'
+    autoload :RidgeRegressionModel,    'spark/mllib/regression/ridge'
+    autoload :RidgeRegressionWithSGD,  'spark/mllib/regression/ridge'
+
+    # Classification
+    autoload :ClassificationModel,         'spark/mllib/classification/common'
+    autoload :LogisticRegressionWithSGD,   'spark/mllib/classification/logistic_regression'
+    autoload :LogisticRegressionWithLBFGS, 'spark/mllib/classification/logistic_regression'
+    autoload :SVMModel,                    'spark/mllib/classification/svm'
+    autoload :SVMWithSGD,                  'spark/mllib/classification/svm'
+    autoload :NaiveBayesModel,             'spark/mllib/classification/naive_bayes'
+    autoload :NaiveBayes,                  'spark/mllib/classification/naive_bayes'
+
+    # Clustering
+    autoload :KMeans,               'spark/mllib/clustering/kmeans'
+    autoload :KMeansModel,          'spark/mllib/clustering/kmeans'
+    autoload :GaussianMixture,      'spark/mllib/clustering/gaussian_mixture'
+    autoload :GaussianMixtureModel, 'spark/mllib/clustering/gaussian_mixture'
+
+    # Stat
+    autoload :MultivariateGaussian, 'spark/mllib/stat/distribution'
+
+    def self.prepare
+      return if @prepared
+
+      # if narray?
+      #   require 'spark/mllib/narray/vector'
+      #   require 'spark/mllib/narray/matrix'
+      # elsif mdarray?
+      #   require 'spark/mllib/mdarray/vector'
+      #   require 'spark/mllib/mdarray/matrix'
+      # else
+      #   require 'spark/mllib/matrix/vector'
+      #   require 'spark/mllib/matrix/matrix'
+      # end
+
+      require 'spark/mllib/ruby_matrix/vector_adapter'
+      require 'spark/mllib/ruby_matrix/matrix_adapter'
+
+      @prepared = true
+      nil
+    end
+
+    def self.import(to=Object)
+      @for_importing.each do |klass|
+        to.const_set(klass, const_get(klass))
+      end
+      nil
+    end
+
+    def self.narray?
+      Gem::Specification::find_all_by_name('narray').any?
+    end
+
+    def self.mdarray?
+      Gem::Specification::find_all_by_name('mdarray').any?
+    end
+  end
+end
+
+Spark::Mllib.prepare

data/lib/spark/mllib/classification/common.rb

@@ -0,0 +1,31 @@
+module Spark
+  module Mllib
+    class ClassificationModel
+
+      attr_reader :weights, :intercept, :threshold
+
+      def initialize(weights, intercept)
+        @weights = Spark::Mllib::Vectors.to_vector(weights)
+        @intercept = intercept.to_f
+        @threshold = nil
+      end
+
+      def threshold=(value)
+        @threshold = value.to_f
+      end
+
+      def clear_threshold
+        @threshold = nil
+      end
+
+    end
+  end
+end
+
+module Spark
+  module Mllib
+    class ClassificationMethodBase < RegressionMethodBase
+
+    end
+  end
+end

data/lib/spark/mllib/classification/logistic_regression.rb

@@ -0,0 +1,223 @@
+module Spark
+  module Mllib
+    ##
+    # LogisticRegressionModel
+    #
+    # A linear binary classification model derived from logistic regression.
+    #
+    # == Examples:
+    #
+    #   Spark::Mllib.import
+    #
+    #   # Dense vectors
+    #   data = [
+    #     LabeledPoint.new(0.0, [0.0, 1.0]),
+    #     LabeledPoint.new(1.0, [1.0, 0.0]),
+    #   ]
+    #   lrm = LogisticRegressionWithSGD.train($sc.parallelize(data))
+    #
+    #   lrm.predict([1.0, 0.0])
+    #   # => 1
+    #   lrm.predict([0.0, 1.0])
+    #   # => 0
+    #
+    #   lrm.clear_threshold
+    #   lrm.predict([0.0, 1.0])
+    #   # => 0.123...
+    #
+    #
+    #   # Sparse vectors
+    #   data = [
+    #     LabeledPoint.new(0.0, SparseVector.new(2, {0 => 0.0})),
+    #     LabeledPoint.new(1.0, SparseVector.new(2, {1 => 1.0})),
+    #     LabeledPoint.new(0.0, SparseVector.new(2, {0 => 1.0})),
+    #     LabeledPoint.new(1.0, SparseVector.new(2, {1 => 2.0}))
+    #   ]
+    #   lrm = LogisticRegressionWithSGD.train($sc.parallelize(data))
+    #
+    #   lrm.predict([0.0, 1.0])
+    #   # => 1
+    #   lrm.predict([1.0, 0.0])
+    #   # => 0
+    #   lrm.predict(SparseVector.new(2, {1 => 1.0}))
+    #   # => 1
+    #   lrm.predict(SparseVector.new(2, {0 => 1.0}))
+    #   # => 0
+    #
+    #
+    #   # LogisticRegressionWithLBFGS
+    #   data = [
+    #     LabeledPoint.new(0.0, [0.0, 1.0]),
+    #     LabeledPoint.new(1.0, [1.0, 0.0]),
+    #   ]
+    #   lrm = LogisticRegressionWithLBFGS.train($sc.parallelize(data))
+    #
+    #   lrm.predict([1.0, 0.0])
+    #   # => 1
+    #   lrm.predict([0.0, 1.0])
+    #   # => 0
+    #
+    class LogisticRegressionModel < ClassificationModel
+
+      def initialize(*args)
+        super
+        @threshold = 0.5
+      end
+
+      # Predict values for a single data point or an RDD of points using
+      # the model trained.
+      def predict(vector)
+        vector = Spark::Mllib::Vectors.to_vector(vector)
+        margin = weights.dot(vector) + intercept
+        score = 1.0 / (1.0 + Math.exp(-margin))
+
+        if threshold.nil?
+          return score
+        end
+
+        if score > threshold
+          1
+        else
+          0
+        end
+      end
+
+    end
+  end
+end
+
+module Spark
+  module Mllib
+    class LogisticRegressionWithSGD < ClassificationMethodBase
+
+      DEFAULT_OPTIONS = {
+        iterations: 100,
+        step: 1.0,
+        mini_batch_fraction: 1.0,
+        initial_weights: nil,
+        reg_param: 0.01,
+        reg_type: 'l2',
+        intercept: false
+      }
+
+      # Train a logistic regression model on the given data.
+      #
+      # == Arguments:
+      # rdd::
+      #   The training data, an RDD of LabeledPoint.
+      #
+      # iterations::
+      #   The number of iterations (default: 100).
+      #
+      # step::
+      #   The step parameter used in SGD (default: 1.0).
+      #
+      # mini_batch_fraction::
+      #   Fraction of data to be used for each SGD iteration.
+      #
+      # initial_weights::
+      #   The initial weights (default: nil).
+      #
+      # reg_param::
+      #   The regularizer parameter (default: 0.01).
+      #
+      # reg_type::
+      #   The type of regularizer used for training our model (default: "l2").
+      #
+      #   Allowed values:
+      #   - "l1" for using L1 regularization
+      #   - "l2" for using L2 regularization
+      #   - nil for no regularization
+      #
+      # intercept::
+      #   Boolean parameter which indicates the use
+      #   or not of the augmented representation for
+      #   training data (i.e. whether bias features
+      #   are activated or not).
+      #
+      def self.train(rdd, options={})
+        super
+
+        weights, intercept = Spark.jb.call(RubyMLLibAPI.new, 'trainLogisticRegressionModelWithSGD', rdd,
+                                           options[:iterations].to_i,
+                                           options[:step].to_f,
+                                           options[:mini_batch_fraction].to_f,
+                                           options[:initial_weights],
+                                           options[:reg_param].to_f,
+                                           options[:reg_type],
+                                           options[:intercept])
+
+        LogisticRegressionModel.new(weights, intercept)
+      end
+
+    end
+  end
+end
+
+module Spark
+  module Mllib
+    class LogisticRegressionWithLBFGS < ClassificationMethodBase
+
+      DEFAULT_OPTIONS = {
+        iterations: 100,
+        initial_weights: nil,
+        reg_param: 0.01,
+        reg_type: 'l2',
+        intercept: false,
+        corrections: 10,
+        tolerance: 0.0001
+      }
+
+      # Train a logistic regression model on the given data.
+      #
+      # == Arguments:
+      # rdd::
+      #   The training data, an RDD of LabeledPoint.
+      #
+      # iterations::
+      #   The number of iterations (default: 100).
+      #
+      # initial_weights::
+      #   The initial weights (default: nil).
+      #
+      # reg_param::
+      #   The regularizer parameter (default: 0.01).
+      #
+      # reg_type::
+      #   The type of regularizer used for training our model (default: "l2").
+      #
+      #   Allowed values:
+      #   - "l1" for using L1 regularization
+      #   - "l2" for using L2 regularization
+      #   - nil for no regularization
+      #
+      # intercept::
+      #   Boolean parameter which indicates the use
+      #   or not of the augmented representation for
+      #   training data (i.e. whether bias features
+      #   are activated or not).
+      #
+      # corrections::
+      #   The number of corrections used in the LBFGS update (default: 10).
+      #
+      # tolerance::
+      #   The convergence tolerance of iterations for L-BFGS (default: 0.0001).
+      #
+      def self.train(rdd, options={})
+        super
+
+        weights, intercept = Spark.jb.call(RubyMLLibAPI.new, 'trainLogisticRegressionModelWithLBFGS', rdd,
+                                           options[:iterations].to_i,
+                                           options[:initial_weights],
+                                           options[:reg_param].to_f,
+                                           options[:reg_type],
+                                           options[:intercept],
+                                           options[:corrections].to_i,
+                                           options[:tolerance].to_f)
+
+        LogisticRegressionModel.new(weights, intercept)
+      end
+
+    end
+  end
+end