ruby-spark 1.1.0.1 → 1.2.0

data/lib/spark/context.rb

@@ -38,6 +38,12 @@ module Spark
       set_call_site('Ruby') # description of stage
     end
 
+    def inspect
+      result  = %{#<#{self.class.name}:0x#{object_id}\n}
+      result << %{Tempdir: "#{temp_dir}">}
+      result
+    end
+
     def stop
       Spark::Accumulator::Server.stop
       log_info('Ruby accumulator server was stopped')
@@ -108,14 +114,11 @@ module Spark
     # Support function for API backtraces.
     #
     def set_call_site(site)
-      set_local_property('externalCallSite', site)
+      jcontext.setCallSite(site)
     end
 
-    # Capture the current user callsite and return a formatted version for printing. If the user
-    # has overridden the call site, this will return the user's version.
-    #
-    def get_call_site
-      jcontext.getCallSite
+    def clear_call_site
+      jcontext.clearCallSite
     end
 
     # Return a copy of this SparkContext's configuration. The configuration *cannot*
@@ -313,7 +316,7 @@ module Spark
     alias_method :setLocalProperty, :set_local_property
     alias_method :getLocalProperty, :get_local_property
     alias_method :setCallSite, :set_call_site
-    alias_method :getCallSite, :get_call_site
+    alias_method :clearCallSite, :clear_call_site
     alias_method :runJob, :run_job
     alias_method :runJobWithCommand, :run_job_with_command
     alias_method :addFile, :add_file

data/lib/spark/error.rb

@@ -47,4 +47,8 @@ module Spark
   # Wrong instances
   class MllibError < StandardError
   end
+
+  # Missing Java class
+  class JavaBridgeError < StandardError
+  end
 end

data/lib/spark/helper/statistic.rb

@@ -77,9 +77,13 @@ module Spark
       # == Example:
       #   data = [0,1,2,3,4,5,6,7,8,9,10]
       #   determine_bounds(data, 3)
-      #   # => [2, 5, 8]
+      #   # => [3, 7]
       #
       def determine_bounds(data, num_partitions)
+        if num_partitions > data.size
+          return data
+        end
+
         bounds = []
         count = data.size
         (0...(num_partitions-1)).each do |index|

data/lib/spark/java_bridge.rb

@@ -7,12 +7,14 @@ module Spark
 
     include Spark::Helper::System
 
-    def self.get
+    def self.init(*args)
       if jruby?
-        JRuby
+        klass = JRuby
       else
-        RJB
+        klass = RJB
       end
+
+      klass.new(*args)
     end
 
   end

data/lib/spark/java_bridge/base.rb

@@ -41,31 +41,31 @@ module Spark
 
       RUBY_TO_JAVA_SKIP = [Fixnum, Integer]
 
-      def initialize(spark_home)
-        @spark_home = spark_home
+      def initialize(target)
+        @target = target
       end
 
       # Import all important classes into Objects
-      def load
-        return if @loaded
+      def import_all
+        return if @imported
 
         java_objects.each do |name, klass|
           import(name, klass)
         end
 
-        @loaded = true
+        @imported = true
         nil
       end
 
       # Import classes for testing
-      def load_test
-        return if @loaded_test
+      def import_all_test
+        return if @imported_test
 
         java_test_objects.each do |name, klass|
           import(name, klass)
         end
 
-        @loaded_test = true
+        @imported_test = true
         nil
       end
 
@@ -168,13 +168,9 @@ module Spark
       private
 
         def jars
-          result = []
-          if File.file?(@spark_home)
-            result << @spark_home
-          else
-            result << Dir.glob(File.join(@spark_home, '*.jar'))
-          end
-          result.flatten
+          result = Dir.glob(File.join(@target, '*.jar'))
+          result.flatten!
+          result
         end
 
         def objects_with_names(objects)
@@ -198,6 +194,10 @@ module Spark
           objects_with_names(JAVA_TEST_OBJECTS)
         end
 
+        def raise_missing_class(klass)
+          raise Spark::JavaBridgeError, "Class #{klass} is missing. Make sure that Spark and RubySpark is assembled."
+        end
+
     end
   end
 end

data/lib/spark/java_bridge/jruby.rb

@@ -11,7 +11,9 @@ module Spark
 
       def import(name, klass)
         klass = "Java::#{klass}"
-        Object.const_set(name, eval(klass)) rescue nil
+        Object.const_set(name, eval(klass))
+      rescue NameError
+        raise_missing_class(klass)
       end
 
       def java_object?(object)

data/lib/spark/java_bridge/rjb.rb

@@ -16,6 +16,8 @@ module Spark
 
       def import(name, klass)
         Object.const_set(name, silence_warnings { Rjb.import(klass) })
+      rescue NoClassDefFoundError
+        raise_missing_class(klass)
       end
 
       def java_object?(object)

data/lib/spark/mllib/classification/logistic_regression.rb

@@ -97,7 +97,8 @@ module Spark
         initial_weights: nil,
         reg_param: 0.01,
         reg_type: 'l2',
-        intercept: false
+        intercept: false,
+        validate: true
       }
 
       # Train a logistic regression model on the given data.
@@ -134,6 +135,12 @@ module Spark
       #   or not of the augmented representation for
       #   training data (i.e. whether bias features
       #   are activated or not).
+      #   (default: false)
+      #
+      # validate::
+      #   Boolean parameter which indicates if the
+      #   algorithm should validate data before training.
+      #   (default: true)
       #
       def self.train(rdd, options={})
         super
@@ -145,7 +152,8 @@ module Spark
                                            options[:initial_weights],
                                            options[:reg_param].to_f,
                                            options[:reg_type],
-                                           options[:intercept])
+                                           options[:intercept],
+                                           options[:validate])
 
         LogisticRegressionModel.new(weights, intercept)
       end

data/lib/spark/mllib/classification/svm.rb

@@ -78,7 +78,8 @@ module Spark
         mini_batch_fraction: 1.0,
         initial_weights: nil,
         reg_type: 'l2',
-        intercept: false
+        intercept: false,
+        validate: true
       }
 
       # Train a support vector machine on the given data.
@@ -114,6 +115,12 @@ module Spark
       #   or not of the augmented representation for
       #   training data (i.e. whether bias features
       #   are activated or not).
+      #   (default: false)
+      #
+      # validateData::
+      #   Boolean parameter which indicates if the
+      #   algorithm should validate data before training.
+      #   (default: true)
       #
       def self.train(rdd, options={})
         super
@@ -125,7 +132,8 @@ module Spark
                                            options[:mini_batch_fraction].to_f,
                                            options[:initial_weights],
                                            options[:reg_type],
-                                           options[:intercept])
+                                           options[:intercept],
+                                           options[:validate])
 
         SVMModel.new(weights, intercept)
       end

data/lib/spark/mllib/clustering/kmeans.rb

@@ -107,10 +107,14 @@ module Spark
       # seed::
       #   Random seed value for cluster initialization.
       #
-      def self.train(rdd, k, max_iterations: 100, runs: 1, initialization_mode: 'k-means||', seed: nil)
+      # epsilon::
+      #   The distance threshold within which we've consider centers to have converged.
+      #
+      def self.train(rdd, k, max_iterations: 100, runs: 1, initialization_mode: 'k-means||', seed: nil,
+                             initialization_steps: 5, epsilon: 0.0001)
         # Call returns KMeansModel
         Spark.jb.call(RubyMLLibAPI.new, 'trainKMeansModel', rdd,
-                      k, max_iterations, runs, initialization_mode, Spark.jb.to_long(seed))
+                      k, max_iterations, runs, initialization_mode, Spark.jb.to_long(seed), initialization_steps, epsilon)
       end
 
     end

data/lib/spark/mllib/regression/lasso.rb

@@ -58,7 +58,9 @@ module Spark
         step: 1.0,
         reg_param: 0.01,
         mini_batch_fraction: 1.0,
-        initial_weights: nil
+        initial_weights: nil,
+        intercept: false,
+        validate: true
       }
 
       # Train a Lasso regression model on the given data.
@@ -82,6 +84,18 @@ module Spark
       # initial_weights::
       #   The initial weights (default: nil).
       #
+      # intercept::
+      #   Boolean parameter which indicates the use
+      #   or not of the augmented representation for
+      #   training data (i.e. whether bias features
+      #   are activated or not).
+      #   (default: false)
+      #
+      # validate::
+      #   Boolean parameter which indicates if the
+      #   algorithm should validate data before training.
+      #   (default: true)
+      #
       def self.train(rdd, options={})
         super
 
@@ -90,7 +104,9 @@ module Spark
                                            options[:step].to_f,
                                            options[:reg_param].to_f,
                                            options[:mini_batch_fraction].to_f,
-                                           options[:initial_weights])
+                                           options[:initial_weights],
+                                           options[:intercept],
+                                           options[:validate])
 
         LassoModel.new(weights, intercept)
       end

data/lib/spark/mllib/regression/linear.rb

@@ -66,7 +66,8 @@ module Spark
         initial_weights: nil,
         reg_param: 0.0,
         reg_type: nil,
-        intercept: false
+        intercept: false,
+        validate: true
       }
 
       # Train a linear regression model on the given data.
@@ -102,7 +103,13 @@ module Spark
       #   Boolean parameter which indicates the use
       #   or not of the augmented representation for
       #   training data (i.e. whether bias features
-      #   are activated or not). (default: False)
+      #   are activated or not).
+      #   (default: false)
+      #
+      # validate::
+      #   Boolean parameter which indicates if the
+      #   algorithm should validate data before training.
+      #   (default: true)
       #
       def self.train(rdd, options={})
         super
@@ -114,7 +121,8 @@ module Spark
                                            options[:initial_weights],
                                            options[:reg_param].to_f,
                                            options[:reg_type],
-                                           options[:intercept])
+                                           options[:intercept],
+                                           options[:validate])
 
         LinearRegressionModel.new(weights, intercept)
       end

data/lib/spark/mllib/regression/ridge.rb

@@ -55,7 +55,9 @@ module Spark
         step: 1.0,
         reg_param: 0.01,
         mini_batch_fraction: 1.0,
-        initial_weights: nil
+        initial_weights: nil,
+        intercept: false,
+        validate: true
       }
 
       # Train a ridge regression model on the given data.
@@ -79,6 +81,18 @@ module Spark
       # initial_weights::
       #   The initial weights (default: nil).
       #
+      # intercept::
+      #   Boolean parameter which indicates the use
+      #   or not of the augmented representation for
+      #   training data (i.e. whether bias features
+      #   are activated or not).
+      #   (default: false)
+      #
+      # validate::
+      #   Boolean parameter which indicates if the
+      #   algorithm should validate data before training.
+      #   (default: true)
+      #
       def self.train(rdd, options={})
         super
 
@@ -87,7 +101,9 @@ module Spark
                                            options[:step].to_f,
                                            options[:reg_param].to_f,
                                            options[:mini_batch_fraction].to_f,
-                                           options[:initial_weights])
+                                           options[:initial_weights],
+                                           options[:intercept],
+                                           options[:validate])
 
         RidgeRegressionModel.new(weights, intercept)
       end

data/lib/spark/rdd.rb

@@ -39,6 +39,7 @@ module Spark
 
       result  = %{#<#{self.class.name}:0x#{object_id}}
       result << %{ (#{comms})} unless comms.empty?
+      result << %{ (cached)} if cached?
       result << %{\n}
       result << %{  Serializer: "#{serializer}"\n}
       result << %{Deserializer: "#{deserializer}"}
@@ -166,8 +167,13 @@ module Spark
 
     # Assign a name to this RDD.
     #
-    def set_name(name)
-      jrdd.setName(name)
+    def set_name(value)
+      jrdd.setName(value)
+      value
+    end
+
+    def name=(value)
+      set_name(value)
     end
 
     def to_java
@@ -193,11 +199,14 @@ module Spark
     def collect(as_enum=false)
       file = Tempfile.new('collect', context.temp_dir)
 
+      context.set_call_site(caller.first)
       RubyRDD.writeRDDToFile(jrdd.rdd, file.path)
 
       collect_from_file(file, as_enum)
     rescue => e
       raise Spark::RDDError, e.message
+    ensure
+      context.clear_call_site
     end
 
     def collect_from_file(file, as_enum=false)

data/lib/spark/serializer.rb

@@ -55,7 +55,7 @@ module Spark
       if block_given?
         class_eval(&block)
       else
-        class_eval(text.to_s)
+        class_eval(text.to_s.downcase)
       end
     end
 

data/lib/spark/serializer/auto_batched.rb

@@ -16,6 +16,13 @@ module Spark
         error('Batch size must be greater than 1') if @best_size < 2
       end
 
+      def batched?
+        true
+      end
+
+      def unbatch!
+      end
+
       def name
         "AutoBatched(#{@best_size})"
       end

data/lib/spark/version.rb

@@ -1,3 +1,3 @@
 module Spark
-  VERSION = '1.1.0.1'
+  VERSION = '1.2.0'
 end

data/ruby-spark.gemspec

@@ -21,9 +21,13 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   if RUBY_PLATFORM =~ /java/
+    spec.platform = 'java'
+
     extensions = ['ext/ruby_java/extconf.rb']
   else
     extensions = ['ext/ruby_c/extconf.rb']
+
+    spec.add_dependency 'rjb'
   end
 
   spec.extensions = extensions
@@ -38,11 +42,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'nio4r'
   spec.add_dependency 'distribution'
 
-  if RUBY_PLATFORM =~ /java/
-  else
-    spec.add_dependency 'rjb'
-  end
-
   spec.add_development_dependency 'bundler', '~> 1.6'
   spec.add_development_dependency 'rake'
 end