ruby-spark 1.0.0

data/lib/spark/helper/serialize.rb

@@ -0,0 +1,71 @@
+module Spark
+  module Helper
+    module Serialize
+
+      DIRECTIVE_INTEGER_BIG_ENDIAN = 'l>'
+      DIRECTIVE_INTEGERS_BIG_ENDIAN = 'l>*'
+      DIRECTIVE_LONG_BIG_ENDIAN = 'q>'
+      DIRECTIVE_LONGS_BIG_ENDIAN = 'q>*'
+      DIRECTIVE_DOUBLE_BIG_ENDIAN = 'G'
+      DIRECTIVE_DOUBLES_BIG_ENDIAN = 'G*'
+      DIRECTIVE_UNSIGNED_CHARS = 'C*'
+      DIRECTIVE_CHARS = 'c*'
+
+      # Packing
+
+      def pack_int(data)
+        [data].pack(DIRECTIVE_INTEGER_BIG_ENDIAN)
+      end
+
+      def pack_long(data)
+        [data].pack(DIRECTIVE_LONG_BIG_ENDIAN)
+      end
+
+      def pack_double(data)
+        [data].pack(DIRECTIVE_DOUBLE_BIG_ENDIAN)
+      end
+
+      def pack_unsigned_chars(data)
+        data.pack(DIRECTIVE_UNSIGNED_CHARS)
+      end
+
+      def pack_ints(data)
+        __check_array(data)
+        data.pack(DIRECTIVE_INTEGERS_BIG_ENDIAN)
+      end
+
+      def pack_longs(data)
+        __check_array(data)
+        data.pack(DIRECTIVE_LONGS_BIG_ENDIAN)
+      end
+
+      def pack_doubles(data)
+        __check_array(data)
+        data.pack(DIRECTIVE_DOUBLES_BIG_ENDIAN)
+      end
+
+      # Unpacking
+
+      def unpack_int(data)
+        data.unpack(DIRECTIVE_INTEGER_BIG_ENDIAN)[0]
+      end
+
+      def unpack_long(data)
+        data.unpack(DIRECTIVE_LONG_BIG_ENDIAN)[0]
+      end
+
+      def unpack_chars(data)
+        data.unpack(DIRECTIVE_CHARS)
+      end
+
+      private
+
+        def __check_array(data)
+          unless data.is_a?(Array)
+            raise ArgumentError, 'Data must be an Array.'
+          end
+        end
+
+    end
+  end
+end

data/lib/spark/helper/statistic.rb

@@ -0,0 +1,93 @@
+module Spark
+  module Helper
+    module Statistic
+
+      # Returns a sampling rate that guarantees a sample of size >= sampleSizeLowerBound 99.99% of the time.
+      #
+      # == How the sampling rate is determined:
+      # Let p = num / total, where num is the sample size and total is the total number of
+      # datapoints in the RDD. We're trying to compute q > p such that
+      # * when sampling with replacement, we're drawing each datapoint with prob_i ~ Pois(q),
+      #   where we want to guarantee Pr[s < num] < 0.0001 for s = sum(prob_i for i from 0 to total),
+      #   i.e. the failure rate of not having a sufficiently large sample < 0.0001.
+      #   Setting q = p + 5 * sqrt(p/total) is sufficient to guarantee 0.9999 success rate for
+      #   num > 12, but we need a slightly larger q (9 empirically determined).
+      # * when sampling without replacement, we're drawing each datapoint with prob_i
+      #   ~ Binomial(total, fraction) and our choice of q guarantees 1-delta, or 0.9999 success
+      #   rate, where success rate is defined the same as in sampling with replacement.
+      #
+      def compute_fraction(lower_bound, total, with_replacement)
+        lower_bound = lower_bound.to_f
+
+        if with_replacement
+          upper_poisson_bound(lower_bound) / total
+        else
+          fraction = lower_bound / total
+          upper_binomial_bound(0.00001, total, fraction)
+        end
+      end
+
+      def upper_poisson_bound(bound)
+        num_std = if bound < 6
+          12
+        elsif bound < 16
+          9
+        else
+          6
+        end.to_f
+
+        [bound + num_std * Math.sqrt(bound), 1e-10].max
+      end
+
+      def upper_binomial_bound(delta, total, fraction)
+        gamma = -Math.log(delta) / total
+        [1, fraction + gamma + Math.sqrt(gamma*gamma + 2*gamma*fraction)].min
+      end
+
+      # Bisect right
+      #
+      # == Examples:
+      #   data = [1,5,6,8,96,120,133]
+      #
+      #   bisect_right(data, 0)   # => 0
+      #   bisect_right(data, 1)   # => 1
+      #   bisect_right(data, 5)   # => 2
+      #   bisect_right(data, 9)   # => 4
+      #   bisect_right(data, 150) # => 7
+      #
+      def bisect_right(data, value, low=0, high=data.size)
+        if low < 0
+          raise ArgumentError, 'Low must be >= 0.'
+        end
+
+        while low < high
+          mid = (low + high) / 2
+          if value < data[mid]
+            high = mid
+          else
+            low = mid + 1
+          end
+        end
+
+        low
+      end
+
+      # Determine bound of partitioning
+      #
+      # == Example:
+      #   data = [0,1,2,3,4,5,6,7,8,9,10]
+      #   determine_bounds(data, 3)
+      #   # => [2, 5, 8]
+      #
+      def determine_bounds(data, num_partitions)
+        bounds = []
+        count = data.size
+        (0...(num_partitions-1)).each do |index|
+          bounds << data[count * (index+1) / num_partitions]
+        end
+        bounds
+      end
+
+    end
+  end
+end

data/lib/spark/helper/system.rb

@@ -0,0 +1,42 @@
+module Spark
+  module Helper
+    module System
+
+      def self.included(base)
+        base.send :extend,  Methods
+        base.send :include, Methods
+      end
+     
+      module Methods
+        def windows?
+          RbConfig::CONFIG['host_os'] =~ /mswin|mingw/
+        end
+
+        def mri?
+          RbConfig::CONFIG['ruby_install_name'] == 'ruby'
+        end
+
+        def jruby?
+          RbConfig::CONFIG['ruby_install_name'] == 'jruby'
+        end
+
+        def pry?
+          !!Thread.current[:__pry__]
+        end
+
+        # Memory usage in kb
+        def memory_usage
+          if jruby?
+            runtime = java.lang.Runtime.getRuntime
+            (runtime.totalMemory - runtime.freeMemory) >> 10
+          elsif windows?
+            # not yet
+          else
+            `ps -o rss= -p #{Process.pid}`.to_i
+          end
+        end
+      end # Methods
+
+    end # System
+  end # Helper
+end # Spark

data/lib/spark/java_bridge.rb

@@ -0,0 +1,19 @@
+module Spark
+  module JavaBridge
+
+    autoload :Base,  'spark/java_bridge/base'
+    autoload :JRuby, 'spark/java_bridge/jruby'
+    autoload :RJB,   'spark/java_bridge/rjb'
+
+    include Spark::Helper::System
+
+    def self.get
+      if jruby?
+        JRuby
+      else
+        RJB
+      end
+    end
+
+  end
+end

data/lib/spark/java_bridge/base.rb

@@ -0,0 +1,203 @@
+##
+# Spark::JavaBridge::Base
+#
+# Parent for all adapter (ruby - java)
+#
+module Spark
+  module JavaBridge
+    class Base
+
+      include Spark::Helper::System
+
+      JAVA_OBJECTS = [
+        'java.util.ArrayList',
+        'org.apache.spark.SparkConf',
+        'org.apache.spark.api.java.JavaSparkContext',
+        'org.apache.spark.api.ruby.RubyRDD',
+        'org.apache.spark.api.ruby.RubyUtils',
+        'org.apache.spark.api.ruby.RubyWorker',
+        'org.apache.spark.api.ruby.PairwiseRDD',
+        'org.apache.spark.api.ruby.RubyAccumulatorParam',
+        'org.apache.spark.api.ruby.RubySerializer',
+        'org.apache.spark.api.python.PythonRDD',
+        'org.apache.spark.api.python.PythonPartitioner',
+        'org.apache.spark.ui.ruby.RubyTab',
+        'org.apache.spark.mllib.api.ruby.RubyMLLibAPI',
+        'scala.collection.mutable.HashMap',
+        :JInteger  => 'java.lang.Integer',
+        :JLong     => 'java.lang.Long',
+        :JLogger   => 'org.apache.log4j.Logger',
+        :JLevel    => 'org.apache.log4j.Level',
+        :JPriority => 'org.apache.log4j.Priority',
+        :JUtils    => 'org.apache.spark.util.Utils',
+        :JStorageLevel => 'org.apache.spark.storage.StorageLevel',
+        :JDenseVector => 'org.apache.spark.mllib.linalg.DenseVector',
+        :JDenseMatrix => 'org.apache.spark.mllib.linalg.DenseMatrix'
+      ]
+
+      JAVA_TEST_OBJECTS = [
+        'org.apache.spark.mllib.api.ruby.RubyMLLibUtilAPI'
+      ]
+
+      RUBY_TO_JAVA_SKIP = [Fixnum, Integer]
+
+      def initialize(spark_home)
+        @spark_home = spark_home
+      end
+
+      # Import all important classes into Objects
+      def load
+        return if @loaded
+
+        java_objects.each do |name, klass|
+          import(name, klass)
+        end
+
+        @loaded = true
+        nil
+      end
+
+      # Import classes for testing
+      def load_test
+        return if @loaded_test
+
+        java_test_objects.each do |name, klass|
+          import(name, klass)
+        end
+
+        @loaded_test = true
+        nil
+      end
+
+      # Call java object
+      def call(klass, method, *args)
+        # To java
+        args.map!{|item| to_java(item)}
+
+        # Call java
+        result = klass.__send__(method, *args)
+
+        # To ruby
+        to_ruby(result)
+      end
+
+      def to_java_array_list(array)
+        array_list = ArrayList.new
+        array.each do |item|
+          array_list.add(to_java(item))
+        end
+        array_list
+      end
+
+      def to_long(number)
+        return nil if number.nil?
+        JLong.new(number)
+      end
+
+      def to_java(object)
+        if RUBY_TO_JAVA_SKIP.include?(object.class)
+          # Some object are convert automatically
+          # This is for preventing errors
+          # For example: jruby store integer as long so 1.to_java is Long
+          object
+        elsif object.respond_to?(:to_java)
+          object.to_java
+        elsif object.is_a?(Array)
+          to_java_array_list(object)
+        else
+          object
+        end
+      end
+
+      # Array problem:
+      #   Rjb:   object.toArray -> Array
+      #   Jruby: object.toArray -> java.lang.Object
+      #
+      def to_ruby(object)
+        if java_object?(object)
+          class_name = object.getClass.getSimpleName
+          case class_name
+          when 'ArraySeq'
+            result = []
+            iterator = object.iterator
+            while iterator.hasNext
+              result << to_ruby(iterator.next)
+            end
+            result
+          when 'Map2', 'Map3', 'Map4', 'HashTrieMap'
+            Hash[
+              object.toSeq.array.to_a.map!{|item| [item._1, item._2]}
+            ]
+          when 'SeqWrapper'; object.toArray.to_a.map!{|item| to_ruby(item)}
+          when 'ofRef';      object.array.to_a.map!{|item| to_ruby(item)} # WrappedArray$ofRef
+          when 'LabeledPoint'; Spark::Mllib::LabeledPoint.from_java(object)
+          when 'DenseVector';  Spark::Mllib::DenseVector.from_java(object)
+          when 'KMeansModel';  Spark::Mllib::KMeansModel.from_java(object)
+          when 'DenseMatrix';  Spark::Mllib::DenseMatrix.from_java(object)
+          else
+            # Some RDD
+            if class_name != 'JavaRDD' && class_name.end_with?('RDD')
+              object = object.toJavaRDD
+              class_name = 'JavaRDD'
+            end
+
+            # JavaRDD
+            if class_name == 'JavaRDD'
+              jrdd = RubyRDD.toRuby(object)
+
+              serializer   = Spark.sc.get_serializer('marshal', nil)
+              deserializer = Spark.sc.get_serializer('marshal', 2) # is fully batched
+
+              return Spark::RDD.new(jrdd, Spark.sc, serializer, deserializer)
+            end
+
+            # Unknow
+            Spark.logger.warn("Java object '#{object.getClass.name}' was not converted.")
+            object
+          end
+
+        else
+          # Already transfered
+          object
+        end
+      end
+
+      alias_method :java_to_ruby, :to_ruby
+      alias_method :ruby_to_java, :to_java
+
+      private
+
+        def jars
+          result = []
+          if File.file?(@spark_home)
+            result << @spark_home
+          else
+            result << Dir.glob(File.join(@spark_home, '*.jar'))
+          end
+          result.flatten
+        end
+
+        def objects_with_names(objects)
+          hash = {}
+          objects.each do |object|
+            if object.is_a?(Hash)
+              hash.merge!(object)
+            else
+              key = object.split('.').last.to_sym
+              hash[key] = object
+            end
+          end
+          hash
+        end
+
+        def java_objects
+          objects_with_names(JAVA_OBJECTS)
+        end
+
+        def java_test_objects
+          objects_with_names(JAVA_TEST_OBJECTS)
+        end
+
+    end
+  end
+end

data/lib/spark/java_bridge/jruby.rb

@@ -0,0 +1,23 @@
+require 'java'
+
+module Spark
+  module JavaBridge
+    class JRuby < Base
+
+      def initialize(*args)
+        super
+        jars.each {|jar| require jar}
+      end
+
+      def import(name, klass)
+        klass = "Java::#{klass}"
+        Object.const_set(name, eval(klass)) rescue nil
+      end
+
+      def java_object?(object)
+        object.is_a?(JavaProxy)
+      end
+
+    end
+  end
+end