ruby-spark 1.0.0

data/lib/spark/sampler.rb

@@ -0,0 +1,92 @@
+require 'distribution'
+
+# Random Generators
+module Spark
+  module RandomGenerator
+    class Poisson
+
+      def initialize(mean, seed)
+        generator = Random.new(seed)
+        @exp_rng = Distribution::Exponential.rng(1.0/mean, random: generator)
+      end
+
+      def rand
+        t = 0.0
+        number = 0
+
+        loop{
+          t += @exp_rng.call
+          if t > 1
+            return number
+          end
+          number += 1
+        }
+      end
+
+    end
+  end
+end
+
+# Samplers
+module Spark
+  module Sampler
+
+    class Base
+      attr_reader :fraction, :seed
+
+      def initialize(fraction, seed=nil)
+        @fraction = fraction
+        @seed = seed || Random.new_seed
+      end
+    end
+
+    # Poisson Sampler
+    # -------------------------------------------------------------------------
+    class Poisson < Base
+
+      def sample(iterator)
+        iterator.map! do |item|
+          count = rng.rand
+          Array.new(count) { item }
+        end
+        iterator.flatten!
+        iterator.compact!
+        iterator
+      end
+
+      def lazy_sample(iterator)
+        Enumerator::Lazy.new(iterator) do |yielder, value|
+          count = rng.rand
+          count.times { yielder << value }
+        end
+      end
+
+      def rng
+        @rng ||= Spark::RandomGenerator::Poisson.new(fraction, seed)
+      end
+
+    end
+
+    # Uniform Sampler
+    # -------------------------------------------------------------------------
+    class Uniform < Base
+
+      def sample(iterator)
+        iterator.select!{|item| rng.rand <= fraction}
+        iterator
+      end
+
+      def lazy_sample(iterator)
+        iterator.select do |item|
+          rng.rand <= fraction
+        end
+      end
+
+      def rng
+        @rng ||= Random.new(seed)
+      end
+
+    end
+
+  end
+end

data/lib/spark/serializer.rb

@@ -0,0 +1,24 @@
+module Spark
+  module Serializer
+    autoload :Base,        'spark/serializer/base'
+    autoload :UTF8,        'spark/serializer/utf8'
+    autoload :Marshal,     'spark/serializer/marshal'
+    autoload :MessagePack, 'spark/serializer/message_pack'
+    autoload :Oj,          'spark/serializer/oj'
+    autoload :Pair,        'spark/serializer/pair'
+    autoload :Cartesian,   'spark/serializer/cartesian'
+
+    DEFAULT_BATCH_SIZE = 1024
+    DEFAULT_SERIALIZER_NAME = 'marshal'
+
+    def self.get(suggestion)
+      const_get(suggestion.to_s.camelize) rescue nil
+    end
+
+    def self.get!(suggestion)
+      const_get(suggestion.to_s.camelize)
+    rescue
+      raise Spark::NotImplemented, "Serializer #{suggestion.to_s.camelize} not exist."
+    end
+  end
+end

data/lib/spark/serializer/base.rb

@@ -0,0 +1,170 @@
+module Spark
+  module Serializer
+    # @abstract Parent for all type of serializers
+    class Base
+
+      include Spark::Helper::Serialize
+      include Spark::Constant
+
+      attr_reader :batch_size
+
+      # Set default values
+      def initialize(batch_size=nil)
+        self.batch_size = batch_size
+      end
+
+      def ==(other)
+        self.class == other.class && self.batch_size == other.batch_size
+      end
+
+      # Set values given by user
+      def set(batch_size)
+        self.batch_size = batch_size unless batch_size.nil?
+        self
+      end
+
+      def batch_size=(size)
+        @batch_size = size.to_i
+      end
+
+      def unbatch!
+        self.batch_size = 1
+      end
+
+      # nil, 0, 1 are considered as non-batched
+      def batched?
+        batch_size > 1
+      end
+
+      # ===========================================================================
+      # Load
+
+      # Load and deserialize an Array from IO, Array of Java iterator
+      #   mri:   respond_to?(:iterator) => false
+      #   jruby: respond_to?(:iterator) => true
+      #
+      def load(source)
+        # Tempfile is Delegator for File so it is not IO
+        # second wasy is __getobj__.is_a?(IO)
+        if source.is_a?(IO) || source.is_a?(Tempfile)
+          load_from_io(source)
+        # elsif source.is_a?(Array)
+        #   load_from_array(source)
+        elsif try(source, :iterator)
+          load_from_iterator(source.iterator)
+        end
+      end
+
+      # Load data from IO. Data must have a format:
+      #
+      #   +------------+--------+
+      #   | signed int |  data  |
+      #   |     4B     |        |
+      #   +------------+--------+
+      #
+      def load_from_io(io)
+        return to_enum(__callee__, io) unless block_given?
+
+        loop do
+          lenght = read_int(io)
+          break if lenght == DATA_EOF
+
+          result = load_next_from_io(io, lenght)
+          if batched? && result.respond_to?(:each)
+            result.each {|item| yield item }
+          else
+            yield result
+          end
+        end # loop
+      end # load_from_io
+
+      def load_next_from_io(io, lenght)
+        deserialize(io.read(lenght))
+      end
+
+      # Load from Java iterator by calling hasNext and next
+      #
+      def load_from_iterator(iterator)
+        result = []
+        while iterator.hasNext
+          item = iterator.next
+
+          # mri: data are String
+          # jruby: data are bytes Array
+
+          if item.is_a?(String)
+            # Serialized data
+            result << deserialize(item)
+          else
+            # Java object
+            if try(item, :getClass)
+              case item.getClass.name
+              when '[B'
+                # Array of bytes
+                result << deserialize(pack_unsigned_chars(item.to_a))
+              when 'scala.Tuple2'
+                # Tuple2
+                result << deserialize(item._1, item._2)
+              end
+            end
+          end
+
+        end
+
+        result.flatten!(1) if batched?
+        result
+      end
+
+      def read_int(io)
+        bytes = io.read(4)
+        return DATA_EOF if bytes.nil?
+        unpack_int(bytes)
+      end
+
+      # ===========================================================================
+      # Dump
+
+      # Serialize and send data into IO. Check 'load_from_io' for data format.
+      def dump(data, io)
+        if !data.is_a?(Array) && !data.is_a?(Enumerator)
+          data = [data]
+        end
+        data = data.each_slice(batch_size) if batched?
+
+        data.each do |item|
+          serialized = serialize(item)
+
+          # Size and data can have different encoding
+          # Marshal: both ASCII
+          # Oj: ASCII and UTF-8
+          io.write(pack_int(serialized.bytesize))
+          io.write(serialized)
+        end
+
+        io.flush
+      end
+
+      # For direct serialization
+      def dump_to_java(data)
+        data.map! do |item|
+          serialize(item).to_java_bytes
+        end
+      end
+
+      # Rescue cannot be defined
+      #
+      #   mri   => RuntimeError
+      #   jruby => NoMethodError
+      #
+      def try(object, method)
+        begin
+          object.__send__(method)
+          return true
+        rescue
+          return false
+        end
+      end
+
+    end
+  end
+end

data/lib/spark/serializer/cartesian.rb

@@ -0,0 +1,37 @@
+module Spark
+  module Serializer
+    class Cartesian < Base
+
+      attr_reader :first, :second
+
+      def set(first, second)
+        @first  = first
+        @second = second
+        self
+      end
+
+      # Little hack
+      # Data does not have to be batched but items are added by <<
+      def batched?
+        true
+      end
+
+      def load_next_from_io(io, lenght)
+        item1 = io.read(lenght)
+        item2 = io.read_string
+        deserialize(item1, item2)
+      end
+
+      def deserialize(item1, item2)
+        deserialized_item1 = @first.deserialize(item1)
+        deserialized_item2 = @second.deserialize(item2)
+
+        deserialized_item1 = [deserialized_item1] unless @first.batched?
+        deserialized_item2 = [deserialized_item2] unless @second.batched?
+
+        deserialized_item1.product(deserialized_item2)
+      end
+
+    end
+  end
+end

data/lib/spark/serializer/marshal.rb

@@ -0,0 +1,19 @@
+module Spark
+  module Serializer
+    class Marshal < Base
+
+      def name
+        'marshal'
+      end
+
+      def serialize(data)
+        ::Marshal::dump(data)
+      end
+
+      def deserialize(data)
+        ::Marshal::load(data)
+      end
+
+    end
+  end
+end

data/lib/spark/serializer/message_pack.rb

@@ -0,0 +1,25 @@
+module Spark
+  module Serializer
+    class MessagePack < Marshal
+
+      def name
+        'message_pack'
+      end
+
+      def self.serialize(data)
+        ::MessagePack::dump(data)
+      end
+
+      def self.deserialize(data)
+        ::MessagePack::load(data)
+      end
+
+    end
+  end
+end
+
+begin
+  require 'msgpack'
+rescue LoadError
+  Spark::Serializer::MessagePack = Spark::Serializer::Marshal
+end

data/lib/spark/serializer/oj.rb

@@ -0,0 +1,25 @@
+module Spark
+  module Serializer
+    class Oj < Marshal
+
+      def name
+        'oj'
+      end
+
+      def serialize(data)
+        ::Oj::dump(data)
+      end
+
+      def deserialize(data)
+        ::Oj::load(data)
+      end
+
+    end
+  end
+end
+
+begin
+  require 'oj'
+rescue LoadError
+  Spark::Serializer::Oj = Spark::Serializer::Marshal
+end

data/lib/spark/serializer/pair.rb

@@ -0,0 +1,27 @@
+module Spark
+  module Serializer
+    class Pair < Base
+
+      attr_reader :first, :second
+
+      def set(first, second)
+        unbatch!
+        @first  = first
+        @second = second
+        self
+      end
+
+      def batched?
+        false
+      end
+
+      def load_next_from_io(io, lenght)
+        key_value = []
+        key_value << @first.load_next_from_io(io, lenght)
+        key_value << @second.load_next_from_io(io, read_int(io))
+        key_value
+      end
+
+    end
+  end
+end

data/lib/spark/serializer/utf8.rb

@@ -0,0 +1,25 @@
+module Spark
+  module Serializer
+    ##
+    # Used for file
+    #
+    # File is sended as String but worker use serialization
+    #
+    class UTF8 < Base
+
+      def set(*)
+        unbatch!
+        self
+      end
+
+      def batched?
+        false
+      end
+
+      def load_next_from_io(io, lenght)
+        io.read(lenght).force_encoding(Encoding::UTF_8)
+      end
+
+    end
+  end
+end