ruby-spark 1.1.0.1-java

data/lib/spark/sampler.rb

@@ -0,0 +1,92 @@
+require 'distribution'
+
+# Random Generators
+module Spark
+  module RandomGenerator
+    class Poisson
+
+      def initialize(mean, seed)
+        generator = Random.new(seed)
+        @exp_rng = Distribution::Exponential.rng(1.0/mean, random: generator)
+      end
+
+      def rand
+        t = 0.0
+        number = 0
+
+        loop{
+          t += @exp_rng.call
+          if t > 1
+            return number
+          end
+          number += 1
+        }
+      end
+
+    end
+  end
+end
+
+# Samplers
+module Spark
+  module Sampler
+
+    class Base
+      attr_reader :fraction, :seed
+
+      def initialize(fraction, seed=nil)
+        @fraction = fraction
+        @seed = seed || Random.new_seed
+      end
+    end
+
+    # Poisson Sampler
+    # -------------------------------------------------------------------------
+    class Poisson < Base
+
+      def sample(iterator)
+        iterator.map! do |item|
+          count = rng.rand
+          Array.new(count) { item }
+        end
+        iterator.flatten!
+        iterator.compact!
+        iterator
+      end
+
+      def lazy_sample(iterator)
+        Enumerator::Lazy.new(iterator) do |yielder, value|
+          count = rng.rand
+          count.times { yielder << value }
+        end
+      end
+
+      def rng
+        @rng ||= Spark::RandomGenerator::Poisson.new(fraction, seed)
+      end
+
+    end
+
+    # Uniform Sampler
+    # -------------------------------------------------------------------------
+    class Uniform < Base
+
+      def sample(iterator)
+        iterator.select!{|item| rng.rand <= fraction}
+        iterator
+      end
+
+      def lazy_sample(iterator)
+        iterator.select do |item|
+          rng.rand <= fraction
+        end
+      end
+
+      def rng
+        @rng ||= Random.new(seed)
+      end
+
+    end
+
+  end
+end

data/lib/spark/serializer.rb

@@ -0,0 +1,79 @@
+module Spark
+  ##
+  # Serializer
+  #
+  module Serializer
+
+    DEFAULT_COMPRESS = false
+    DEFAULT_BATCH_SIZE = 1024
+    DEFAULT_SERIALIZER_NAME = 'marshal'
+
+    @@registered = {}
+
+    # Register class and create method for quick access.
+    # Class will be available also as __name__ for using
+    # in build method (Proc binding problem).
+    #
+    # == Examples:
+    #   register('test1', 'test2', Class)
+    #
+    #   Spark::Serializer.test1
+    #   Spark::Serializer.test2
+    #
+    #   # Proc binding problem
+    #   build { marshal } # => Spark::Serializer::Marshal
+    #
+    #   marshal = 1
+    #   build { marshal } # => 1
+    #
+    #   build { __marshal__ } # => Spark::Serializer::Marshal
+    #
+    def self.register(*args)
+      klass = args.pop
+      args.each do |arg|
+        @@registered[arg] = klass
+        define_singleton_method(arg.to_sym){|*args| klass.new(*args) }
+        define_singleton_method("__#{arg}__".to_sym){|*args| klass.new(*args) }
+      end
+    end
+
+    def self.find(name)
+      @@registered[name.to_s.downcase]
+    end
+
+    def self.find!(name)
+      klass = find(name)
+
+      if klass.nil?
+        raise Spark::SerializeError, "Unknow serializer #{name}."
+      end
+
+      klass
+    end
+
+    def self.build(text=nil, &block)
+      if block_given?
+        class_eval(&block)
+      else
+        class_eval(text.to_s)
+      end
+    end
+
+  end
+end
+
+# Parent
+require 'spark/serializer/base'
+
+# Basic
+require 'spark/serializer/oj'
+require 'spark/serializer/marshal'
+require 'spark/serializer/message_pack'
+require 'spark/serializer/text'
+
+# Others
+require 'spark/serializer/batched'
+require 'spark/serializer/auto_batched'
+require 'spark/serializer/compressed'
+require 'spark/serializer/pair'
+require 'spark/serializer/cartesian'

data/lib/spark/serializer/auto_batched.rb

@@ -0,0 +1,59 @@
+module Spark
+  module Serializer
+    ##
+    # AutoBatched serializator
+    #
+    # Batch size is computed automatically. Simillar to Python's AutoBatchedSerializer.
+    #
+    class AutoBatched < Batched
+
+      MAX_RATIO = 10
+
+      def initialize(serializer, best_size=65536)
+        @serializer = serializer
+        @best_size = best_size.to_i
+
+        error('Batch size must be greater than 1') if @best_size < 2
+      end
+
+      def name
+        "AutoBatched(#{@best_size})"
+      end
+
+      def dump_to_io(data, io)
+        check_each(data)
+
+        # Only Array have .slice
+        data = data.to_a
+
+        index = 0
+        batch = 2
+        max = @best_size * MAX_RATIO
+
+        loop do
+          chunk = data.slice(index, batch)
+          if chunk.nil? || chunk.empty?
+            break
+          end
+
+          serialized = @serializer.dump(chunk)
+          io.write_string(serialized)
+
+          index += batch
+
+          size = serialized.bytesize
+          if size < @best_size
+            batch *= 2
+          elsif size > max && batch > 1
+            batch /= 2
+          end
+        end
+
+        io.flush
+      end
+
+    end
+  end
+end
+
+Spark::Serializer.register('auto_batched', 'autobatched', Spark::Serializer::AutoBatched)

data/lib/spark/serializer/base.rb

@@ -0,0 +1,63 @@
+module Spark
+  module Serializer
+    # @abstract Parent for all serializers
+    class Base
+
+      def load_from_io(io)
+        return to_enum(__callee__, io) unless block_given?
+
+        loop do
+          size = io.read_int_or_eof
+          break if size == Spark::Constant::DATA_EOF
+
+          yield load(io.read(size))
+        end
+      end
+
+      def load_from_file(file, *args)
+        return to_enum(__callee__, file, *args) unless block_given?
+
+        load_from_io(file, *args).each do |item|
+          yield item
+        end
+
+        file.close
+        file.unlink
+      end
+
+      def ==(other)
+        self.to_s == other.to_s
+      end
+
+      def batched?
+        false
+      end
+
+      def unbatch!
+      end
+
+      def check_each(data)
+        unless data.respond_to?(:each)
+          error('Data must be iterable.')
+        end
+      end
+
+      def error(message)
+        raise Spark::SerializeError, message
+      end
+
+      def name
+        self.class.name.split('::').last
+      end
+
+      def to_s
+        name
+      end
+
+      def inspect
+        %{#<Spark::Serializer:0x#{object_id}  "#{self}">}
+      end
+
+    end
+  end
+end

data/lib/spark/serializer/batched.rb

@@ -0,0 +1,84 @@
+module Spark
+  module Serializer
+    class Batched < Base
+
+      attr_writer :serializer
+
+      def initialize(serializer, batch_size=nil)
+        batch_size ||= Spark::Serializer::DEFAULT_BATCH_SIZE
+
+        @serializer = serializer
+        @batch_size = batch_size.to_i
+
+        error('Batch size must be greater than 0') if @batch_size < 1
+      end
+
+      # Really batched
+      def batched?
+        @batch_size > 1
+      end
+
+      def unbatch!
+        @batch_size = 1
+      end
+
+      def load(data)
+        @serializer.load(data)
+      end
+
+      def dump(data)
+        @serializer.dump(data)
+      end
+
+      def name
+        "Batched(#{@batch_size})"
+      end
+
+      def to_s
+        "#{name} -> #{@serializer}"
+      end
+
+
+      # === Dump ==============================================================
+
+      def dump_to_io(data, io)
+        check_each(data)
+
+        if batched?
+          data = data.each_slice(@batch_size)
+        end
+
+        data.each do |item|
+          serialized = dump(item)
+          io.write_string(serialized)
+        end
+
+        io.flush
+      end
+
+
+      # === Load ==============================================================
+
+      def load_from_io(io)
+        return to_enum(__callee__, io) unless block_given?
+
+        loop do
+          size = io.read_int_or_eof
+          break if size == Spark::Constant::DATA_EOF
+
+          data = io.read(size)
+          data = load(data)
+
+          if batched?
+            data.each{|item| yield item }
+          else
+            yield data
+          end
+        end
+      end
+
+    end
+  end
+end
+
+Spark::Serializer.register('batched', Spark::Serializer::Batched)

data/lib/spark/serializer/cartesian.rb

@@ -0,0 +1,13 @@
+module Spark
+  module Serializer
+    class Cartesian < Pair
+
+      def aggregate(item1, item2)
+        item1.product(item2)
+      end
+
+    end
+  end
+end
+
+Spark::Serializer.register('cartesian', Spark::Serializer::Cartesian)

data/lib/spark/serializer/compressed.rb

@@ -0,0 +1,27 @@
+module Spark
+  module Serializer
+    class Compressed < Base
+
+      def initialize(serializer)
+        @serializer = serializer
+      end
+
+      def dump(data)
+        Zlib::Deflate.deflate(@serializer.dump(data))
+      end
+
+      def load(data)
+        @serializer.load(Zlib::Inflate.inflate(data))
+      end
+
+    end
+  end
+end
+
+begin
+  # TODO: require only if it is necessary
+  require 'zlib'
+
+  Spark::Serializer.register('compress', 'compressed', Spark::Serializer::Compressed)
+rescue LoadError
+end

data/lib/spark/serializer/marshal.rb

@@ -0,0 +1,17 @@
+module Spark
+  module Serializer
+    class Marshal < Base
+
+      def dump(data)
+        ::Marshal.dump(data)
+      end
+
+      def load(data)
+        ::Marshal.load(data)
+      end
+
+    end
+  end
+end
+
+Spark::Serializer.register('marshal', Spark::Serializer::Marshal)

data/lib/spark/serializer/message_pack.rb

@@ -0,0 +1,23 @@
+module Spark
+  module Serializer
+    class MessagePack < Base
+
+      def dump(data)
+        ::MessagePack.dump(data)
+      end
+
+      def load(data)
+        ::MessagePack.load(data)
+      end
+
+    end
+  end
+end
+
+begin
+  # TODO: require only if it is necessary
+  require 'msgpack'
+
+  Spark::Serializer.register('messagepack', 'message_pack', 'msgpack', 'msg_pack', Spark::Serializer::MessagePack)
+rescue LoadError
+end