ruby-spark 1.0.0 → 1.1.0.1

data/lib/spark/serializer.rb

@@ -1,24 +1,79 @@
 module Spark
+  ##
+  # Serializer
+  #
   module Serializer
-    autoload :Base,        'spark/serializer/base'
-    autoload :UTF8,        'spark/serializer/utf8'
-    autoload :Marshal,     'spark/serializer/marshal'
-    autoload :MessagePack, 'spark/serializer/message_pack'
-    autoload :Oj,          'spark/serializer/oj'
-    autoload :Pair,        'spark/serializer/pair'
-    autoload :Cartesian,   'spark/serializer/cartesian'
 
+    DEFAULT_COMPRESS = false
     DEFAULT_BATCH_SIZE = 1024
     DEFAULT_SERIALIZER_NAME = 'marshal'
 
-    def self.get(suggestion)
-      const_get(suggestion.to_s.camelize) rescue nil
+    @@registered = {}
+
+    # Register class and create method for quick access.
+    # Class will be available also as __name__ for using
+    # in build method (Proc binding problem).
+    #
+    # == Examples:
+    #   register('test1', 'test2', Class)
+    #
+    #   Spark::Serializer.test1
+    #   Spark::Serializer.test2
+    #
+    #   # Proc binding problem
+    #   build { marshal } # => Spark::Serializer::Marshal
+    #
+    #   marshal = 1
+    #   build { marshal } # => 1
+    #
+    #   build { __marshal__ } # => Spark::Serializer::Marshal
+    #
+    def self.register(*args)
+      klass = args.pop
+      args.each do |arg|
+        @@registered[arg] = klass
+        define_singleton_method(arg.to_sym){|*args| klass.new(*args) }
+        define_singleton_method("__#{arg}__".to_sym){|*args| klass.new(*args) }
+      end
+    end
+
+    def self.find(name)
+      @@registered[name.to_s.downcase]
     end
 
-    def self.get!(suggestion)
-      const_get(suggestion.to_s.camelize)
-    rescue
-      raise Spark::NotImplemented, "Serializer #{suggestion.to_s.camelize} not exist."
+    def self.find!(name)
+      klass = find(name)
+
+      if klass.nil?
+        raise Spark::SerializeError, "Unknow serializer #{name}."
+      end
+
+      klass
     end
+
+    def self.build(text=nil, &block)
+      if block_given?
+        class_eval(&block)
+      else
+        class_eval(text.to_s)
+      end
+    end
+
   end
 end
+
+# Parent
+require 'spark/serializer/base'
+
+# Basic
+require 'spark/serializer/oj'
+require 'spark/serializer/marshal'
+require 'spark/serializer/message_pack'
+require 'spark/serializer/text'
+
+# Others
+require 'spark/serializer/batched'
+require 'spark/serializer/auto_batched'
+require 'spark/serializer/compressed'
+require 'spark/serializer/pair'
+require 'spark/serializer/cartesian'

data/lib/spark/serializer/auto_batched.rb

@@ -0,0 +1,59 @@
+module Spark
+  module Serializer
+    ##
+    # AutoBatched serializator
+    #
+    # Batch size is computed automatically. Simillar to Python's AutoBatchedSerializer.
+    #
+    class AutoBatched < Batched
+
+      MAX_RATIO = 10
+
+      def initialize(serializer, best_size=65536)
+        @serializer = serializer
+        @best_size = best_size.to_i
+
+        error('Batch size must be greater than 1') if @best_size < 2
+      end
+
+      def name
+        "AutoBatched(#{@best_size})"
+      end
+
+      def dump_to_io(data, io)
+        check_each(data)
+
+        # Only Array have .slice
+        data = data.to_a
+
+        index = 0
+        batch = 2
+        max = @best_size * MAX_RATIO
+
+        loop do
+          chunk = data.slice(index, batch)
+          if chunk.nil? || chunk.empty?
+            break
+          end
+
+          serialized = @serializer.dump(chunk)
+          io.write_string(serialized)
+
+          index += batch
+
+          size = serialized.bytesize
+          if size < @best_size
+            batch *= 2
+          elsif size > max && batch > 1
+            batch /= 2
+          end
+        end
+
+        io.flush
+      end
+
+    end
+  end
+end
+
+Spark::Serializer.register('auto_batched', 'autobatched', Spark::Serializer::AutoBatched)

data/lib/spark/serializer/base.rb

@@ -1,168 +1,61 @@
 module Spark
   module Serializer
-    # @abstract Parent for all type of serializers
+    # @abstract Parent for all serializers
     class Base
 
-      include Spark::Helper::Serialize
-      include Spark::Constant
+      def load_from_io(io)
+        return to_enum(__callee__, io) unless block_given?
 
-      attr_reader :batch_size
+        loop do
+          size = io.read_int_or_eof
+          break if size == Spark::Constant::DATA_EOF
 
-      # Set default values
-      def initialize(batch_size=nil)
-        self.batch_size = batch_size
+          yield load(io.read(size))
+        end
       end
 
-      def ==(other)
-        self.class == other.class && self.batch_size == other.batch_size
-      end
+      def load_from_file(file, *args)
+        return to_enum(__callee__, file, *args) unless block_given?
 
-      # Set values given by user
-      def set(batch_size)
-        self.batch_size = batch_size unless batch_size.nil?
-        self
-      end
+        load_from_io(file, *args).each do |item|
+          yield item
+        end
 
-      def batch_size=(size)
-        @batch_size = size.to_i
+        file.close
+        file.unlink
       end
 
-      def unbatch!
-        self.batch_size = 1
+      def ==(other)
+        self.to_s == other.to_s
       end
 
-      # nil, 0, 1 are considered as non-batched
       def batched?
-        batch_size > 1
-      end
-
-      # ===========================================================================
-      # Load
-
-      # Load and deserialize an Array from IO, Array of Java iterator
-      #   mri:   respond_to?(:iterator) => false
-      #   jruby: respond_to?(:iterator) => true
-      #
-      def load(source)
-        # Tempfile is Delegator for File so it is not IO
-        # second wasy is __getobj__.is_a?(IO)
-        if source.is_a?(IO) || source.is_a?(Tempfile)
-          load_from_io(source)
-        # elsif source.is_a?(Array)
-        #   load_from_array(source)
-        elsif try(source, :iterator)
-          load_from_iterator(source.iterator)
-        end
+        false
       end
 
-      # Load data from IO. Data must have a format:
-      #
-      #   +------------+--------+
-      #   | signed int |  data  |
-      #   |     4B     |        |
-      #   +------------+--------+
-      #
-      def load_from_io(io)
-        return to_enum(__callee__, io) unless block_given?
-
-        loop do
-          lenght = read_int(io)
-          break if lenght == DATA_EOF
-
-          result = load_next_from_io(io, lenght)
-          if batched? && result.respond_to?(:each)
-            result.each {|item| yield item }
-          else
-            yield result
-          end
-        end # loop
-      end # load_from_io
-
-      def load_next_from_io(io, lenght)
-        deserialize(io.read(lenght))
+      def unbatch!
       end
 
-      # Load from Java iterator by calling hasNext and next
-      #
-      def load_from_iterator(iterator)
-        result = []
-        while iterator.hasNext
-          item = iterator.next
-
-          # mri: data are String
-          # jruby: data are bytes Array
-
-          if item.is_a?(String)
-            # Serialized data
-            result << deserialize(item)
-          else
-            # Java object
-            if try(item, :getClass)
-              case item.getClass.name
-              when '[B'
-                # Array of bytes
-                result << deserialize(pack_unsigned_chars(item.to_a))
-              when 'scala.Tuple2'
-                # Tuple2
-                result << deserialize(item._1, item._2)
-              end
-            end
-          end
-
+      def check_each(data)
+        unless data.respond_to?(:each)
+          error('Data must be iterable.')
         end
-
-        result.flatten!(1) if batched?
-        result
       end
 
-      def read_int(io)
-        bytes = io.read(4)
-        return DATA_EOF if bytes.nil?
-        unpack_int(bytes)
+      def error(message)
+        raise Spark::SerializeError, message
       end
 
-      # ===========================================================================
-      # Dump
-
-      # Serialize and send data into IO. Check 'load_from_io' for data format.
-      def dump(data, io)
-        if !data.is_a?(Array) && !data.is_a?(Enumerator)
-          data = [data]
-        end
-        data = data.each_slice(batch_size) if batched?
-
-        data.each do |item|
-          serialized = serialize(item)
-
-          # Size and data can have different encoding
-          # Marshal: both ASCII
-          # Oj: ASCII and UTF-8
-          io.write(pack_int(serialized.bytesize))
-          io.write(serialized)
-        end
-
-        io.flush
+      def name
+        self.class.name.split('::').last
       end
 
-      # For direct serialization
-      def dump_to_java(data)
-        data.map! do |item|
-          serialize(item).to_java_bytes
-        end
+      def to_s
+        name
       end
 
-      # Rescue cannot be defined
-      #
-      #   mri   => RuntimeError
-      #   jruby => NoMethodError
-      #
-      def try(object, method)
-        begin
-          object.__send__(method)
-          return true
-        rescue
-          return false
-        end
+      def inspect
+        %{#<Spark::Serializer:0x#{object_id}  "#{self}">}
       end
 
     end

data/lib/spark/serializer/batched.rb

@@ -0,0 +1,84 @@
+module Spark
+  module Serializer
+    class Batched < Base
+
+      attr_writer :serializer
+
+      def initialize(serializer, batch_size=nil)
+        batch_size ||= Spark::Serializer::DEFAULT_BATCH_SIZE
+
+        @serializer = serializer
+        @batch_size = batch_size.to_i
+
+        error('Batch size must be greater than 0') if @batch_size < 1
+      end
+
+      # Really batched
+      def batched?
+        @batch_size > 1
+      end
+
+      def unbatch!
+        @batch_size = 1
+      end
+
+      def load(data)
+        @serializer.load(data)
+      end
+
+      def dump(data)
+        @serializer.dump(data)
+      end
+
+      def name
+        "Batched(#{@batch_size})"
+      end
+
+      def to_s
+        "#{name} -> #{@serializer}"
+      end
+
+
+      # === Dump ==============================================================
+
+      def dump_to_io(data, io)
+        check_each(data)
+
+        if batched?
+          data = data.each_slice(@batch_size)
+        end
+
+        data.each do |item|
+          serialized = dump(item)
+          io.write_string(serialized)
+        end
+
+        io.flush
+      end
+
+
+      # === Load ==============================================================
+
+      def load_from_io(io)
+        return to_enum(__callee__, io) unless block_given?
+
+        loop do
+          size = io.read_int_or_eof
+          break if size == Spark::Constant::DATA_EOF
+
+          data = io.read(size)
+          data = load(data)
+
+          if batched?
+            data.each{|item| yield item }
+          else
+            yield data
+          end
+        end
+      end
+
+    end
+  end
+end
+
+Spark::Serializer.register('batched', Spark::Serializer::Batched)

data/lib/spark/serializer/cartesian.rb

@@ -1,37 +1,13 @@
 module Spark
   module Serializer
-    class Cartesian < Base
+    class Cartesian < Pair
 
-      attr_reader :first, :second
-
-      def set(first, second)
-        @first  = first
-        @second = second
-        self
-      end
-
-      # Little hack
-      # Data does not have to be batched but items are added by <<
-      def batched?
-        true
-      end
-
-      def load_next_from_io(io, lenght)
-        item1 = io.read(lenght)
-        item2 = io.read_string
-        deserialize(item1, item2)
-      end
-
-      def deserialize(item1, item2)
-        deserialized_item1 = @first.deserialize(item1)
-        deserialized_item2 = @second.deserialize(item2)
-
-        deserialized_item1 = [deserialized_item1] unless @first.batched?
-        deserialized_item2 = [deserialized_item2] unless @second.batched?
-
-        deserialized_item1.product(deserialized_item2)
+      def aggregate(item1, item2)
+        item1.product(item2)
       end
 
     end
   end
 end
+
+Spark::Serializer.register('cartesian', Spark::Serializer::Cartesian)