ruby-spark 1.0.0 → 1.1.0.1

data/lib/spark/serializer/compressed.rb

@@ -0,0 +1,27 @@
+module Spark
+  module Serializer
+    class Compressed < Base
+
+      def initialize(serializer)
+        @serializer = serializer
+      end
+
+      def dump(data)
+        Zlib::Deflate.deflate(@serializer.dump(data))
+      end
+
+      def load(data)
+        @serializer.load(Zlib::Inflate.inflate(data))
+      end
+
+    end
+  end
+end
+
+begin
+  # TODO: require only if it is necessary
+  require 'zlib'
+
+  Spark::Serializer.register('compress', 'compressed', Spark::Serializer::Compressed)
+rescue LoadError
+end

data/lib/spark/serializer/marshal.rb

@@ -2,18 +2,16 @@ module Spark
   module Serializer
     class Marshal < Base
 
-      def name
-        'marshal'
+      def dump(data)
+        ::Marshal.dump(data)
       end
 
-      def serialize(data)
-        ::Marshal::dump(data)
-      end
-
-      def deserialize(data)
-        ::Marshal::load(data)
+      def load(data)
+        ::Marshal.load(data)
       end
 
     end
   end
 end
+
+Spark::Serializer.register('marshal', Spark::Serializer::Marshal)

data/lib/spark/serializer/message_pack.rb

@@ -1,17 +1,13 @@
 module Spark
   module Serializer
-    class MessagePack < Marshal
+    class MessagePack < Base
 
-      def name
-        'message_pack'
+      def dump(data)
+        ::MessagePack.dump(data)
       end
 
-      def self.serialize(data)
-        ::MessagePack::dump(data)
-      end
-
-      def self.deserialize(data)
-        ::MessagePack::load(data)
+      def load(data)
+        ::MessagePack.load(data)
       end
 
     end
@@ -19,7 +15,9 @@ module Spark
 end
 
 begin
+  # TODO: require only if it is necessary
   require 'msgpack'
+
+  Spark::Serializer.register('messagepack', 'message_pack', 'msgpack', 'msg_pack', Spark::Serializer::MessagePack)
 rescue LoadError
-  Spark::Serializer::MessagePack = Spark::Serializer::Marshal
 end

data/lib/spark/serializer/oj.rb

@@ -1,17 +1,13 @@
 module Spark
   module Serializer
-    class Oj < Marshal
+    class Oj < Base
 
-      def name
-        'oj'
+      def dump(data)
+        ::Oj.dump(data)
       end
 
-      def serialize(data)
-        ::Oj::dump(data)
-      end
-
-      def deserialize(data)
-        ::Oj::load(data)
+      def load(data)
+        ::Oj.load(data)
       end
 
     end
@@ -19,7 +15,9 @@ module Spark
 end
 
 begin
+  # TODO: require only if it is necessary
   require 'oj'
+
+  Spark::Serializer.register('oj', Spark::Serializer::Oj)
 rescue LoadError
-  Spark::Serializer::Oj = Spark::Serializer::Marshal
 end

data/lib/spark/serializer/pair.rb

@@ -2,26 +2,40 @@ module Spark
   module Serializer
     class Pair < Base
 
-      attr_reader :first, :second
+      def initialize(serializer1, serializer2)
+        @serializer1 = serializer1
+        @serializer2 = serializer2
+      end
 
-      def set(first, second)
-        unbatch!
-        @first  = first
-        @second = second
-        self
+      def to_s
+        "#{name}(#{@serializer1}, #{@serializer2})"
       end
 
-      def batched?
-        false
+      def aggregate(item1, item2)
+        item1.zip(item2)
       end
 
-      def load_next_from_io(io, lenght)
-        key_value = []
-        key_value << @first.load_next_from_io(io, lenght)
-        key_value << @second.load_next_from_io(io, read_int(io))
-        key_value
+      def load_from_io(io)
+        return to_enum(__callee__, io) unless block_given?
+
+        loop do
+          size = io.read_int_or_eof
+          break if size == Spark::Constant::DATA_EOF
+
+          item1 = @serializer1.load(io.read(size))
+          item2 = @serializer2.load(io.read_string)
+
+          item1 = [item1] unless @serializer1.batched?
+          item2 = [item2] unless @serializer2.batched?
+
+          aggregate(item1, item2).each do |item|
+            yield item
+          end
+        end
       end
 
     end
   end
 end
+
+Spark::Serializer.register('pair', Spark::Serializer::Pair)

data/lib/spark/serializer/text.rb

@@ -0,0 +1,25 @@
+module Spark
+  module Serializer
+    class Text < Base
+
+      attr_reader :encoding
+
+      def initialize(encoding=Encoding::UTF_8)
+        error('Encoding must be an instance of Encoding') unless encoding.is_a?(Encoding)
+
+        @encoding = encoding
+      end
+
+      def load(data)
+        data.to_s.force_encoding(@encoding)
+      end
+
+      def to_s
+        "Text(#{@encoding})"
+      end
+
+    end
+  end
+end
+
+Spark::Serializer.register('string', 'text', Spark::Serializer::Text)

data/lib/spark/version.rb

@@ -1,3 +1,3 @@
 module Spark
-  VERSION = '1.0.0'
+  VERSION = '1.1.0.1'
 end

data/lib/spark/worker/worker.rb

@@ -73,13 +73,16 @@ module Worker
         @command = socket.read_data
 
         # Load iterator
-        @iterator = @command.deserializer.load(socket).lazy
+        @iterator = @command.deserializer.load_from_io(socket).lazy
 
         # Compute
         @iterator = @command.execute(@iterator, @split_index)
 
+        # Result is not iterable
+        @iterator = [@iterator] unless @iterator.respond_to?(:each)
+
         # Send result
-        @command.serializer.dump(@iterator, socket)
+        @command.serializer.dump_to_io(@iterator, socket)
       end
 
       def send_error(e)

data/ruby-spark.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.version       = Spark::VERSION
   spec.authors       = ['Ondřej Moravčík']
   spec.email         = ['moravcik.ondrej@gmail.com']
-  spec.summary       = %q{Ruby wrapper for Spark}
+  spec.summary       = %q{Ruby wrapper for Apache Spark}
   spec.description   = %q{}
   spec.homepage      = ''
   spec.license       = 'MIT'
@@ -31,6 +31,18 @@ Gem::Specification.new do |spec|
 
   spec.requirements << 'java, scala'
 
+  spec.add_dependency 'sourcify', '0.6.0.rc4'
+  spec.add_dependency 'method_source'
+  spec.add_dependency 'commander'
+  spec.add_dependency 'pry'
+  spec.add_dependency 'nio4r'
+  spec.add_dependency 'distribution'
+
+  if RUBY_PLATFORM =~ /java/
+  else
+    spec.add_dependency 'rjb'
+  end
+
   spec.add_development_dependency 'bundler', '~> 1.6'
   spec.add_development_dependency 'rake'
 end

data/spec/lib/context_spec.rb

@@ -7,7 +7,9 @@ RSpec.describe Spark::Context do
     numbers = (0...100).to_a
     func = lambda{|part| part.size}
 
-    rdd = $sc.parallelize(numbers, workers, batch_size: 1)
+    ser = Spark::Serializer.build { __batched__(__marshal__, 1) }
+
+    rdd = $sc.parallelize(numbers, workers, ser)
 
     rdd_result = $sc.run_job(rdd, func)
     result = numbers.each_slice(numbers.size/workers).map(&func)

data/spec/lib/manipulation_spec.rb

@@ -8,7 +8,9 @@ RSpec::describe "Spark::RDD" do
     rdd = $sc.parallelize(numbers, 1).glom
     expect(rdd.collect).to eql([numbers.to_a])
 
-    rdd = $sc.parallelize(numbers, 5, batch_size: 1).glom
+    ser = Spark::Serializer.build { __batched__(__marshal__, 1) }
+
+    rdd = $sc.parallelize(numbers, 5, ser).glom
     expect(rdd.collect).to eql(numbers.each_slice(20).to_a)
   end
 
@@ -42,9 +44,9 @@ RSpec::describe "Spark::RDD" do
     end
 
     it "with a different serializer" do
-      rdd1 = $sc.parallelize(numbers, 1, serializer: "marshal")
-      rdd2 = $sc.parallelize(numbers, 1, serializer: "oj")
-      
+      rdd1 = $sc.parallelize(numbers, 1, Spark::Serializer.build{ __batched__(__marshal__) })
+      rdd2 = $sc.parallelize(numbers, 1, Spark::Serializer.build{ __batched__(__oj__) })
+
       expect { rdd1.union(rdd2).collect }.to_not raise_error
     end
 
@@ -59,14 +61,15 @@ RSpec::describe "Spark::RDD" do
   it ".compact" do
     data = [nil, nil , 0, 0, 1, 2, nil, 6]
     result = data.compact
+    ser = Spark::Serializer.build { __batched__(__marshal__, 1) }
 
     rdd = $sc.parallelize(data, 1).compact
     expect(rdd.collect).to eql(result)
 
-    rdd = $sc.parallelize(data, 5, batch_size: 1).compact
+    rdd = $sc.parallelize(data, 5, ser).compact
     expect(rdd.collect).to eql(result)
 
-    rdd = $sc.parallelize(data, 1, batch_size: 1).compact
+    rdd = $sc.parallelize(data, 1, ser).compact
     expect(rdd.collect).to eql(result)
   end
 
@@ -93,8 +96,10 @@ RSpec::describe "Spark::RDD" do
     let(:result) { data1.product(data2).map(&:to_s).sort }
 
     it "unbatched" do
-      rdd1 = $sc.parallelize(data1, 2, batch_size: 1)
-      rdd2 = $sc.parallelize(data2, 2, batch_size: 1)
+      ser = Spark::Serializer.build { __batched__(__marshal__, 1) }
+
+      rdd1 = $sc.parallelize(data1, 2, ser)
+      rdd2 = $sc.parallelize(data2, 2, ser)
 
       rdd = rdd1.cartesian(rdd2).map(lambda{|x| x.to_s})
 
@@ -102,8 +107,11 @@ RSpec::describe "Spark::RDD" do
     end
 
     it "batched" do
-      rdd1 = $sc.parallelize(data1, 2, batch_size: rand(4..10))
-      rdd2 = $sc.parallelize(data2, 2, batch_size: rand(4..10))
+      ser1 = Spark::Serializer.build { __batched__(__marshal__, rand(4..10)) }
+      ser2 = Spark::Serializer.build { __batched__(__marshal__, rand(4..10)) }
+
+      rdd1 = $sc.parallelize(data1, 2, ser1)
+      rdd2 = $sc.parallelize(data2, 2, ser2)
 
       rdd = rdd1.cartesian(rdd2).map(lambda{|x| x.to_s})
 

data/spec/lib/map_partitions_spec.rb

@@ -1,18 +1,18 @@
-require "spec_helper"
+require 'spec_helper'
 
 def func3(x)
   x.map(&:to_i).reduce(:+)
 end
 
 def func4_with_index(data, index)
-  {
+  [{
     index => data.map(&:to_i).reduce(:*)
-  }
+  }]
 end
 
-RSpec::shared_examples "a map partitions" do |workers|
+RSpec.shared_examples 'a map partitions' do |workers|
   context "with #{workers || 'default'} worker" do
-    it "without index" do
+    it 'without index' do
       rdd2 = rdd(workers).map_partitions(func1)
       result = func1.call(numbers)
 
@@ -35,7 +35,7 @@ RSpec::shared_examples "a map partitions" do |workers|
       expect(rdd4.collect).to eql(rdd3.collect)
     end
 
-    it "with index" do
+    it 'with index' do
       rdd2 = rdd(workers).map_partitions_with_index(method(:func4_with_index))
       result = rdd2.collect
 
@@ -52,7 +52,7 @@ RSpec::shared_examples "a map partitions" do |workers|
   end
 end
 
-RSpec::describe "Spark::RDD.map_partitions(_with_index)" do
+RSpec::describe 'Spark::RDD.map_partitions(_with_index)' do
   let(:func1) { lambda{|x| x.map(&:to_i)} }
   let(:func2) {
     lambda{|x|
@@ -60,28 +60,28 @@ RSpec::describe "Spark::RDD.map_partitions(_with_index)" do
     }
   }
 
-  context "throught parallelize" do
+  context 'throught parallelize' do
     let(:numbers) { 0..1000 }
 
     def rdd(workers)
       $sc.parallelize(numbers, workers)
     end
 
-    it_behaves_like "a map partitions", nil
-    it_behaves_like "a map partitions", 1
-    it_behaves_like "a map partitions", rand(2..10)
+    it_behaves_like 'a map partitions', nil
+    it_behaves_like 'a map partitions', 1
+    it_behaves_like 'a map partitions', rand(2..10)
   end
 
-  context "throught text_file" do
-    let(:file)    { File.join("spec", "inputs", "numbers_0_100.txt") }
+  context 'throught text_file' do
+    let(:file)    { File.join('spec', 'inputs', 'numbers_0_100.txt') }
     let(:numbers) { File.readlines(file).map(&:strip) }
 
     def rdd(workers)
       $sc.text_file(file, workers)
     end
 
-    it_behaves_like "a map partitions", nil
-    it_behaves_like "a map partitions", 1
-    it_behaves_like "a map partitions", rand(2..10)
+    it_behaves_like 'a map partitions', nil
+    it_behaves_like 'a map partitions', 1
+    it_behaves_like 'a map partitions', rand(2..10)
   end
 end

data/spec/lib/serializer_spec.rb

@@ -1,13 +1,88 @@
-require "spec_helper"
+require 'spec_helper'
+require 'zlib'
 
-RSpec::describe Spark::Serializer do
-  
-  it ".get" do
-    expect(described_class.get(nil)).to eql(nil)
-    expect(described_class.get("MARSHAL")).to eql(nil)
-    expect(described_class.get("Marshal")).to eql(described_class::Marshal)
-    expect(described_class.get("marshal")).to eql(described_class::Marshal)
-    expect(described_class.get("message_pack")).to eql(described_class::MessagePack)
+RSpec.describe Spark::Serializer do
+  let(:data) { [1, 'test', 2.0, [3], {key: 'value'}, :test, String] }
+
+  it 'find' do
+    expect(described_class.find('not_existed_class')).to eql(nil)
+
+    expect(described_class.find('Marshal')).to eq(described_class::Marshal)
+    expect(described_class.find('marshal')).to eq(described_class::Marshal)
+    expect(described_class.find(:marshal)).to eq(described_class::Marshal)
+    expect(described_class.find('batched')).to eq(described_class::Batched)
+  end
+
+  it 'find!' do
+    expect { expect(described_class.find!('not_existed_class')) }.to raise_error(Spark::SerializeError)
+    expect { expect(described_class.find!('marshal')) }.to_not raise_error
+    expect { expect(described_class.find!('batched')) }.to_not raise_error
+  end
+
+  it 'register' do
+    NewSerializer = Class.new
+
+    expect(described_class.find('new_serializer_1')).to eql(nil)
+    expect(described_class.find('new_serializer_2')).to eql(nil)
+    expect(described_class.find('new_serializer_3')).to eql(nil)
+
+    described_class.register('new_serializer_1', 'new_serializer_2', 'new_serializer_3', NewSerializer)
+
+    expect(described_class.find('new_serializer_1')).to eql(NewSerializer)
+    expect(described_class.find('new_serializer_2')).to eql(NewSerializer)
+    expect(described_class.find('new_serializer_3')).to eql(NewSerializer)
+  end
+
+  it '==' do
+    # One class
+    marshal1 = described_class::Marshal.new
+    marshal2 = described_class::Marshal.new
+
+    expect(marshal1).to eq(marshal1)
+    expect(marshal1).to eq(marshal2)
+
+    # Two classes
+    compressed1 = described_class::Compressed.new(marshal1)
+    compressed2 = described_class::Compressed.new(marshal2)
+
+    expect(compressed1).to eq(compressed1)
+    expect(compressed1).to eq(compressed2)
+
+    # Three classes
+    batched1 = described_class::Batched.new(compressed1, 1)
+    batched2 = described_class::Batched.new(compressed2, 1)
+    batched3 = described_class::Batched.new(compressed1, 2)
+
+    expect(batched1).to eq(batched2)
+    expect(batched1).to_not eq(batched3)
   end
 
+  context 'build' do
+    let(:marshal1)    { described_class::Marshal.new }
+    let(:compressed1) { described_class::Compressed.new(marshal1) }
+    let(:batched1)    { described_class::Batched.new(compressed1, 1) }
+
+    it 'block' do
+      expect(described_class.build{ marshal }).to eq(marshal1)
+      expect(described_class.build{ marshal }).to eq(described_class.build{ __marshal__ })
+      expect(described_class.build{ compressed(marshal) }).to eq(compressed1)
+      expect(described_class.build{ batched(compressed(marshal), 1) }).to eq(batched1)
+    end
+
+    it 'text' do
+      expect(described_class.build('marshal')).to eq(marshal1)
+      expect(described_class.build('compressed(marshal)')).to eq(compressed1)
+      expect(described_class.build('batched(compressed(marshal), 1)')).to eq(batched1)
+    end
+  end
+
+  it 'serialization' do
+    marshal1 = described_class.build{ marshal }
+    compressed1 = described_class.build{ compressed(marshal) }
+
+    expect(marshal1.dump(data)).to eq(Marshal.dump(data))
+    expect(compressed1.dump(data)).to eq(
+      Zlib::Deflate.deflate(Marshal.dump(data))
+    )
+  end
 end