ruby-spark 1.1.0.1 → 1.2.0

data/spec/lib/reduce_by_key_spec.rb

@@ -1,4 +1,4 @@
-require "spec_helper"
+require 'spec_helper'
 
 def flat_map(line)
   line.split
@@ -12,7 +12,7 @@ def reduce(x,y)
   x+y
 end
 
-RSpec::shared_examples "a words counting" do |workers|
+RSpec.shared_examples 'a words counting' do |workers|
   context "with #{workers || 'default'} worker" do
     let(:result) do
       keyyed = lines.flat_map{|x| x.split}.map{|x| [x,1]}
@@ -27,7 +27,7 @@ RSpec::shared_examples "a words counting" do |workers|
       result
     end
 
-    it "when lambda" do
+    it 'when lambda' do
       rdd2 = rdd(workers)
       rdd2 = rdd2.flat_map(lambda{|line| line.split})
       rdd2 = rdd2.map(lambda{|word| [word, 1]})
@@ -36,7 +36,7 @@ RSpec::shared_examples "a words counting" do |workers|
       expect(rdd2.collect_as_hash).to eql(result)
     end
 
-    it "when method" do
+    it 'when method' do
       rdd2 = rdd(workers)
       rdd2 = rdd2.flat_map(method(:flat_map))
       rdd2 = rdd2.map(method(:map))
@@ -45,7 +45,7 @@ RSpec::shared_examples "a words counting" do |workers|
       expect(rdd2.collect_as_hash).to eql(result)
     end
 
-    it "keys, values" do
+    it 'keys, values' do
       rdd2 = rdd(workers)
       rdd2 = rdd2.flat_map(method(:flat_map))
       rdd2 = rdd2.map(method(:map))
@@ -57,35 +57,35 @@ RSpec::shared_examples "a words counting" do |workers|
   end
 end
 
-RSpec::describe "Spark::RDD" do
-  context ".reduce_by_key" do
-    context "throught parallelize" do
+RSpec.describe 'Spark::RDD' do
+  context '.reduce_by_key' do
+    context 'throught parallelize' do
       let(:lines) { Generator.lines }
 
       def rdd(workers)
         $sc.parallelize(lines, workers)
       end
 
-      it_behaves_like "a words counting", nil
-      it_behaves_like "a words counting", 1
-      it_behaves_like "a words counting", rand(2..10)
+      it_behaves_like 'a words counting', 2
+      # it_behaves_like 'a words counting', nil
+      # it_behaves_like 'a words counting', rand(2..10)
     end
 
-    context "throught text_file" do
-      let(:file)  { File.join("spec", "inputs", "lorem_300.txt") }
+    context 'throught text_file' do
+      let(:file)  { File.join('spec', 'inputs', 'lorem_300.txt') }
       let(:lines) { File.readlines(file).map(&:strip) }
 
       def rdd(workers)
         $sc.text_file(file, workers)
       end
 
-      it_behaves_like "a words counting", nil
-      it_behaves_like "a words counting", 1
-      it_behaves_like "a words counting", rand(2..10)
+      it_behaves_like 'a words counting', 2
+      # it_behaves_like 'a words counting', nil
+      # it_behaves_like 'a words counting', rand(2..10)
     end
   end
 
-  context ".fold_by_key" do
+  context '.fold_by_key' do
     let(:numbers)    { Generator.numbers }
     let(:zero_value) { 0 }
     let(:rdd)        { $sc.parallelize(numbers) }
@@ -105,11 +105,11 @@ RSpec::describe "Spark::RDD" do
       rdd.map(map).fold_by_key(zero_value, add, num_partitions).collect_as_hash
     end
 
-    it "default num_partitions" do
+    it 'default num_partitions' do
       expect(fold_by_key).to eq(result)
     end
 
-    it "default num_partitions" do
+    it 'default num_partitions' do
       expect(
         fold_by_key rand(1..10)
       ).to eq(result)

data/spec/lib/reduce_spec.rb

@@ -1,12 +1,12 @@
-require "spec_helper"
+require 'spec_helper'
 
 def longest_words(memo, word)
   memo.length > word.length ? memo : word
 end
 
-RSpec::shared_examples "a reducing" do |workers|
+RSpec.shared_examples 'a reducing' do |workers|
   context "with #{workers || 'default'} worker" do
-    it ".reduce" do
+    it '.reduce' do
       rdd2 = rdd_numbers(workers)
       rdd2 = rdd2.map(to_i)
       rdd2 = rdd2.reduce(func1)
@@ -30,7 +30,7 @@ RSpec::shared_examples "a reducing" do |workers|
       expect(rdd4).to eql(result)
     end
 
-    it ".fold" do
+    it '.fold' do
       rdd2 = rdd_numbers(workers)
       rdd2 = rdd2.map(to_i)
       rdd_result = rdd2.fold(1, func1)
@@ -41,7 +41,7 @@ RSpec::shared_examples "a reducing" do |workers|
       expect(rdd_result).to eql(result)
     end
 
-    it ".aggregate" do
+    it '.aggregate' do
       rdd2 = rdd_numbers(workers)
       rdd2 = rdd2.map(to_i)
 
@@ -55,28 +55,28 @@ RSpec::shared_examples "a reducing" do |workers|
       expect(rdd_result).to eql(result)
     end
 
-    it ".max" do
+    it '.max' do
       rdd2 = rdd_numbers(workers)
       rdd2 = rdd2.map(to_i)
 
       expect(rdd2.max).to eql(numbers.map(&:to_i).max)
     end
 
-    it ".min" do
+    it '.min' do
       rdd2 = rdd_numbers(workers)
       rdd2 = rdd2.map(to_i)
 
       expect(rdd2.min).to eql(numbers.map(&:to_i).min)
     end
 
-    it ".sum" do
+    it '.sum' do
       rdd2 = rdd_numbers(workers)
       rdd2 = rdd2.map(to_i)
 
       expect(rdd2.sum).to eql(numbers.map(&:to_i).reduce(:+))
     end
 
-    it ".count" do
+    it '.count' do
       rdd2 = rdd_numbers(workers)
       rdd2 = rdd2.map(to_i)
 
@@ -85,14 +85,14 @@ RSpec::shared_examples "a reducing" do |workers|
   end
 end
 
-RSpec::describe "Spark::RDD" do
+RSpec.describe 'Spark::RDD' do
   let(:func1) { lambda{|sum, x| sum+x} }
   let(:func2) { lambda{|product, x| product*x} }
 
   let(:to_i)  { lambda{|item| item.to_i} }
   let(:split) { lambda{|item| item.split} }
 
-  context "throught parallelize" do
+  context 'throught parallelize' do
     let(:numbers) { Generator.numbers }
     let(:lines)   { Generator.lines }
 
@@ -104,14 +104,15 @@ RSpec::describe "Spark::RDD" do
       $sc.parallelize(lines, workers)
     end
 
-    it_behaves_like "a reducing", nil
-    it_behaves_like "a reducing", 1
-    it_behaves_like "a reducing", rand(2..10)
+    it_behaves_like 'a reducing', 1
+    it_behaves_like 'a reducing', 2
+    # it_behaves_like 'a reducing', nil
+    # it_behaves_like 'a reducing', rand(2..10)
   end
 
-  context "throught text_file" do
-    let(:file)       { File.join("spec", "inputs", "numbers_0_100.txt") }
-    let(:file_lines) { File.join("spec", "inputs", "lorem_300.txt") }
+  context 'throught text_file' do
+    let(:file)       { File.join('spec', 'inputs', 'numbers_0_100.txt') }
+    let(:file_lines) { File.join('spec', 'inputs', 'lorem_300.txt') }
 
     let(:numbers) { File.readlines(file).map(&:strip).map(&:to_i) }
     let(:lines)   { File.readlines(file_lines).map(&:strip) }
@@ -124,8 +125,9 @@ RSpec::describe "Spark::RDD" do
       $sc.text_file(file_lines, workers)
     end
 
-    it_behaves_like "a reducing", nil
-    it_behaves_like "a reducing", 1
-    it_behaves_like "a reducing", rand(2..10)
+    it_behaves_like 'a reducing', 1
+    it_behaves_like 'a reducing', 2
+    # it_behaves_like 'a reducing', nil
+    # it_behaves_like 'a reducing', rand(2..10)
   end
 end

data/spec/lib/sample_spec.rb

@@ -1,30 +1,30 @@
-require "spec_helper"
+require 'spec_helper'
 
 # Sample method can not be tested because of random generator
 # Just test it for raising error
 
-RSpec::shared_examples "a sampler" do |workers|
+RSpec.shared_examples 'a sampler' do |workers|
   context "with #{workers || 'default'} worker" do
 
-    context ".sample" do
-      it "with replacement" do
+    context '.sample' do
+      it 'with replacement' do
         rdd2 = rdd(workers).sample(true, rand)
         expect { rdd2.collect }.to_not raise_error
       end
 
-      it "without replacement" do
+      it 'without replacement' do
         rdd2 = rdd(workers).sample(false, rand)
         expect { rdd2.collect }.to_not raise_error
       end
     end
 
-    context ".take_sample" do
-      it "with replacement" do
+    context '.take_sample' do
+      it 'with replacement' do
         size = rand(10..999)
         expect(rdd(workers).take_sample(true, size).size).to eql(size)
       end
 
-      it "without replacement" do
+      it 'without replacement' do
         size = rand(10..999)
         expect(rdd(workers).take_sample(false, size).size).to eql(size)
       end
@@ -33,14 +33,15 @@ RSpec::shared_examples "a sampler" do |workers|
   end
 end
 
-RSpec::describe "Spark::RDD" do
+RSpec.describe 'Spark::RDD' do
   let(:numbers) { Generator.numbers(1000) }
 
   def rdd(workers)
     $sc.parallelize(numbers, workers)
   end
 
-  it_behaves_like "a sampler", nil
-  it_behaves_like "a sampler", 1
-  it_behaves_like "a sampler", rand(2..10)
+  it_behaves_like 'a sampler', 1
+  it_behaves_like 'a sampler', 2
+  # it_behaves_like 'a sampler', nil
+  # it_behaves_like 'a sampler', rand(2..10)
 end

data/spec/lib/serializer_spec.rb

@@ -85,4 +85,31 @@ RSpec.describe Spark::Serializer do
       Zlib::Deflate.deflate(Marshal.dump(data))
     )
   end
+
+  context 'Auto batched' do
+    let(:klass) { Spark::Serializer::AutoBatched }
+    let(:marshal) { Spark::Serializer::Marshal.new }
+    let(:numbers) { Generator.numbers }
+
+    it 'initialize' do
+      expect { klass.new }.to raise_error(ArgumentError)
+      expect { klass.new(marshal) }.to_not raise_error
+      expect { klass.new(marshal, 1) }.to raise_error(Spark::SerializeError)
+    end
+
+    it 'serialization' do
+      serializer1 = klass.new(marshal)
+      serializer2 = klass.new(marshal, 2)
+
+      rdd1 = Spark.sc.parallelize(numbers, 2, serializer1)
+      rdd2 = Spark.sc.parallelize(numbers, 2, serializer2).map(:to_i)
+
+      result = rdd1.collect
+
+      expect(rdd1.serializer).to eq(serializer1)
+      expect(result).to eq(numbers)
+      expect(result).to eq(rdd2.collect)
+    end
+
+  end
 end

data/spec/lib/sort_spec.rb

@@ -1,6 +1,6 @@
-require "spec_helper"
+require 'spec_helper'
 
-RSpec::shared_examples "a sorting" do |workers|
+RSpec.shared_examples 'a sorting' do |workers|
   it "with #{workers || 'default'} worker" do
     rdd2 = rdd(workers)
 
@@ -22,37 +22,39 @@ RSpec::shared_examples "a sorting" do |workers|
 end
 
 
-RSpec::describe "Spark::RDD" do
+RSpec.describe 'Spark::RDD' do
   let(:split)   { lambda{|x| x.split} }
   let(:map)     { lambda{|x| [x.to_s, 1]} }
   let(:len_map) { lambda{|x| [x.size, x]} }
 
-  context "throught parallelize" do
-    context ".map" do
+  context 'throught parallelize' do
+    context '.map' do
       let(:lines) { Generator.lines }
 
       def rdd(workers)
         $sc.parallelize(lines, workers)
       end
 
-      it_behaves_like "a sorting", nil
-      it_behaves_like "a sorting", 1
-      it_behaves_like "a sorting", rand(2..10)
+      it_behaves_like 'a sorting', 1
+      it_behaves_like 'a sorting', 2
+      # it_behaves_like 'a sorting', nil
+      # it_behaves_like 'a sorting', rand(2..10)
     end
   end
 
-  context "throught text_file" do
-    context ".map" do
-      let(:file)  { File.join("spec", "inputs", "lorem_300.txt") }
+  context 'throught text_file' do
+    context '.map' do
+      let(:file)  { File.join('spec', 'inputs', 'lorem_300.txt') }
       let(:lines) { File.readlines(file).map(&:strip) }
 
       def rdd(workers)
         $sc.text_file(file, workers)
       end
 
-      it_behaves_like "a sorting", nil
-      it_behaves_like "a sorting", 1
-      it_behaves_like "a sorting", rand(2..10)
+      it_behaves_like 'a sorting', 1
+      it_behaves_like 'a sorting', 2
+      # it_behaves_like 'a sorting', nil
+      # it_behaves_like 'a sorting', rand(2..10)
     end
   end
 end

data/spec/lib/statistic_spec.rb

@@ -160,11 +160,13 @@ RSpec.describe Spark::RDD do
 
   context '.stats' do
     it_behaves_like 'a stats', 1
-    it_behaves_like 'a stats', rand(2..5)
+    it_behaves_like 'a stats', 2
+    # it_behaves_like 'a stats', rand(2..5)
   end
 
   context '.histogram' do
     it_behaves_like 'a histogram', 1
-    it_behaves_like 'a histogram', rand(2..5)
+    it_behaves_like 'a histogram', 2
+    # it_behaves_like 'a histogram', rand(2..5)
   end
 end

data/spec/lib/whole_text_files_spec.rb

@@ -1,6 +1,6 @@
-require "spec_helper"
+require 'spec_helper'
 
-RSpec::shared_examples "a whole_text_files" do |workers|
+RSpec.shared_examples 'a whole_text_files' do |workers|
   it "with #{workers || 'default'} worker" do
     rdd2 = rdd(workers).map(get_numbers)
     result = files.size
@@ -17,17 +17,18 @@ RSpec::shared_examples "a whole_text_files" do |workers|
   end
 end
 
-RSpec::describe "Spark::Context" do
+RSpec.describe 'Spark::Context' do
   let(:get_numbers) { lambda{|file, content| content.split.map(&:to_i)} }
 
-  let(:dir)   { File.join("spec", "inputs", "numbers") }
-  let(:files) { Dir.glob(File.join(dir, "*")) }
+  let(:dir)   { File.join('spec', 'inputs', 'numbers') }
+  let(:files) { Dir.glob(File.join(dir, '*')) }
 
   def rdd(workers)
     $sc.whole_text_files(dir, workers)
   end
 
-  it_behaves_like "a whole_text_files", nil
-  it_behaves_like "a whole_text_files", 1
-  it_behaves_like "a whole_text_files", rand(2..10)
+  it_behaves_like 'a whole_text_files', 1
+  it_behaves_like 'a whole_text_files', 2
+  # it_behaves_like 'a whole_text_files', nil
+  # it_behaves_like 'a whole_text_files', rand(2..10)
 end

data/spec/spec_helper.rb

@@ -1,5 +1,5 @@
-# require 'simplecov'
-# SimpleCov.start
+require 'simplecov'
+SimpleCov.start
 
 $LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
 require 'ruby-spark'
@@ -7,7 +7,7 @@ require 'generator'
 
 # Loading
 Spark.load_lib
-Spark.jb.load_test
+Spark.jb.import_all_test
 Spark::Mllib.import
 
 # Keep it on method because its called from config test

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: ruby-spark
 version: !ruby/object:Gem::Version
-  version: 1.1.0.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Ondřej Moravčík
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-16 00:00:00.000000000 Z
+date: 2015-06-15 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: rjb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sourcify
   requirement: !ruby/object:Gem::Requirement
@@ -94,20 +108,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rjb
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -146,6 +146,8 @@ extensions:
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - Guardfile
 - LICENSE.txt
@@ -344,7 +346,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements:
 - java, scala
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Ruby wrapper for Apache Spark
@@ -400,4 +402,3 @@ test_files:
 - spec/lib/statistic_spec.rb
 - spec/lib/whole_text_files_spec.rb
 - spec/spec_helper.rb
-has_rdoc: