ruby-spark 1.0.0 → 1.1.0.1

data/spec/lib/statistic_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec::shared_examples 'a stats' do |workers|
+RSpec.shared_examples 'a stats' do |workers|
   let(:numbers) { [1.0, 1.0, 2.0, 3.0, 5.0, 8.0] }
 
   context "with #{workers || 'default'} worker" do
@@ -29,23 +29,23 @@ RSpec::shared_examples 'a stats' do |workers|
   end
 end
 
-RSpec::shared_examples 'a histogram' do |workers|
+RSpec.shared_examples 'a histogram' do |workers|
 
   context "with #{workers || 'default'} worker" do
     it 'empty' do
-      rdd = $sc.parallelize([], workers, batch_size: 1)
+      rdd = $sc.parallelize([], workers, ser)
 
       expect( rdd.histogram([0, 10])[1] ).to eq([0])
       expect( rdd.histogram([0, 4, 10])[1] ).to eq([0, 0])
     end
 
     it 'validation' do
-      rdd = $sc.parallelize([], workers, batch_size: 1)
+      rdd = $sc.parallelize([], workers, ser)
       expect { rdd.histogram(0) }.to raise_error(ArgumentError)
     end
 
     it 'double' do
-      rdd = $sc.parallelize([1.0, 2.0, 3.0, 4.0], workers, batch_size: 1)
+      rdd = $sc.parallelize([1.0, 2.0, 3.0, 4.0], workers, ser)
       buckets, counts = rdd.histogram(2)
 
       expect(buckets).to eq([1.0, 2.5, 4.0])
@@ -53,91 +53,91 @@ RSpec::shared_examples 'a histogram' do |workers|
     end
 
     it 'out of range' do
-      rdd = $sc.parallelize([10.01, -0.01], workers, batch_size: 1)
+      rdd = $sc.parallelize([10.01, -0.01], workers, ser)
 
       expect( rdd.histogram([0, 10])[1] ).to eq([0])
       expect( rdd.histogram([0, 4, 10])[1] ).to eq([0, 0])
     end
 
     it 'in range with one bucket' do
-      rdd = $sc.parallelize([1, 2, 3, 4], workers, batch_size: 1)
+      rdd = $sc.parallelize([1, 2, 3, 4], workers, ser)
 
       expect( rdd.histogram([0, 10])[1] ).to eq([4])
       expect( rdd.histogram([0, 4, 10])[1] ).to eq([3, 1])
     end
 
     it 'in range with one bucket exact match' do
-      rdd = $sc.parallelize([1, 2, 3, 4], workers, batch_size: 1)
+      rdd = $sc.parallelize([1, 2, 3, 4], workers, ser)
       expect( rdd.histogram([1, 4])[1] ).to eq([4])
     end
 
     it 'out of range with two buckets' do
-      rdd = $sc.parallelize([10.01, -0.01], workers, batch_size: 1)
+      rdd = $sc.parallelize([10.01, -0.01], workers, ser)
       expect( rdd.histogram([0, 5, 10])[1] ).to eq([0, 0])
     end
 
     it 'out of range with two uneven buckets' do
-      rdd = $sc.parallelize([10.01, -0.01], workers, batch_size: 1)
+      rdd = $sc.parallelize([10.01, -0.01], workers, ser)
       expect( rdd.histogram([0, 4, 10])[1] ).to eq([0, 0])
     end
 
     it 'in range with two buckets' do
-      rdd = $sc.parallelize([1, 2, 3, 5, 6], workers, batch_size: 1)
+      rdd = $sc.parallelize([1, 2, 3, 5, 6], workers, ser)
       expect( rdd.histogram([0, 5, 10])[1] ).to eq([3, 2])
     end
 
     it 'in range with two bucket and nil' do
-      rdd = $sc.parallelize([1, 2, 3, 5, 6, nil, Float::NAN], workers, batch_size: 1)
+      rdd = $sc.parallelize([1, 2, 3, 5, 6, nil, Float::NAN], workers, ser)
       expect( rdd.histogram([0, 5, 10])[1] ).to eq([3, 2])
     end
 
     it 'in range with two uneven buckets' do
-      rdd = $sc.parallelize([1, 2, 3, 5, 6], workers, batch_size: 1)
+      rdd = $sc.parallelize([1, 2, 3, 5, 6], workers, ser)
       expect( rdd.histogram([0, 5, 11])[1] ).to eq([3, 2])
     end
 
     it 'mixed range with two uneven buckets' do
-      rdd = $sc.parallelize([-0.01, 0.0, 1, 2, 3, 5, 6, 11.0, 11.01], workers, batch_size: 1)
+      rdd = $sc.parallelize([-0.01, 0.0, 1, 2, 3, 5, 6, 11.0, 11.01], workers, ser)
       expect( rdd.histogram([0, 5, 11])[1] ).to eq([4, 3])
     end
 
     it 'mixed range with four uneven buckets' do
-      rdd = $sc.parallelize([-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0, 200.0, 200.1], workers, batch_size: 1)
+      rdd = $sc.parallelize([-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0, 200.0, 200.1], workers, ser)
       expect( rdd.histogram([0.0, 5.0, 11.0, 12.0, 200.0])[1] ).to eq([4, 2, 1, 3])
     end
 
     it 'mixed range with uneven buckets and NaN' do
-      rdd = $sc.parallelize([-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0, 200.0, 200.1, nil, Float::NAN], workers, batch_size: 1)
+      rdd = $sc.parallelize([-0.01, 0.0, 1, 2, 3, 5, 6, 11.01, 12.0, 199.0, 200.0, 200.1, nil, Float::NAN], workers, ser)
       expect( rdd.histogram([0.0, 5.0, 11.0, 12.0, 200.0])[1] ).to eq([4, 2, 1, 3])
     end
 
     it 'out of range with infinite buckets' do
-      rdd = $sc.parallelize([10.01, -0.01, Float::NAN, Float::INFINITY], workers, batch_size: 1)
+      rdd = $sc.parallelize([10.01, -0.01, Float::NAN, Float::INFINITY], workers, ser)
       expect( rdd.histogram([-Float::INFINITY, 0, Float::INFINITY])[1] ).to eq([1, 1])
     end
 
     it 'without buckets' do
-      rdd = $sc.parallelize([1, 2, 3, 4], workers, batch_size: 1)
+      rdd = $sc.parallelize([1, 2, 3, 4], workers, ser)
       expect( rdd.histogram(1) ).to eq([[1, 4], [4]])
     end
 
     it 'without buckets single element' do
-      rdd = $sc.parallelize([1], workers, batch_size: 1)
+      rdd = $sc.parallelize([1], workers, ser)
       expect( rdd.histogram(1) ).to eq([[1, 1], [1]])
     end
 
     it 'without bucket no range' do
-      rdd = $sc.parallelize([1, 1, 1, 1], workers, batch_size: 1)
+      rdd = $sc.parallelize([1, 1, 1, 1], workers, ser)
       expect( rdd.histogram(1) ).to eq([[1, 1], [4]])
     end
 
     it 'without buckets basic two' do
-      rdd = $sc.parallelize([1, 2, 3, 4], workers, batch_size: 1)
+      rdd = $sc.parallelize([1, 2, 3, 4], workers, ser)
       expect( rdd.histogram(2) ).to eq([[1, 2.5, 4], [2, 2]])
     end
 
     it 'without buckets with more requested than elements' do
-      rdd = $sc.parallelize([1, 2], workers, batch_size: 1)
+      rdd = $sc.parallelize([1, 2], workers, ser)
       buckets = [1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0]
       hist = [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]
 
@@ -145,7 +145,7 @@ RSpec::shared_examples 'a histogram' do |workers|
     end
 
     it 'string' do
-      rdd = $sc.parallelize(['ab', 'ac', 'b', 'bd', 'ef'], workers, batch_size: 1)
+      rdd = $sc.parallelize(['ab', 'ac', 'b', 'bd', 'ef'], workers, ser)
 
       expect( rdd.histogram(['a', 'b', 'c'])[1] ).to eq([2, 2])
       expect( rdd.histogram(1) ).to eq([['ab', 'ef'], [5]])
@@ -155,7 +155,9 @@ RSpec::shared_examples 'a histogram' do |workers|
   end
 end
 
-RSpec::describe Spark::RDD do
+RSpec.describe Spark::RDD do
+  let(:ser) { Spark::Serializer.build { __batched__(__marshal__, 1) } }
+
   context '.stats' do
     it_behaves_like 'a stats', 1
     it_behaves_like 'a stats', rand(2..5)

data/spec/spec_helper.rb

@@ -14,8 +14,7 @@ Spark::Mllib.import
 def spark_start
   Spark.logger.disable
   Spark.config do
-    set 'spark.ruby.parallelize_strategy', 'deep_copy'
-    set 'spark.ruby.batch_size', 100
+    set 'spark.ruby.serializer.batch_size', 100
   end
   Spark.start
   $sc = Spark.context

metadata

@@ -1,15 +1,113 @@
 --- !ruby/object:Gem::Specification
 name: ruby-spark
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0.1
 platform: ruby
 authors:
 - Ondřej Moravčík
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-04 00:00:00.000000000 Z
+date: 2015-05-16 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: sourcify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.6.0.rc4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.6.0.rc4
+- !ruby/object:Gem::Dependency
+  name: method_source
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: commander
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nio4r
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: distribution
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rjb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -56,21 +154,22 @@ files:
 - TODO.md
 - benchmark/aggregate.rb
 - benchmark/bisect.rb
+- benchmark/comparison/prepare.sh
+- benchmark/comparison/python.py
+- benchmark/comparison/r.r
+- benchmark/comparison/ruby.rb
+- benchmark/comparison/run-all.sh
+- benchmark/comparison/scala.scala
 - benchmark/custom_marshal.rb
 - benchmark/digest.rb
 - benchmark/enumerator.rb
-- benchmark/performance/prepare.sh
-- benchmark/performance/python.py
-- benchmark/performance/r.r
-- benchmark/performance/ruby.rb
-- benchmark/performance/run-all.sh
-- benchmark/performance/scala.scala
 - benchmark/serializer.rb
 - benchmark/sort.rb
 - benchmark/sort2.rb
 - benchmark/take.rb
 - bin/ruby-spark
 - example/pi.rb
+- example/website_search.rb
 - ext/ruby_c/extconf.rb
 - ext/ruby_c/murmur.c
 - ext/ruby_c/murmur.h
@@ -155,13 +254,16 @@ files:
 - lib/spark/rdd.rb
 - lib/spark/sampler.rb
 - lib/spark/serializer.rb
+- lib/spark/serializer/auto_batched.rb
 - lib/spark/serializer/base.rb
+- lib/spark/serializer/batched.rb
 - lib/spark/serializer/cartesian.rb
+- lib/spark/serializer/compressed.rb
 - lib/spark/serializer/marshal.rb
 - lib/spark/serializer/message_pack.rb
 - lib/spark/serializer/oj.rb
 - lib/spark/serializer/pair.rb
-- lib/spark/serializer/utf8.rb
+- lib/spark/serializer/text.rb
 - lib/spark/sort.rb
 - lib/spark/stat_counter.rb
 - lib/spark/storage_level.rb
@@ -245,7 +347,7 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: Ruby wrapper for Spark
+summary: Ruby wrapper for Apache Spark
 test_files:
 - spec/generator.rb
 - spec/inputs/lorem_300.txt

data/lib/spark/serializer/utf8.rb

@@ -1,25 +0,0 @@
-module Spark
-  module Serializer
-    ##
-    # Used for file
-    #
-    # File is sended as String but worker use serialization
-    #
-    class UTF8 < Base
-
-      def set(*)
-        unbatch!
-        self
-      end
-
-      def batched?
-        false
-      end
-
-      def load_next_from_io(io, lenght)
-        io.read(lenght).force_encoding(Encoding::UTF_8)
-      end
-
-    end
-  end
-end