ruby-spark 1.0.0

data/spec/lib/context_spec.rb

@@ -0,0 +1,163 @@
+require 'spec_helper'
+
+RSpec.describe Spark::Context do
+
+  it '.run_job' do
+    workers = 5
+    numbers = (0...100).to_a
+    func = lambda{|part| part.size}
+
+    rdd = $sc.parallelize(numbers, workers, batch_size: 1)
+
+    rdd_result = $sc.run_job(rdd, func)
+    result = numbers.each_slice(numbers.size/workers).map(&func)
+    expect(rdd_result).to eql(result)
+
+    parts = [0, 2]
+    func = lambda{|part| part.to_s}
+
+    rdd_result = $sc.run_job(rdd, func, parts)
+    result = []
+    sliced_numbers = numbers.each_slice(numbers.size/workers).to_a
+    parts.each do |part|
+      result << func.call(sliced_numbers[part])
+    end
+
+    expect(rdd_result).to eql(result)
+  end
+
+  it '.broadcast' do
+    workers = rand(1..5)
+
+    values1 = [1,2,3]
+    values2 = [4,5,6]
+
+    broadcast1 = $sc.broadcast(values1)
+    broadcast2 = $sc.broadcast(values2)
+
+    rdd = $sc.parallelize(0..5, workers)
+    rdd = rdd.bind(broadcast1: broadcast1, broadcast2: broadcast2)
+    rdd = rdd.map_partitions(lambda{|_| broadcast1.value + broadcast2.value })
+
+    expect(rdd.sum).to eql(
+      (values1 + values2).reduce(:+) * workers
+    )
+  end
+
+  context '.accumulator' do
+    it 'test' do
+      accum1 = $sc.accumulator(0,)
+      accum2 = $sc.accumulator(1, :*, 1)
+      accum3 = $sc.accumulator(0, lambda{|max, val| val > max ? val : max})
+
+      accum1 += 1
+
+      accum2.add(2)
+      accum2.add(2)
+      accum2.add(2)
+
+      accum3.add(9)
+      accum3.add(6)
+      accum3.add(7)
+
+      expect(accum1.value).to eql(1)
+      expect(accum2.value).to eql(8)
+      expect(accum3.value).to eql(9)
+
+      func = Proc.new do |_, index|
+        accum1.add(1)
+        accum2.add(2)
+        accum3.add(index * 10)
+      end
+
+      rdd = $sc.parallelize(0..4, 4)
+      rdd = rdd.bind(accum1: accum1, accum2: accum2, accum3: accum3)
+      rdd = rdd.map_partitions_with_index(func)
+      rdd.collect
+
+      # http://stackoverflow.com/questions/28560133/ruby-server-java-scala-client-deadlock
+      sleep(1)
+
+      expect(accum1.value).to eql(5)
+      expect(accum2.value).to eql(128)
+      expect(accum3.value).to eql(30)
+    end
+
+    context 'accum param' do
+      it 'symbol' do
+        accum1 = $sc.accumulator(1, :+, 0)
+        accum2 = $sc.accumulator(5, :-, 3)
+        accum3 = $sc.accumulator(1, :*, 1)
+        accum4 = $sc.accumulator(1.0, :/, 1.0)
+        accum5 = $sc.accumulator(2, :**, 2)
+
+        func = Proc.new do |_|
+          accum1.add(1)
+          accum2.add(1)
+          accum3.add(2)
+          accum4.add(2)
+          accum5.add(2)
+        end
+
+        rdd = $sc.parallelize(0..4, 2)
+        rdd = rdd.bind(accum1: accum1, accum2: accum2, accum3: accum3, accum4: accum4, accum5: accum5)
+        rdd = rdd.map_partitions(func)
+        rdd.collect
+
+        # http://stackoverflow.com/questions/28560133/ruby-server-java-scala-client-deadlock
+        sleep(1)
+
+        expect(accum1.value).to eq(3)
+        expect(accum2.value).to eq(1)
+        expect(accum3.value).to eq(4)
+        expect(accum4.value).to eq(4)
+        expect(accum5.value).to eq(65536)
+      end
+
+      it 'proc' do
+        accum1 = $sc.accumulator(1, lambda{|mem, val| mem + val}, 0)
+        accum2 = $sc.accumulator('a', lambda{|mem, val| mem + val}, '')
+        accum3 = $sc.accumulator([], lambda{|mem, val| mem << val}, [])
+
+        func = Proc.new do |_|
+          accum1.add(1)
+          accum2.add('a')
+          accum3.add(1)
+        end
+
+        rdd = $sc.parallelize(0..4, 2)
+        rdd = rdd.bind(accum1: accum1, accum2: accum2, accum3: accum3)
+        rdd = rdd.map_partitions(func)
+        rdd.collect
+
+        # http://stackoverflow.com/questions/28560133/ruby-server-java-scala-client-deadlock
+        sleep(1)
+
+        expect(accum1.value).to eq(3)
+        expect(accum2.value).to eq('aaa')
+        expect(accum3.value).to eq([[1], [1]])
+      end
+
+      it 'string' do
+        expect { $sc.accumulator(1, '0') }.to raise_error(Spark::SerializeError)
+
+        accum = $sc.accumulator(1, 'lambda{|mem, val| mem + val}', 0)
+
+        func = Proc.new do |_|
+          accum.add(1)
+        end
+
+        rdd = $sc.parallelize(0..4, 2)
+        rdd = rdd.bind(accum: accum)
+        rdd = rdd.map_partitions(func)
+        rdd.collect
+
+        # http://stackoverflow.com/questions/28560133/ruby-server-java-scala-client-deadlock
+        sleep(1)
+
+        expect(accum.value).to eq(3)
+      end
+    end
+  end
+
+end

data/spec/lib/ext_spec.rb

@@ -0,0 +1,72 @@
+require "spec_helper"
+
+RSpec.describe Array do
+
+  it ".deep_copy" do
+    data = ["a", "b", "c"]
+    new_data = data.dup
+
+    data[0] << "a"
+
+    expect(data).to eql(new_data)
+
+    new_data = data.deep_copy
+
+    data[1] << "b"
+
+    expect(data).to_not eql(new_data)
+  end
+
+end
+
+RSpec.describe Hash do
+
+  it ".stringify_keys!" do
+    data = {
+      a: "a",
+      b: "b",
+      c: "c"
+    }
+
+    data.stringify_keys!
+
+    expect(data).to eql({
+      "a" => "a",
+      "b" => "b",
+      "c" => "c"
+    })
+  end
+
+end
+
+RSpec.describe String do
+
+  it ".camelize" do
+    data = "aaa_bbb_ccc".camelize
+    expect(data).to eql("AaaBbbCcc")
+  end
+
+end
+
+RSpec.describe IO do
+
+  it 'serialize' do
+    file = Tempfile.new('serialize')
+    file.binmode
+
+    file.write_int(1)
+    file.write_long(2)
+    file.write_string("3")
+    file.write_data([4])
+
+    file.rewind
+
+    expect(file.read_int).to eq(1)
+    expect(file.read_long).to eq(2)
+    expect(file.read_string).to eq("3")
+    expect(file.read_data).to eq([4])
+
+    file.unlink
+  end
+
+end

data/spec/lib/external_apps_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+RSpec::describe Spark::RDD do
+
+  context '.pipe' do
+    let(:words)   { Generator.words }
+    let(:numbers) { Generator.numbers }
+
+    it 'single program' do
+      skip if windows?
+
+      rdd = $sc.parallelize(words, 1)
+      rdd = rdd.pipe('tr a b')
+
+      result = words.dup
+      result.map! do |x|
+        x.gsub('a', 'b')
+      end
+
+      expect(rdd.collect).to eql(result)
+    end
+
+    it 'multiple program' do
+      skip if windows?
+
+      rdd = $sc.parallelize(numbers, 1)
+      rdd = rdd.pipe("tr 1 5", "awk '{print $1*10}'")
+      rdd = rdd.map(lambda{|x| x.to_i * 100})
+
+      result = numbers.dup
+      result.map! do |x|
+        x.to_s.gsub('1', '5')
+      end
+      result.map! do |x|
+        x.to_i * 10
+      end
+      result.map! do |x|
+        x * 100
+      end
+
+      expect(rdd.collect).to eql(result)
+    end
+  end
+
+end

data/spec/lib/filter_spec.rb

@@ -0,0 +1,80 @@
+require "spec_helper"
+
+def func4(item)
+  item.start_with?("a") && item.size > 3 && item[1].to_s.ord > 106
+end
+
+RSpec::shared_examples "a filtering" do |workers|
+  context "with #{workers || 'default'} worker" do
+    it "when numbers" do
+      rdd2 = rdd_numbers(workers)
+      rdd2 = rdd2.filter(func1)
+      result = numbers.select(&func1)
+
+      expect(rdd2.collect).to eql(result)
+
+      rdd3 = rdd_numbers(workers)
+      rdd3 = rdd3.filter(func1)
+      rdd3 = rdd3.filter(func2)
+
+      expect(rdd3.collect).to eql([])
+    end
+
+    it "when words" do
+      rdd2 = rdd_words(workers)
+      rdd2 = rdd2.filter(func3)
+      result = words.select{|x| func3.call(x)}
+
+      expect(rdd2.collect).to eql(result)
+
+      rdd3 = rdd_words(workers)
+      rdd3 = rdd3.filter(method(:func4))
+      result = words.select{|x| func4(x)}
+
+      expect(rdd3.collect).to eql(result)
+    end
+  end
+end
+
+RSpec::describe "Spark::RDD.filter" do
+  let(:func1) { lambda{|x| x.to_i.even?} }
+  let(:func2) { lambda{|x| x.to_i.odd?} }
+  let(:func3) { lambda{|x| x.to_s.start_with?("b")} }
+
+  context "throught parallelize" do
+    let(:numbers) { Generator.numbers_with_zero }
+    let(:words)   { Generator.words }
+
+    def rdd_numbers(workers)
+      $sc.parallelize(numbers, workers)
+    end
+
+    def rdd_words(workers)
+      $sc.parallelize(words, workers)
+    end
+
+    it_behaves_like "a filtering", nil
+    it_behaves_like "a filtering", 1
+    it_behaves_like "a filtering", rand(2..10)
+  end
+
+  context "throught text_file" do
+    let(:file_numbers) { File.join("spec", "inputs", "numbers_0_100.txt") }
+    let(:file_words)   { File.join("spec", "inputs", "lorem_300.txt") }
+
+    let(:numbers) { File.readlines(file_numbers).map(&:strip) }
+    let(:words)   { File.readlines(file_words).map(&:strip) }
+
+    def rdd_numbers(workers)
+      $sc.text_file(file_numbers, workers)
+    end
+
+    def rdd_words(workers)
+      $sc.text_file(file_words, workers)
+    end
+
+    it_behaves_like "a filtering", nil
+    it_behaves_like "a filtering", 1
+    it_behaves_like "a filtering", rand(2..10)
+  end
+end

data/spec/lib/flat_map_spec.rb

@@ -0,0 +1,100 @@
+require "spec_helper"
+
+RSpec::shared_examples "a flat mapping" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd2 = rdd(workers).map(func1)
+    result = numbers.flat_map(&func1)
+
+    expect(rdd2.collect).to eql(result)
+
+    rdd3 = rdd(workers)
+    rdd3 = rdd3.flat_map(func1)
+    rdd3 = rdd3.flat_map(func2)
+    rdd3 = rdd3.flat_map(func3)
+    result = numbers.flat_map(&func1).flat_map(&func2).flat_map(&func3)
+
+    expect(rdd3.collect).to eql(result)
+
+    rdd4 = rdd(workers)
+    rdd4 = rdd4.flat_map(func1)
+    rdd4 = rdd4.flat_map(func2)
+    rdd4 = rdd4.flat_map(func3)
+
+    expect(rdd4.collect).to eql(rdd3.collect)
+  end
+end
+
+RSpec::shared_examples "a flat mapping values" do |workers|
+  it "with #{workers || 'default'} worker" do
+    rdd2 = rdd(workers).flat_map_values(func1)
+    result = []
+    hash_with_values.each do |(key, values)|
+      values = func1.call(values).flatten
+      values.each do |value|
+        result << [key, value]
+      end
+    end
+
+    expect(rdd2.collect).to eql(result)
+
+    rdd2 = rdd(workers).flat_map_values(func2)
+    result = []
+    hash_with_values.each do |(key, values)|
+      values = func2.call(values).flatten
+      values.each do |value|
+        result << [key, value]
+      end
+    end
+
+    expect(rdd2.collect).to eql(result)
+  end
+end
+
+RSpec::describe "Spark::RDD" do
+  let(:func1) { lambda{|x| x*2} }
+  let(:func2) { lambda{|x| [x*3, 1, 1]} }
+  let(:func3) { lambda{|x| [x*4, 2, 2]} }
+
+  context "throught parallelize" do
+    context ".flat_map" do
+      let(:numbers) { Generator.numbers_with_zero }
+
+      def rdd(workers)
+        $sc.parallelize(numbers, workers)
+      end
+
+      it_behaves_like "a flat mapping", nil
+      it_behaves_like "a flat mapping", 1
+      it_behaves_like "a flat mapping", rand(2..10)
+    end
+
+    context ".flat_map_values" do
+      let(:func1) { lambda{|x| x*2} }
+      let(:func2) { lambda{|x| [x.first]} }
+      let(:hash_with_values) { Generator.hash_with_values }
+
+      def rdd(workers)
+        $sc.parallelize(hash_with_values, workers)
+      end
+
+      it_behaves_like "a flat mapping values", nil
+      it_behaves_like "a flat mapping values", 1
+      it_behaves_like "a flat mapping values", rand(2..10)
+    end
+  end
+
+  context "throught text_file" do
+    context ".flat_map" do
+      let(:file)    { File.join("spec", "inputs", "numbers_0_100.txt") }
+      let(:numbers) { File.readlines(file).map(&:strip) }
+
+      def rdd(workers)
+        $sc.text_file(file, workers)
+      end
+
+      it_behaves_like "a flat mapping", nil
+      it_behaves_like "a flat mapping", 1
+      it_behaves_like "a flat mapping", rand(2..10)
+    end
+  end
+end