ruby-spark 1.1.0.1-java

data/benchmark/enumerator.rb

@@ -0,0 +1,88 @@
+require "benchmark"
+
+class Enumerator
+  def defer(&blk)
+    self.class.new do |y|
+      each do |*input|
+        blk.call(y, *input)
+      end
+    end
+  end
+end
+
+ARRAY_SIZE = 50_000_000
+
+def type_yield
+  return to_enum(__callee__) unless block_given?
+
+  ARRAY_SIZE.times { |i|
+    yield i
+  }
+end
+
+def yield_map_x2(enum)
+  return to_enum(__callee__, enum) unless block_given?
+  
+  enum.each do |item|
+    yield item*2
+  end
+end
+
+def type_enumerator_new
+  Enumerator.new do |e|
+    ARRAY_SIZE.times { |i|
+      e << i
+    }
+  end
+end
+
+def enumerator_new_map_x2(enum)
+  Enumerator.new do |e|
+    enum.each do |item|
+      e << item*2
+    end
+  end
+end
+
+def enumerator_defer_x2(enum)
+  enum.defer do |out, inp|
+    out << inp*2
+  end
+end
+
+Benchmark.bm(26) do |x|
+  x.report("yield max") do
+    type_yield.max
+  end
+
+  x.report("yield sum") do
+    type_yield.reduce(:+)
+  end
+
+  x.report("yield map x*2 sum") do
+    yield_map_x2(type_yield).reduce(:+)
+  end
+
+  x.report("yield defer map x*2 sum") do
+    enumerator_defer_x2(type_yield).reduce(:+)
+  end
+
+  x.report("-----"){}
+
+  x.report("Enum.new max") do
+    type_enumerator_new.max
+  end
+
+  x.report("Enum.new sum") do
+    type_enumerator_new.reduce(:+)
+  end
+
+  x.report("Enum.new map x*2 sum") do
+    enumerator_new_map_x2(type_enumerator_new).reduce(:+)
+  end
+
+  x.report("Enum.new defer map x*2 sum") do
+    enumerator_defer_x2(type_enumerator_new).reduce(:+)
+  end
+
+end

data/benchmark/serializer.rb

@@ -0,0 +1,82 @@
+require "benchmark"
+require "yaml"
+require "msgpack"
+require "oj"
+# require "thrift"
+ 
+puts "Simple"
+
+data = (0..100000).to_a
+
+Benchmark.bmbm do |x|
+  x.report("YAML") do
+    serialized = YAML.dump(data)
+    deserialized = YAML.load(serialized)
+    puts "Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  x.report("Marshal") do
+    serialized = Marshal.dump(data)
+    deserialized = Marshal.load(serialized)
+    puts "Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  x.report("MessagePack") do
+    serialized = MessagePack.dump(data)
+    deserialized = MessagePack.load(serialized)
+    puts "Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  x.report("Oj") do
+    serialized = Oj.dump(data)
+    deserialized = Oj.load(serialized)
+    puts "Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  # x.report("Thrift") do
+  #   serializer = Thrift::Serializer.new
+  #   deserializer = Thrift::Deserializer.new
+
+  #   serialized = serializer.serialize(data)
+  # end
+end
+
+puts ""
+puts "More complex"
+
+data = Array.new(10000000) { 
+  [rand(97..122).chr, rand(10000000)]
+}
+
+Benchmark.bm do |x|
+  # Take too long
+  # x.report("YAML") do
+  #   serialized = YAML.dump(data)
+  #   YAML.load(serialized)
+  # end
+
+  x.report("Marshal") do
+    serialized = Marshal.dump(data)
+    deserialized = Marshal.load(serialized)
+    puts " Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  x.report("MessagePack") do
+    serialized = MessagePack.dump(data)
+    deserialized = MessagePack.load(serialized)
+    puts " Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  x.report("Oj") do
+    serialized = Oj.dump(data)
+    deserialized = Oj.load(serialized)
+    puts " Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  # x.report("Thrift") do
+  #   serializer = Thrift::Serializer.new
+  #   deserializer = Thrift::Deserializer.new
+
+  #   serialized = serializer.serialize(data)
+  # end
+end

data/benchmark/sort.rb

@@ -0,0 +1,43 @@
+require "benchmark"
+
+array = []
+1000.times { 
+  array << {:bar => rand(1000)} 
+}
+
+n = 500
+Benchmark.bm(20) do |x|
+  x.report("sort")               { n.times { array.sort{ |a,b| b[:bar] <=> a[:bar] } } }
+  x.report("sort reverse")       { n.times { array.sort{ |a,b| a[:bar] <=> b[:bar] }.reverse } }
+  x.report("sort_by -a[:bar]")   { n.times { array.sort_by{ |a| -a[:bar] } } }
+  x.report("sort_by a[:bar]*-1") { n.times { array.sort_by{ |a| a[:bar]*-1 } } }
+  x.report("sort_by.reverse!")   { n.times { array.sort_by{ |a| a[:bar] }.reverse } }
+end
+
+
+array = Array.new(10000) { Array.new(rand(1..10)){(97+rand(26)).chr}.join }
+
+Benchmark.bm(20) do |x|
+  x.report("sort asc")         { n.times { array.sort } }
+  x.report("sort asc block")   { n.times { array.sort{|a,b| a <=> b} } }
+  x.report("sort desc")        { n.times { array.sort{|a,b| b <=> a} } }
+  x.report("sort asc reverse") { n.times { array.sort.reverse } }
+end
+
+
+key_value = Struct.new(:key, :value) do
+  def <=>(other)
+    key <=> other.key
+  end
+end
+
+count = 10000
+item_range = 1000000
+array1 = Array.new(count) { [rand(item_range), rand(item_range)] }
+array2 = Array.new(count) { key_value.new rand(item_range), rand(item_range) }
+
+Benchmark.bm(20) do |x|
+  x.report("sort_by")       { n.times { array1.sort_by {|a| a[0]} } }
+  x.report("sort struct")   { n.times { array2.sort } }
+end
+

data/benchmark/sort2.rb

@@ -0,0 +1,164 @@
+require "benchmark"
+require "algorithms"
+
+NUMBER_OF_SORTING = 1
+NUMBER_OF_ARRAY   = 10
+WORDS_IN_ARRAY    = 100000
+MAX_WORD_SIZE     = 10
+EVAL_N_VALUES     = 10
+
+puts "NUMBER_OF_SORTING: #{NUMBER_OF_SORTING}"
+puts "NUMBER_OF_ARRAY: #{NUMBER_OF_ARRAY}"
+puts "WORDS_IN_ARRAY: #{WORDS_IN_ARRAY}"
+puts "MAX_WORD_SIZE: #{MAX_WORD_SIZE}"
+puts "EVAL_N_VALUES: #{EVAL_N_VALUES}"
+
+def words
+  Array.new(WORDS_IN_ARRAY) { word }
+end
+
+def word
+  Array.new(rand(1..MAX_WORD_SIZE)){(97+rand(26)).chr}.join
+end
+
+@array = Array.new(NUMBER_OF_ARRAY) { words.sort }
+
+
+# =================================================================================================
+# Sort1
+
+# Vrátí nový (nevyhodnocený) enumerator
+def sort1(data)
+  return to_enum(__callee__, data) unless block_given?
+
+  heap = []
+
+  # Inicializuji heap s prvními položkami
+  # připojím samotné enumeratory pro volání .next
+  data.each do |a|
+    heap << [a.next, a]
+  end
+
+  while data.any?
+    begin
+      # Seřadím pole podle hodnot
+      heap.sort_by!{|(item,_)| item}
+      # Uložím si hodnotu a enumerator
+      item, enum = heap.shift
+      # Hodnota půjde do výsledku
+      yield item
+      # Místo odstraněné položky nahradí další ze stejného seznamu
+      heap << [enum.next, enum]
+    rescue StopIteration
+      # Enumerator je prázdný
+      data.delete(enum)
+    end
+  end
+end
+
+
+# =================================================================================================
+# Sort1_2
+
+# Vrátí nový (nevyhodnocený) enumerator
+def sort1_2(data)
+  return to_enum(__callee__, data) unless block_given?
+
+  heap = []
+  enums = []
+
+  # Inicializuji heap s prvními položkami
+  # připojím samotné enumeratory pro volání .next
+  data.each do |a|
+    EVAL_N_VALUES.times {
+      begin
+        heap << [a.next, a]
+      rescue StopIteration
+      end
+    }
+  end
+
+  while data.any? || heap.any?
+      # Seřadím pole podle hodnot
+      heap.sort_by!{|(item,_)| item}
+
+      # Minimálně můžu vzít EVAL_N_VALUES
+      EVAL_N_VALUES.times {
+        break if heap.empty?
+
+        # Uložím si hodnotu a enumerator
+        item, enum = heap.shift
+        # Hodnota půjde do výsledku
+        yield item
+
+        enums << enum
+      }
+
+    while (enum = enums.shift)
+      begin
+        heap << [enum.next, enum]
+      rescue StopIteration
+        data.delete(enum)
+        enums.delete(enum)
+      end
+    end
+
+  end
+end
+
+
+# =================================================================================================
+# Sort 2
+
+def sort2(data)
+  return to_enum(__callee__, data) unless block_given?
+
+  heap = Containers::Heap.new
+
+  data.each do |enum|
+    item = enum.next
+    heap.push(item, [item, enum])
+  end
+
+  while data.any?
+    begin
+      item, enum = heap.pop
+      yield item
+
+      item = enum.next
+      heap.push(item, [item, enum])
+    rescue StopIteration
+      data.delete(enum)
+    end
+  end
+end
+
+
+# =================================================================================================
+# Benchmark
+
+Benchmark.bm(10) do |x|
+  x.report("sort") do
+    NUMBER_OF_SORTING.times {
+      @result = @array.flatten.sort
+    }
+  end
+
+  x.report("sort 1") do
+    NUMBER_OF_SORTING.times { 
+      raise "Bad sorting" if @result != sort1(@array.map(&:each)).to_a
+    }
+  end
+
+  x.report("sort 1_2") do
+    NUMBER_OF_SORTING.times { 
+      raise "Bad sorting" if @result != sort1_2(@array.map(&:each)).to_a
+    }
+  end
+
+  # x.report("sort 2") do
+  #   NUMBER_OF_SORTING.times {
+  #     raise "Bad sorting" if @result != sort2(@array.map(&:each)).to_a
+  #   }
+  # end
+end

data/benchmark/take.rb

@@ -0,0 +1,28 @@
+require "benchmark"
+
+SIZE = 100_000_000
+
+@array1 = (0..SIZE).to_a;
+@array2 = (0..SIZE).to_a;
+@array3 = (0..SIZE).to_a;
+
+TAKE = 100_000
+
+Benchmark.bm(15) do |x|
+  # Fastest
+  x.report("take"){
+    a=@array1.take(TAKE)
+  }
+
+  # Slowest and take most memory
+  x.report("reverse drop"){
+    @array2.reverse!
+    @array2.drop(@array2.size - TAKE)
+    @array2.reverse!
+  }
+
+  # Least memory
+  x.report("splice"){
+    a=@array2.slice!(0, TAKE)
+  }
+end

data/bin/ruby-spark

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'ruby-spark'
+
+Spark::CLI.new.run

data/example/pi.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'ruby-spark'
+
+Spark.logger.disable
+Spark.start
+
+slices = 3
+n = 100000 * slices
+
+def map(_)
+  x = rand * 2 - 1
+  y = rand * 2 - 1
+
+  if x**2 + y**2 < 1
+    return 1
+  else
+    return 0
+  end
+end
+
+rdd = Spark.context.parallelize(1..n, slices)
+rdd = rdd.map(method(:map))
+
+puts 'Pi is roughly %f' % (4.0 * rdd.sum / n)