ruby-spark 1.0.0

data/benchmark/serializer.rb

@@ -0,0 +1,82 @@
+require "benchmark"
+require "yaml"
+require "msgpack"
+require "oj"
+# require "thrift"
+ 
+puts "Simple"
+
+data = (0..100000).to_a
+
+Benchmark.bmbm do |x|
+  x.report("YAML") do
+    serialized = YAML.dump(data)
+    deserialized = YAML.load(serialized)
+    puts "Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  x.report("Marshal") do
+    serialized = Marshal.dump(data)
+    deserialized = Marshal.load(serialized)
+    puts "Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  x.report("MessagePack") do
+    serialized = MessagePack.dump(data)
+    deserialized = MessagePack.load(serialized)
+    puts "Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  x.report("Oj") do
+    serialized = Oj.dump(data)
+    deserialized = Oj.load(serialized)
+    puts "Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  # x.report("Thrift") do
+  #   serializer = Thrift::Serializer.new
+  #   deserializer = Thrift::Deserializer.new
+
+  #   serialized = serializer.serialize(data)
+  # end
+end
+
+puts ""
+puts "More complex"
+
+data = Array.new(10000000) { 
+  [rand(97..122).chr, rand(10000000)]
+}
+
+Benchmark.bm do |x|
+  # Take too long
+  # x.report("YAML") do
+  #   serialized = YAML.dump(data)
+  #   YAML.load(serialized)
+  # end
+
+  x.report("Marshal") do
+    serialized = Marshal.dump(data)
+    deserialized = Marshal.load(serialized)
+    puts " Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  x.report("MessagePack") do
+    serialized = MessagePack.dump(data)
+    deserialized = MessagePack.load(serialized)
+    puts " Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  x.report("Oj") do
+    serialized = Oj.dump(data)
+    deserialized = Oj.load(serialized)
+    puts " Size: #{serialized.size}, Equal: #{deserialized == data}"
+  end
+
+  # x.report("Thrift") do
+  #   serializer = Thrift::Serializer.new
+  #   deserializer = Thrift::Deserializer.new
+
+  #   serialized = serializer.serialize(data)
+  # end
+end

data/benchmark/sort.rb

@@ -0,0 +1,43 @@
+require "benchmark"
+
+array = []
+1000.times { 
+  array << {:bar => rand(1000)} 
+}
+
+n = 500
+Benchmark.bm(20) do |x|
+  x.report("sort")               { n.times { array.sort{ |a,b| b[:bar] <=> a[:bar] } } }
+  x.report("sort reverse")       { n.times { array.sort{ |a,b| a[:bar] <=> b[:bar] }.reverse } }
+  x.report("sort_by -a[:bar]")   { n.times { array.sort_by{ |a| -a[:bar] } } }
+  x.report("sort_by a[:bar]*-1") { n.times { array.sort_by{ |a| a[:bar]*-1 } } }
+  x.report("sort_by.reverse!")   { n.times { array.sort_by{ |a| a[:bar] }.reverse } }
+end
+
+
+array = Array.new(10000) { Array.new(rand(1..10)){(97+rand(26)).chr}.join }
+
+Benchmark.bm(20) do |x|
+  x.report("sort asc")         { n.times { array.sort } }
+  x.report("sort asc block")   { n.times { array.sort{|a,b| a <=> b} } }
+  x.report("sort desc")        { n.times { array.sort{|a,b| b <=> a} } }
+  x.report("sort asc reverse") { n.times { array.sort.reverse } }
+end
+
+
+key_value = Struct.new(:key, :value) do
+  def <=>(other)
+    key <=> other.key
+  end
+end
+
+count = 10000
+item_range = 1000000
+array1 = Array.new(count) { [rand(item_range), rand(item_range)] }
+array2 = Array.new(count) { key_value.new rand(item_range), rand(item_range) }
+
+Benchmark.bm(20) do |x|
+  x.report("sort_by")       { n.times { array1.sort_by {|a| a[0]} } }
+  x.report("sort struct")   { n.times { array2.sort } }
+end
+

data/benchmark/sort2.rb

@@ -0,0 +1,164 @@
+require "benchmark"
+require "algorithms"
+
+NUMBER_OF_SORTING = 1
+NUMBER_OF_ARRAY   = 10
+WORDS_IN_ARRAY    = 100000
+MAX_WORD_SIZE     = 10
+EVAL_N_VALUES     = 10
+
+puts "NUMBER_OF_SORTING: #{NUMBER_OF_SORTING}"
+puts "NUMBER_OF_ARRAY: #{NUMBER_OF_ARRAY}"
+puts "WORDS_IN_ARRAY: #{WORDS_IN_ARRAY}"
+puts "MAX_WORD_SIZE: #{MAX_WORD_SIZE}"
+puts "EVAL_N_VALUES: #{EVAL_N_VALUES}"
+
+def words
+  Array.new(WORDS_IN_ARRAY) { word }
+end
+
+def word
+  Array.new(rand(1..MAX_WORD_SIZE)){(97+rand(26)).chr}.join
+end
+
+@array = Array.new(NUMBER_OF_ARRAY) { words.sort }
+
+
+# =================================================================================================
+# Sort1
+
+# Vrátí nový (nevyhodnocený) enumerator
+def sort1(data)
+  return to_enum(__callee__, data) unless block_given?
+
+  heap = []
+
+  # Inicializuji heap s prvními položkami
+  # připojím samotné enumeratory pro volání .next
+  data.each do |a|
+    heap << [a.next, a]
+  end
+
+  while data.any?
+    begin
+      # Seřadím pole podle hodnot
+      heap.sort_by!{|(item,_)| item}
+      # Uložím si hodnotu a enumerator
+      item, enum = heap.shift
+      # Hodnota půjde do výsledku
+      yield item
+      # Místo odstraněné položky nahradí další ze stejného seznamu
+      heap << [enum.next, enum]
+    rescue StopIteration
+      # Enumerator je prázdný
+      data.delete(enum)
+    end
+  end
+end
+
+
+# =================================================================================================
+# Sort1_2
+
+# Vrátí nový (nevyhodnocený) enumerator
+def sort1_2(data)
+  return to_enum(__callee__, data) unless block_given?
+
+  heap = []
+  enums = []
+
+  # Inicializuji heap s prvními položkami
+  # připojím samotné enumeratory pro volání .next
+  data.each do |a|
+    EVAL_N_VALUES.times {
+      begin
+        heap << [a.next, a]
+      rescue StopIteration
+      end
+    }
+  end
+
+  while data.any? || heap.any?
+      # Seřadím pole podle hodnot
+      heap.sort_by!{|(item,_)| item}
+
+      # Minimálně můžu vzít EVAL_N_VALUES
+      EVAL_N_VALUES.times {
+        break if heap.empty?
+
+        # Uložím si hodnotu a enumerator
+        item, enum = heap.shift
+        # Hodnota půjde do výsledku
+        yield item
+
+        enums << enum
+      }
+
+    while (enum = enums.shift)
+      begin
+        heap << [enum.next, enum]
+      rescue StopIteration
+        data.delete(enum)
+        enums.delete(enum)
+      end
+    end
+
+  end
+end
+
+
+# =================================================================================================
+# Sort 2
+
+def sort2(data)
+  return to_enum(__callee__, data) unless block_given?
+
+  heap = Containers::Heap.new
+
+  data.each do |enum|
+    item = enum.next
+    heap.push(item, [item, enum])
+  end
+
+  while data.any?
+    begin
+      item, enum = heap.pop
+      yield item
+
+      item = enum.next
+      heap.push(item, [item, enum])
+    rescue StopIteration
+      data.delete(enum)
+    end
+  end
+end
+
+
+# =================================================================================================
+# Benchmark
+
+Benchmark.bm(10) do |x|
+  x.report("sort") do
+    NUMBER_OF_SORTING.times {
+      @result = @array.flatten.sort
+    }
+  end
+
+  x.report("sort 1") do
+    NUMBER_OF_SORTING.times { 
+      raise "Bad sorting" if @result != sort1(@array.map(&:each)).to_a
+    }
+  end
+
+  x.report("sort 1_2") do
+    NUMBER_OF_SORTING.times { 
+      raise "Bad sorting" if @result != sort1_2(@array.map(&:each)).to_a
+    }
+  end
+
+  # x.report("sort 2") do
+  #   NUMBER_OF_SORTING.times {
+  #     raise "Bad sorting" if @result != sort2(@array.map(&:each)).to_a
+  #   }
+  # end
+end

data/benchmark/take.rb

@@ -0,0 +1,28 @@
+require "benchmark"
+
+SIZE = 100_000_000
+
+@array1 = (0..SIZE).to_a;
+@array2 = (0..SIZE).to_a;
+@array3 = (0..SIZE).to_a;
+
+TAKE = 100_000
+
+Benchmark.bm(15) do |x|
+  # Fastest
+  x.report("take"){
+    a=@array1.take(TAKE)
+  }
+
+  # Slowest and take most memory
+  x.report("reverse drop"){
+    @array2.reverse!
+    @array2.drop(@array2.size - TAKE)
+    @array2.reverse!
+  }
+
+  # Least memory
+  x.report("splice"){
+    a=@array2.slice!(0, TAKE)
+  }
+end

data/bin/ruby-spark

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'ruby-spark'
+
+Spark::CLI.new.run

data/example/pi.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'ruby-spark'
+
+Spark.logger.disable
+Spark.start
+
+slices = 3
+n = 100000 * slices
+
+def map(_)
+  x = rand * 2 - 1
+  y = rand * 2 - 1
+
+  if x**2 + y**2 < 1
+    return 1
+  else
+    return 0
+  end
+end
+
+rdd = Spark.context.parallelize(1..n, slices, serializer: 'oj')
+rdd = rdd.map(method(:map))
+
+puts 'Pi is roughly %f' % (4.0 * rdd.sum / n)

data/ext/ruby_c/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+
+create_makefile("ruby_spark_ext")

data/ext/ruby_c/murmur.c

@@ -0,0 +1,158 @@
+#include "murmur.h"
+
+#if defined(_MSC_VER)
+#define BIG_CONSTANT(x) (x)
+#else
+#define BIG_CONSTANT(x) (x##LLU)
+#endif
+
+/*-----------------------------------------------------------------------------
+// MurmurHash2, 64-bit versions, by Austin Appleby
+//
+// The same caveats as 32-bit MurmurHash2 apply here - beware of alignment 
+// and endian-ness issues if used across multiple platforms.
+//
+// 64-bit hash for 64-bit platforms
+*/
+
+uint64_t MurmurHash64A(const void * key, int len, uint64_t seed)
+{
+  const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
+  const int r = 47;
+
+  uint64_t h = seed ^ (len * m);
+
+  const uint64_t * data = (const uint64_t *)key;
+  const uint64_t * end = data + (len/8);
+
+  while(data != end)
+  {
+    uint64_t k = *data++;
+
+    k *= m; 
+    k ^= k >> r; 
+    k *= m; 
+    
+    h ^= k;
+    h *= m; 
+  }
+
+  const unsigned char * data2 = (const unsigned char*)data;
+
+  switch(len & 7)
+  {
+  case 7: h ^= ((uint64_t) data2[6]) << 48;
+  case 6: h ^= ((uint64_t) data2[5]) << 40;
+  case 5: h ^= ((uint64_t) data2[4]) << 32;
+  case 4: h ^= ((uint64_t) data2[3]) << 24;
+  case 3: h ^= ((uint64_t) data2[2]) << 16;
+  case 2: h ^= ((uint64_t) data2[1]) << 8;
+  case 1: h ^= ((uint64_t) data2[0]);
+          h *= m;
+  };
+ 
+  h ^= h >> r;
+  h *= m;
+  h ^= h >> r;
+
+  return h;
+} 
+
+/* 64-bit hash for 32-bit platforms */
+
+uint64_t MurmurHash64B(const void * key, int len, uint64_t seed)
+{
+  const uint32_t m = 0x5bd1e995;
+  const int r = 24;
+
+  uint32_t h1 = ((uint32_t) seed) ^ len;
+  uint32_t h2 = ((uint32_t) (seed >> 32));
+
+  const uint32_t * data = (const uint32_t *)key;
+
+  while(len >= 8)
+  {
+    uint32_t k1 = *data++;
+    k1 *= m; k1 ^= k1 >> r; k1 *= m;
+    h1 *= m; h1 ^= k1;
+    len -= 4;
+
+    uint32_t k2 = *data++;
+    k2 *= m; k2 ^= k2 >> r; k2 *= m;
+    h2 *= m; h2 ^= k2;
+    len -= 4;
+  }
+
+  if(len >= 4)
+  {
+    uint32_t k1 = *data++;
+    k1 *= m; k1 ^= k1 >> r; k1 *= m;
+    h1 *= m; h1 ^= k1;
+    len -= 4;
+  }
+
+  switch(len)
+  {
+  case 3: h2 ^= ((unsigned char*)data)[2] << 16;
+  case 2: h2 ^= ((unsigned char*)data)[1] << 8;
+  case 1: h2 ^= ((unsigned char*)data)[0];
+      h2 *= m;
+  };
+
+  h1 ^= h2 >> 18; h1 *= m;
+  h2 ^= h1 >> 22; h2 *= m;
+  h1 ^= h2 >> 17; h1 *= m;
+  h2 ^= h1 >> 19; h2 *= m;
+
+  uint64_t h = h1;
+
+  h = (h << 32) | h2;
+
+  return h;
+}
+
+
+
+// ================================================================================================
+// Ruby methods
+
+#define PORTABLE_HASH_SEED 16154832
+
+
+VALUE murmur2_digest(VALUE rb_str, uint64_t seed)
+{
+  StringValue(rb_str);
+
+  void * key = RSTRING_PTR(rb_str);
+  long   len = RSTRING_LEN(rb_str);
+
+  uint64_t result = MurmurHash64A(key, len, seed);
+
+  return LONG2FIX(result);
+}
+
+// ------------------------------------------------------------------------------------------------
+// Spark::Digest::Murmur2.digest
+
+VALUE method_murmur2_digest(int argc, VALUE *argv, VALUE klass)
+{
+  if(argc == 0 || argc > 2){
+    rb_raise(rb_eArgError, "wrong number of arguments (%d for 1..2)", argc);
+  }
+
+  uint64_t seed = (argc == 1 ? 0 : NUM2UINT(argv[1]));
+
+  return murmur2_digest(argv[0], seed);
+}
+
+// ------------------------------------------------------------------------------------------------
+// Spark::Digest.portable_hash
+
+VALUE method_portable_hash(int argc, VALUE *argv, VALUE klass)
+{
+  if(argc != 1){
+    rb_raise(rb_eArgError, "wrong number of arguments (%d for 1)", argc);
+  }
+
+  return murmur2_digest(argv[0], PORTABLE_HASH_SEED);
+}