ruby-spark 1.0.0

data/benchmark/custom_marshal.rb

@@ -0,0 +1,94 @@
+require 'benchmark'
+require 'benchmark/ips'
+
+def pack_int(data)
+  [data].pack('l>')
+end
+
+def pack_long(data)
+  [data].pack('q>')
+end
+
+def pack_doubles(data)
+  data.pack('G*')
+end
+
+module Standard
+  class LabeledPoint
+    def initialize(label, features)
+      @label = label
+      @features = Standard::Vector.new(features)
+    end
+
+    def marshal_dump
+      [@label, @features]
+    end
+
+    def marshal_load(*)
+    end
+  end
+
+  class Vector
+    def initialize(array)
+      @values = array
+    end
+
+    def marshal_dump
+      [@values]
+    end
+
+    def marshal_load(*)
+    end
+  end
+end
+
+module Custom
+  class LabeledPoint
+    def initialize(label, features)
+      @label = label
+      @features = Custom::Vector.new(features)
+    end
+
+    def _dump(*)
+      pack_long(@label) + @features._dump
+    end
+
+    def self._load(*)
+    end
+  end
+
+  class Vector
+    def initialize(array)
+      @values = array
+    end
+
+    def _dump(*)
+      result = 'v'
+      result << pack_int(@values.size)
+      result << pack_doubles(@values)
+      result.encode(Encoding::ASCII_8BIT)
+    end
+
+    def self._load(*)
+    end
+  end
+end
+
+data_size = 10_000
+vector_size = 1_000
+values = Array.new(vector_size) { |x| rand(10_000..100_000) }
+
+@data1 = Array.new(data_size) {|i| Standard::LabeledPoint.new(i, values)}
+@data2 = Array.new(data_size) {|i| Custom::LabeledPoint.new(i, values)}
+
+Benchmark.ips do |r|
+  r.report('standard') do
+    Marshal.dump(@data1)
+  end
+
+  r.report('custom') do
+    Marshal.dump(@data2)
+  end
+
+  r.compare!
+end

data/benchmark/digest.rb

@@ -0,0 +1,150 @@
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+def java?
+  RUBY_PLATFORM =~ /java/
+end
+
+unless java?
+  require 'murmurhash3'
+end
+
+require 'digest'
+require 'benchmark'
+require 'ruby-spark'
+
+TEST = 5_000_000
+WORDS = ["wefwefwef", "rgwefiwefwe", "a", "rujfwgrethrzjrhgawf", "irncrnuggo"]
+
+puts "TEST COUNT = #{TEST*WORDS.size}"
+
+# =================================================================================================
+# Pure ruby mumrumur
+# funny-falcon/murmurhash3-ruby
+
+MASK32 = 0xffffffff
+
+def murmur3_32_rotl(x, r)
+  ((x << r) | (x >> (32 - r))) & MASK32
+end
+
+def murmur3_32_fmix(h)
+  h &= MASK32
+  h ^= h >> 16
+  h = (h * 0x85ebca6b) & MASK32
+  h ^= h >> 13
+  h = (h * 0xc2b2ae35) & MASK32
+  h ^ (h >> 16)
+end
+
+def murmur3_32__mmix(k1)
+  k1 = (k1 * 0xcc9e2d51) & MASK32
+  k1 = murmur3_32_rotl(k1, 15)
+  (k1 * 0x1b873593) & MASK32
+end
+
+def murmur3_32_str_hash(str, seed=0)
+  h1 = seed
+  numbers = str.unpack('V*C*')
+  tailn = str.bytesize % 4
+  tail = numbers.slice!(numbers.size - tailn, tailn)
+  for k1 in numbers
+    h1 ^= murmur3_32__mmix(k1)
+    h1 = murmur3_32_rotl(h1, 13)
+    h1 = (h1*5 + 0xe6546b64) & MASK32
+  end
+
+  unless tail.empty?
+    k1 = 0
+    tail.reverse_each do |c1|
+      k1 = (k1 << 8) | c1
+    end
+    h1 ^= murmur3_32__mmix(k1)
+  end
+
+  h1 ^= str.bytesize
+  murmur3_32_fmix(h1)
+end
+
+
+# =================================================================================================
+# Benchmark
+
+Benchmark.bm(18) do |x|
+
+  x.report("ruby hash"){
+    TEST.times{
+      WORDS.each{ |word|
+        word.hash
+      }
+    }    
+  }
+
+  x.report("ext portable"){
+    TEST.times{
+      WORDS.each{ |word|
+        Spark::Digest.portable_hash(word)
+      }
+    }    
+  }
+
+  x.report("murmur3 32"){
+    TEST.times{
+      WORDS.each{ |word|
+        # MurmurHash3::V128.str_hash(word)
+        # [MurmurHash3::V128.str_hash(word).join.to_i].pack("q>")
+        # MurmurHash3::V128.str_hash(word)
+        # a = MurmurHash3::V32.str_hash(word).to_s
+        # a.slice!(0,8)
+
+        MurmurHash3::V32.str_hash(word)
+      }
+    }    
+  } unless java?
+
+  # Too slow
+  # x.report("murmur3 32 (ruby)"){
+  #   TEST.times{
+  #     WORDS.each{ |word|
+  #       # MurmurHash3::V128.str_hash(word)
+  #       # [MurmurHash3::V128.str_hash(word).join.to_i].pack("q>")
+  #       # MurmurHash3::V128.str_hash(word)
+  #       # a = murmur3_32_str_hash(word).to_s
+  #       # a.slice!(0,8)
+
+  #       murmur3_32_str_hash(word)
+  #     }
+  #   }    
+  # }
+
+  x.report("murmur3 128"){
+    TEST.times{
+      WORDS.each{ |word|
+        # MurmurHash3::V128.str_hash(word)
+        # [MurmurHash3::V128.str_hash(word).join.to_i].pack("q>")
+        # a = MurmurHash3::V128.str_hash(word).to_s
+        # a.slice!(0,8)
+
+        MurmurHash3::V128.str_hash(word)
+      }
+    }    
+  } unless java?
+
+  # x.report("sha256"){
+  #   TEST.times{
+  #     WORDS.each{ |word|
+  #       a = Digest::SHA256.digest(word)
+  #       # a.slice!(0,8)
+  #     }
+  #   }    
+  # }
+
+  # x.report("md5"){
+  #   TEST.times{
+  #     WORDS.each{ |word|
+  #       a = Digest::MD5.digest(word)
+  #       # a.slice!(0,8)
+  #     }
+  #   }    
+  # }
+end

data/benchmark/enumerator.rb

@@ -0,0 +1,88 @@
+require "benchmark"
+
+class Enumerator
+  def defer(&blk)
+    self.class.new do |y|
+      each do |*input|
+        blk.call(y, *input)
+      end
+    end
+  end
+end
+
+ARRAY_SIZE = 50_000_000
+
+def type_yield
+  return to_enum(__callee__) unless block_given?
+
+  ARRAY_SIZE.times { |i|
+    yield i
+  }
+end
+
+def yield_map_x2(enum)
+  return to_enum(__callee__, enum) unless block_given?
+  
+  enum.each do |item|
+    yield item*2
+  end
+end
+
+def type_enumerator_new
+  Enumerator.new do |e|
+    ARRAY_SIZE.times { |i|
+      e << i
+    }
+  end
+end
+
+def enumerator_new_map_x2(enum)
+  Enumerator.new do |e|
+    enum.each do |item|
+      e << item*2
+    end
+  end
+end
+
+def enumerator_defer_x2(enum)
+  enum.defer do |out, inp|
+    out << inp*2
+  end
+end
+
+Benchmark.bm(26) do |x|
+  x.report("yield max") do
+    type_yield.max
+  end
+
+  x.report("yield sum") do
+    type_yield.reduce(:+)
+  end
+
+  x.report("yield map x*2 sum") do
+    yield_map_x2(type_yield).reduce(:+)
+  end
+
+  x.report("yield defer map x*2 sum") do
+    enumerator_defer_x2(type_yield).reduce(:+)
+  end
+
+  x.report("-----"){}
+
+  x.report("Enum.new max") do
+    type_enumerator_new.max
+  end
+
+  x.report("Enum.new sum") do
+    type_enumerator_new.reduce(:+)
+  end
+
+  x.report("Enum.new map x*2 sum") do
+    enumerator_new_map_x2(type_enumerator_new).reduce(:+)
+  end
+
+  x.report("Enum.new defer map x*2 sum") do
+    enumerator_defer_x2(type_enumerator_new).reduce(:+)
+  end
+
+end

data/benchmark/performance/prepare.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Current dir
+cd "$(dirname "$0")"
+
+# Exit immediately if a pipeline returns a non-zero status.
+set -e
+
+# Spark
+wget "http://d3kbcqa49mib13.cloudfront.net/spark-1.3.0-bin-hadoop2.4.tgz" -O spark.tgz
+tar xvzf spark.tgz
+mv spark-1.3.0-bin-hadoop2.4 spark
+rm spark.tgz
+
+# RSpark (only for 1.3.0)
+git clone git@github.com:amplab-extras/SparkR-pkg.git rspark
+cd rspark
+SPARK_VERSION=1.3.0 ./install-dev.sh

data/benchmark/performance/python.py

@@ -0,0 +1,156 @@
+import os
+import math
+from time import time
+from random import random
+from operator import add
+from pyspark import SparkContext
+
+sc = SparkContext(appName="Python", master="local[*]")
+
+log_file = open(os.environ.get('PYTHON_LOG'), 'w')
+
+def log(*values):
+  values = map(lambda x: str(x), values)
+  log_file.write(';'.join(values))
+  log_file.write('\n')
+
+workers = int(os.environ.get('WORKERS'))
+numbers_count = int(os.environ.get('NUMBERS_COUNT'))
+text_file = os.environ.get('TEXT_FILE')
+
+numbers = range(numbers_count)
+floats = [float(i) for i in numbers]
+with open(text_file) as t:
+  strings = t.read().split("\n")
+
+
+# =============================================================================
+# Serialization
+# =============================================================================
+
+t = time()
+rdd_numbers = sc.parallelize(numbers, workers)
+t = time() - t
+log('NumbersSerialization', t)
+
+
+t = time()
+rdd_floats = sc.parallelize(floats, workers)
+t = time() - t
+log('FloatsSerialization', t)
+
+
+t = time()
+rdd_strings = sc.parallelize(strings, workers)
+t = time() - t
+log('StringsSerialization', t)
+
+
+# =============================================================================
+# Computing
+# =============================================================================
+
+
+# --- Is prime? ---------------------------------------------------------------
+
+def is_prime(x):
+  if x < 2:
+    return [x, False]
+  elif x == 2:
+    return [x, True]
+  elif x % 2 == 0:
+    return [x, False]
+  else:
+    upper = int(math.sqrt(float(x)))
+    result = True
+
+    i = 3
+    while i <= upper:
+      if x % i == 0:
+        result = False
+        break
+
+      i += 2
+
+    return [x, result]
+
+t = time()
+rdd_numbers.map(is_prime).collect()
+t = time() - t
+
+log('IsPrime', t)
+
+
+# --- Matrix multiplication ---------------------------------------------------
+
+matrix_size = int(os.environ.get('MATRIX_SIZE'))
+
+matrix = []
+for row in range(matrix_size):
+  matrix.append([])
+  for col in range(matrix_size):
+    matrix[row].append(row+col)
+
+def multiplication_func(matrix):
+  matrix = list(matrix)
+  size = len(matrix)
+
+  new_matrix = []
+  for row in range(size):
+    new_matrix.append([])
+    for col in range(size):
+
+      result = 0
+      for i in range(size):
+        result += matrix[row][i] * matrix[col][i]
+      new_matrix[row].append(result)
+
+  return new_matrix
+
+t = time()
+rdd = sc.parallelize(matrix, 1)
+rdd.mapPartitions(multiplication_func).collect()
+t = time() - t
+
+log('MatrixMultiplication', t)
+
+
+# --- Pi digits ---------------------------------------------------------------
+# http://rosettacode.org/wiki/Pi#Python
+
+pi_digit = int(os.environ.get('PI_DIGIT'))
+
+def pi_func(size):
+  size = size.next()
+  result = ''
+
+  q, r, t, k, n, l = 1, 0, 1, 1, 3, 3
+  while size > 0:
+    if 4*q+r-t < n*t:
+      result += str(n)
+      size -= 1
+      nr = 10*(r-n*t)
+      n  = ((10*(3*q+r))//t)-10*n
+      q  *= 10
+      r  = nr
+    else:
+      nr = (2*q+r)*l
+      nn = (q*(7*k)+2+(r*l))//(t*l)
+      q  *= k
+      t  *= l
+      l  += 2
+      k += 1
+      n  = nn
+      r  = nr
+
+  return [result]
+
+t = time()
+rdd = sc.parallelize([pi_digit], 1)
+rdd.mapPartitions(pi_func).collect()
+t = time() - t
+
+log('PiDigit', t)
+
+
+log_file.close()