ruby-spark 1.1.0.1-java

data/benchmark/bisect.rb

@@ -0,0 +1,88 @@
+require "benchmark"
+
+def bisect_left1(a, x, opts={})
+  return nil if a.nil?
+  return 0 if a.empty?
+
+  lo = (opts[:lo] || opts[:low]).to_i
+  hi = opts[:hi] || opts[:high] || a.length
+
+  while lo < hi
+    mid = (lo + hi) / 2
+    v = a[mid]
+    if v < x
+      lo = mid + 1
+    else
+      hi = mid
+    end
+  end
+  return lo
+end
+
+def bisect_left2(list, item)
+  count = 0
+  list.each{|i|
+    return count if i >= item
+    count += 1
+  }
+  nil
+end
+
+def bisect_left3(list, item, lo = 0, hi = list.size)
+  while lo < hi
+    i = (lo + hi - 1) >> 1
+
+    if 0 <= (list[i] <=> item)
+      hi = i
+    else
+      lo = i + 1
+    end
+  end
+  return hi
+end
+
+array = Array.new(1000000) { rand(0..1000000) };
+to_find = Array.new(500) { rand(0..10000) };
+
+Benchmark.bm(20) do |x|
+  x.report("bisect_left1") do
+    to_find.each do |item|
+      bisect_left1(array, item)
+    end
+  end
+
+  x.report("bisect_left2") do
+    to_find.each do |item|
+      bisect_left2(array, item)
+    end
+  end
+
+  x.report("bisect_left3") do
+    to_find.each do |item|
+      bisect_left3(array, item)
+    end
+  end
+end
+
+array = Array.new(100000) { Array.new(rand(1..10)){(97+rand(26)).chr}.join };
+to_find = Array.new(500) { (97+rand(26)).chr };
+
+Benchmark.bm(20) do |x|
+  x.report("bisect_left1") do
+    to_find.each do |item|
+      bisect_left1(array, item)
+    end
+  end
+
+  x.report("bisect_left2") do
+    to_find.each do |item|
+      bisect_left2(array, item)
+    end
+  end
+
+  x.report("bisect_left3") do
+    to_find.each do |item|
+      bisect_left3(array, item)
+    end
+  end
+end

data/benchmark/comparison/prepare.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Current dir
+cd "$(dirname "$0")"
+
+# Exit immediately if a pipeline returns a non-zero status.
+set -e
+
+# Spark
+wget "http://d3kbcqa49mib13.cloudfront.net/spark-1.3.0-bin-hadoop2.4.tgz" -O spark.tgz
+tar xvzf spark.tgz
+mv spark-1.3.0-bin-hadoop2.4 spark
+rm spark.tgz
+
+# RSpark (only for 1.3.0)
+git clone git@github.com:amplab-extras/SparkR-pkg.git rspark
+cd rspark
+SPARK_VERSION=1.3.0 ./install-dev.sh

data/benchmark/comparison/python.py

@@ -0,0 +1,156 @@
+import os
+import math
+from time import time
+from random import random
+from operator import add
+from pyspark import SparkContext
+
+sc = SparkContext(appName="Python", master="local[*]")
+
+log_file = open(os.environ.get('PYTHON_LOG'), 'w')
+
+def log(*values):
+  values = map(lambda x: str(x), values)
+  log_file.write(';'.join(values))
+  log_file.write('\n')
+
+workers = int(os.environ.get('WORKERS'))
+numbers_count = int(os.environ.get('NUMBERS_COUNT'))
+text_file = os.environ.get('TEXT_FILE')
+
+numbers = range(numbers_count)
+floats = [float(i) for i in numbers]
+with open(text_file) as t:
+  strings = t.read().split("\n")
+
+
+# =============================================================================
+# Serialization
+# =============================================================================
+
+t = time()
+rdd_numbers = sc.parallelize(numbers, workers)
+t = time() - t
+log('NumbersSerialization', t)
+
+
+t = time()
+rdd_floats = sc.parallelize(floats, workers)
+t = time() - t
+log('FloatsSerialization', t)
+
+
+t = time()
+rdd_strings = sc.parallelize(strings, workers)
+t = time() - t
+log('StringsSerialization', t)
+
+
+# =============================================================================
+# Computing
+# =============================================================================
+
+
+# --- Is prime? ---------------------------------------------------------------
+
+def is_prime(x):
+  if x < 2:
+    return [x, False]
+  elif x == 2:
+    return [x, True]
+  elif x % 2 == 0:
+    return [x, False]
+  else:
+    upper = int(math.sqrt(float(x)))
+    result = True
+
+    i = 3
+    while i <= upper:
+      if x % i == 0:
+        result = False
+        break
+
+      i += 2
+
+    return [x, result]
+
+t = time()
+rdd_numbers.map(is_prime).collect()
+t = time() - t
+
+log('IsPrime', t)
+
+
+# --- Matrix multiplication ---------------------------------------------------
+
+matrix_size = int(os.environ.get('MATRIX_SIZE'))
+
+matrix = []
+for row in range(matrix_size):
+  matrix.append([])
+  for col in range(matrix_size):
+    matrix[row].append(row+col)
+
+def multiplication_func(matrix):
+  matrix = list(matrix)
+  size = len(matrix)
+
+  new_matrix = []
+  for row in range(size):
+    new_matrix.append([])
+    for col in range(size):
+
+      result = 0
+      for i in range(size):
+        result += matrix[row][i] * matrix[col][i]
+      new_matrix[row].append(result)
+
+  return new_matrix
+
+t = time()
+rdd = sc.parallelize(matrix, 1)
+rdd.mapPartitions(multiplication_func).collect()
+t = time() - t
+
+log('MatrixMultiplication', t)
+
+
+# --- Pi digits ---------------------------------------------------------------
+# http://rosettacode.org/wiki/Pi#Python
+
+pi_digit = int(os.environ.get('PI_DIGIT'))
+
+def pi_func(size):
+  size = size.next()
+  result = ''
+
+  q, r, t, k, n, l = 1, 0, 1, 1, 3, 3
+  while size > 0:
+    if 4*q+r-t < n*t:
+      result += str(n)
+      size -= 1
+      nr = 10*(r-n*t)
+      n  = ((10*(3*q+r))//t)-10*n
+      q  *= 10
+      r  = nr
+    else:
+      nr = (2*q+r)*l
+      nn = (q*(7*k)+2+(r*l))//(t*l)
+      q  *= k
+      t  *= l
+      l  += 2
+      k += 1
+      n  = nn
+      r  = nr
+
+  return [result]
+
+t = time()
+rdd = sc.parallelize([pi_digit], 1)
+rdd.mapPartitions(pi_func).collect()
+t = time() - t
+
+log('PiDigit', t)
+
+
+log_file.close()

data/benchmark/comparison/r.r

@@ -0,0 +1,69 @@
+library(SparkR)
+sc <- sparkR.init(master="local[*]")
+
+logFile <- file(Sys.getenv("R_LOG"), "w")
+
+logInfo <- function(...){
+  args <- list(...)
+  line <- paste(args, collapse = ";")
+  writeLines(line, logFile)
+}
+
+workers <- as.integer(Sys.getenv('WORKERS'))
+numbersCount <- as.integer(Sys.getenv('NUMBERS_COUNT'))
+textFile <- Sys.getenv('TEXT_FILE')
+
+
+# =============================================================================
+# Serialization
+# =============================================================================
+
+time <- proc.time()
+rddNumbers <- parallelize(sc, as.numeric(seq(0, numbersCount)), workers)
+time <- as.double(proc.time()-time)[3]
+
+logInfo('NumbersSerialization', time)
+
+
+# =============================================================================
+# Computing
+# =============================================================================
+
+isPrime = function(x) {
+  if(x < 2){
+    c(x, FALSE)
+  }
+  else if(x == 2){
+    c(x, TRUE)
+  }
+  else if(x %% 2 == 0){
+    c(x, FALSE)
+  }
+  else{
+    upper <- as.numeric(sqrt(as.double(x)))
+    result <- TRUE
+
+    i <- 3
+    while(i <= upper){
+      if(x %% i == 0){
+        result = FALSE
+        break
+      }
+
+      i <- i+2
+    }
+
+    c(x, result)
+  }
+}
+
+time <- proc.time()
+rdd <- map(rddNumbers, isPrime)
+capture.output(collect(rdd), file='/dev/null')
+time <- as.double(proc.time()-time)[3]
+
+logInfo('IsPrime', time)
+
+
+close(logFile)
+sparkR.stop()

data/benchmark/comparison/ruby.rb

@@ -0,0 +1,167 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'ruby-spark'
+require 'benchmark'
+
+Spark.start
+sc = Spark.context
+
+$log_file = File.open(ENV['RUBY_LOG'], 'w')
+
+def log(*values)
+  $log_file.puts(values.join(';'))
+end
+
+workers = ENV['WORKERS'].to_i
+numbers_count = ENV['NUMBERS_COUNT'].to_i
+text_file = ENV['TEXT_FILE']
+
+numbers = (0...numbers_count).to_a
+floats = numbers.map(&:to_f)
+strings = File.read(text_file).split("\n")
+
+
+# =============================================================================
+# Serialization
+# =============================================================================
+
+time = Benchmark.realtime do
+  @rdd_numbers = sc.parallelize(numbers, workers)
+end
+
+log('NumbersSerialization', time)
+
+
+time = Benchmark.realtime do
+  @rdd_floats = sc.parallelize(floats, workers)
+end
+
+log('FloatsSerialization', time)
+
+
+time = Benchmark.realtime do
+  @rdd_strings = sc.parallelize(strings, workers)
+end
+
+log('StringsSerialization', time)
+
+
+# =============================================================================
+# Computing
+# =============================================================================
+
+
+# --- Is prime? ---------------------------------------------------------------
+
+is_prime = Proc.new do |x|
+  case
+  when x < 2
+    [x, false]
+  when x == 2
+    [x, true]
+  when x % 2 == 0
+    [x, false]
+  else
+    upper = Math.sqrt(x.to_f).to_i
+    result = true
+
+    i = 3
+    while i <= upper
+      if x % i == 0
+        result = false
+        break
+      end
+
+      i += 2
+    end
+
+    [x, result]
+  end
+end
+
+time = Benchmark.realtime do
+  @rdd_numbers.map(is_prime).collect
+end
+
+log('IsPrime', time)
+
+
+# --- Matrix multiplication ---------------------------------------------------
+
+matrix_size = ENV['MATRIX_SIZE'].to_i
+
+matrix = Array.new(matrix_size) do |row|
+  Array.new(matrix_size) do |col|
+    row+col
+  end
+end;
+
+multiplication_func = Proc.new do |matrix|
+  size = matrix.size
+
+  Array.new(size) do |row|
+    Array.new(size) do |col|
+      matrix[row]
+
+      result = 0
+      size.times do |i|
+        result += matrix[row][i] * matrix[col][i]
+      end
+      result
+    end
+  end
+end
+
+time = Benchmark.realtime do
+  rdd = sc.parallelize(matrix, 1)
+  rdd.map_partitions(multiplication_func).collect
+end
+
+log('MatrixMultiplication', time)
+
+
+# --- Pi digits ---------------------------------------------------------------
+# http://rosettacode.org/wiki/Pi#Ruby
+
+pi_digit = ENV['PI_DIGIT'].to_i
+
+pi_func = Proc.new do |size|
+  size = size.first
+  result = ''
+
+  q, r, t, k, n, l = 1, 0, 1, 1, 3, 3
+  while size > 0
+    if 4*q+r-t < n*t
+      result << n.to_s
+      size -= 1
+      nr = 10*(r-n*t)
+      n = ((10*(3*q+r)) / t) - 10*n
+      q *= 10
+      r = nr
+    else
+      nr = (2*q+r) * l
+      nn = (q*(7*k+2)+r*l) / (t*l)
+      q *= k
+      t *= l
+      l += 2
+      k += 1
+      n = nn
+      r = nr
+    end
+  end
+
+  [result]
+end
+
+time = Benchmark.realtime do
+  rdd = sc.parallelize([pi_digit], 1)
+  rdd.map_partitions(pi_func).collect
+end
+
+log('PiDigit', time)
+
+
+$log_file.close