ruby-spark 1.0.0

data/benchmark/performance/r.r

@@ -0,0 +1,69 @@
+library(SparkR)
+sc <- sparkR.init(master="local[*]")
+
+logFile <- file(Sys.getenv("R_LOG"), "w")
+
+logInfo <- function(...){
+  args <- list(...)
+  line <- paste(args, collapse = ";")
+  writeLines(line, logFile)
+}
+
+workers <- as.integer(Sys.getenv('WORKERS'))
+numbersCount <- as.integer(Sys.getenv('NUMBERS_COUNT'))
+textFile <- Sys.getenv('TEXT_FILE')
+
+
+# =============================================================================
+# Serialization
+# =============================================================================
+
+time <- proc.time()
+rddNumbers <- parallelize(sc, as.numeric(seq(0, numbersCount)), workers)
+time <- as.double(proc.time()-time)[3]
+
+logInfo('NumbersSerialization', time)
+
+
+# =============================================================================
+# Computing
+# =============================================================================
+
+isPrime = function(x) {
+  if(x < 2){
+    c(x, FALSE)
+  }
+  else if(x == 2){
+    c(x, TRUE)
+  }
+  else if(x %% 2 == 0){
+    c(x, FALSE)
+  }
+  else{
+    upper <- as.numeric(sqrt(as.double(x)))
+    result <- TRUE
+
+    i <- 3
+    while(i <= upper){
+      if(x %% i == 0){
+        result = FALSE
+        break
+      }
+
+      i <- i+2
+    }
+
+    c(x, result)
+  }
+}
+
+time <- proc.time()
+rdd <- map(rddNumbers, isPrime)
+capture.output(collect(rdd), file='/dev/null')
+time <- as.double(proc.time()-time)[3]
+
+logInfo('IsPrime', time)
+
+
+close(logFile)
+sparkR.stop()

data/benchmark/performance/ruby.rb

@@ -0,0 +1,167 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'ruby-spark'
+require 'benchmark'
+
+Spark.start
+sc = Spark.context
+
+$log_file = File.open(ENV['RUBY_LOG'], 'w')
+
+def log(*values)
+  $log_file.puts(values.join(';'))
+end
+
+workers = ENV['WORKERS'].to_i
+numbers_count = ENV['NUMBERS_COUNT'].to_i
+text_file = ENV['TEXT_FILE']
+
+numbers = (0...numbers_count).to_a
+floats = numbers.map(&:to_f)
+strings = File.read(text_file).split("\n")
+
+
+# =============================================================================
+# Serialization
+# =============================================================================
+
+time = Benchmark.realtime do
+  @rdd_numbers = sc.parallelize(numbers, workers)
+end
+
+log('NumbersSerialization', time)
+
+
+time = Benchmark.realtime do
+  @rdd_floats = sc.parallelize(floats, workers)
+end
+
+log('FloatsSerialization', time)
+
+
+time = Benchmark.realtime do
+  @rdd_strings = sc.parallelize(strings, workers)
+end
+
+log('StringsSerialization', time)
+
+
+# =============================================================================
+# Computing
+# =============================================================================
+
+
+# --- Is prime? ---------------------------------------------------------------
+
+is_prime = Proc.new do |x|
+  case
+  when x < 2
+    [x, false]
+  when x == 2
+    [x, true]
+  when x % 2 == 0
+    [x, false]
+  else
+    upper = Math.sqrt(x.to_f).to_i
+    result = true
+
+    i = 3
+    while i <= upper
+      if x % i == 0
+        result = false
+        break
+      end
+
+      i += 2
+    end
+
+    [x, result]
+  end
+end
+
+time = Benchmark.realtime do
+  @rdd_numbers.map(is_prime).collect
+end
+
+log('IsPrime', time)
+
+
+# --- Matrix multiplication ---------------------------------------------------
+
+matrix_size = ENV['MATRIX_SIZE'].to_i
+
+matrix = Array.new(matrix_size) do |row|
+  Array.new(matrix_size) do |col|
+    row+col
+  end
+end;
+
+multiplication_func = Proc.new do |matrix|
+  size = matrix.size
+
+  Array.new(size) do |row|
+    Array.new(size) do |col|
+      matrix[row]
+
+      result = 0
+      size.times do |i|
+        result += matrix[row][i] * matrix[col][i]
+      end
+      result
+    end
+  end
+end
+
+time = Benchmark.realtime do
+  rdd = sc.parallelize(matrix, 1)
+  rdd.map_partitions(multiplication_func).collect
+end
+
+log('MatrixMultiplication', time)
+
+
+# --- Pi digits ---------------------------------------------------------------
+# http://rosettacode.org/wiki/Pi#Ruby
+
+pi_digit = ENV['PI_DIGIT'].to_i
+
+pi_func = Proc.new do |size|
+  size = size.first
+  result = ''
+
+  q, r, t, k, n, l = 1, 0, 1, 1, 3, 3
+  while size > 0
+    if 4*q+r-t < n*t
+      result << n.to_s
+      size -= 1
+      nr = 10*(r-n*t)
+      n = ((10*(3*q+r)) / t) - 10*n
+      q *= 10
+      r = nr
+    else
+      nr = (2*q+r) * l
+      nn = (q*(7*k+2)+r*l) / (t*l)
+      q *= k
+      t *= l
+      l += 2
+      k += 1
+      n = nn
+      r = nr
+    end
+  end
+
+  [result]
+end
+
+time = Benchmark.realtime do
+  rdd = sc.parallelize([pi_digit], 1)
+  rdd.map_partitions(pi_func).collect
+end
+
+log('PiDigit', time)
+
+
+$log_file.close

data/benchmark/performance/run-all.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+
+# Current dir
+cd "$(dirname "$0")"
+
+# Exit immediately if a pipeline returns a non-zero status.
+set -e
+
+# Settings
+export WORKERS=2
+export MATRIX_SIZE=100
+export NUMBERS_COUNT=1000000
+export TEXT_FILE=$(mktemp)
+export PI_DIGIT=1000
+export RUBY_BATCH_SIZE=2048
+
+text_file_rows=10
+text_file_per_line=10
+text_file_duplicates=50
+
+mx="4096m"
+ms="4096m"
+
+
+# Parse arguments
+while (( "$#" )); do
+  case $1 in
+    --workers)
+      WORKERS="$2"
+      shift
+      ;;
+    --matrix-size)
+      MATRIX_SIZE="$2"
+      shift
+      ;;
+    --numbers-count)
+      NUMBERS_COUNT="$2"
+      shift
+      ;;
+    --random-file-rows)
+      text_file_rows="$2"
+      shift
+      ;;
+    --text-file-per-line)
+      text_file_per_line="$2"
+      shift
+      ;;
+    --text-file-duplicates)
+      text_file_duplicates="$2"
+      shift
+      ;;
+    --pi-digit)
+      PI_DIGIT="$2"
+      shift
+      ;;
+    --ruby-batch-size)
+      RUBY_BATCH_SIZE="$2"
+      shift
+      ;;
+    --mx)
+      mx="$2"
+      shift
+      ;;
+    --ms)
+      ms="$2"
+      shift
+      ;;
+    *)
+      break
+      ;;
+  esac
+  shift
+done
+
+
+# Generating
+file=$(mktemp)
+
+for (( i=0; i<$text_file_rows; i++ ))
+do
+  shuf -n $text_file_per_line /usr/share/dict/words | tr '\n' ' ' >> $file
+  echo >> $file
+done
+
+for (( i=0; i<$text_file_duplicates; i++ ))
+do
+  cat $file >> $TEXT_FILE
+done
+
+
+# Before run
+if [[ -z "$SPARK_HOME" ]]; then
+  export SPARK_HOME=$(pwd)/spark
+fi
+
+if [[ -z "$RSPARK_HOME" ]]; then
+  export RSPARK_HOME=$(pwd)/rspark
+fi
+
+export SPARK_RUBY_BATCH_SIZE="$RUBY_BATCH_SIZE"
+SPARK_CLASSPATH=$($SPARK_HOME/bin/compute-classpath.sh 2>/dev/null)
+
+export _JAVA_OPTIONS="$_JAVA_OPTIONS -Xms$ms -Xmx$mx"
+
+
+# Log files
+export RUBY_MARSHAL_LOG=$(mktemp)
+export RUBY_OJ_LOG=$(mktemp)
+export PYTHON_LOG=$(mktemp)
+export SCALA_LOG=$(mktemp)
+export R_LOG=$(mktemp)
+
+
+# Run:
+echo "Workers: $WORKERS"
+echo "Matrix size: $MATRIX_SIZE"
+echo "Numbers count: $NUMBERS_COUNT"
+echo "Pi digits: $PI_DIGIT"
+echo "File: rows = $(($text_file_rows * $text_file_duplicates))"
+echo "      per line = $text_file_per_line"
+
+# --- Ruby
+export SPARK_RUBY_SERIALIZER='marshal'
+export RUBY_LOG="$RUBY_MARSHAL_LOG"
+/usr/bin/env ruby ruby.rb &>/dev/null
+
+export SPARK_RUBY_SERIALIZER='oj'
+export RUBY_LOG="$RUBY_OJ_LOG"
+/usr/bin/env ruby ruby.rb &>/dev/null
+
+# # --- Python
+"$SPARK_HOME"/bin/spark-submit --master "local[*]" $(pwd)/python.py &>/dev/null
+
+# # --- Scala
+/usr/bin/env scalac -cp $SPARK_CLASSPATH scala.scala -d scala.jar &>/dev/null
+"$SPARK_HOME"/bin/spark-submit --master "local[*]" $(pwd)/scala.jar &>/dev/null
+
+# --- R
+# "$RSPARK_HOME"/sparkR r.r #&>/dev/null
+
+
+# Parse results
+echo "# Ruby (Marshal)"
+cat $RUBY_MARSHAL_LOG
+echo ""
+
+echo "# Ruby (Oj)"
+cat $RUBY_OJ_LOG
+echo ""
+
+echo "# Python"
+cat $PYTHON_LOG
+echo ""
+
+echo "# Scala"
+cat $SCALA_LOG
+echo ""
+
+echo "# R"
+cat $R_LOG

data/benchmark/performance/scala.scala

@@ -0,0 +1,181 @@
+import java.io._
+import scala.math
+import scala.io.Source
+import org.apache.spark._
+
+object Scala {
+
+  val logFile = new PrintWriter(new File(System.getenv("SCALA_LOG")))
+
+  def log(args: Any*) {
+    logFile.write(args.mkString(";"))
+    logFile.write("\n")
+  }
+
+  def main(args: Array[String]) {
+    val conf = new SparkConf().setAppName("Scala")
+    val sc = new SparkContext(conf)
+
+    val workers = System.getenv("WORKERS").toInt
+    val numbersCount = System.getenv("NUMBERS_COUNT").toInt
+    val textFile = System.getenv("TEXT_FILE")
+
+    val numbers = 0 until numbersCount
+    val floats = numbers.map(_.toDouble)
+    val strings = Source.fromFile(textFile).mkString.split("\n")
+
+
+    // =============================================================================
+    // Serialization
+    // =============================================================================
+
+    var time: Long = 0
+
+    time = System.currentTimeMillis
+    val rddNumbers = sc.parallelize(numbers, workers)
+    time = System.currentTimeMillis - time
+
+    log("NumbersSerialization", time/1000.0)
+
+
+    time = System.currentTimeMillis
+    val rddFloats = sc.parallelize(floats, workers)
+    time = System.currentTimeMillis - time
+
+    log("FloatsSerialization", time/1000.0)
+
+
+    time = System.currentTimeMillis
+    val rddStrings = sc.parallelize(strings, workers)
+    time = System.currentTimeMillis - time
+
+    log("StringsSerialization", time/1000.0)
+
+
+    // =============================================================================
+    // Computing
+    // =============================================================================
+
+    // --- Is prime? ---------------------------------------------------------------
+
+    time = System.currentTimeMillis
+    val primes = rddNumbers.map{ x =>
+      if(x < 2){
+        (x, false)
+      }
+      else if(x == 2){
+        (x, true)
+      }
+      else if(x % 2 == 0){
+        (x, false)
+      }
+      else{
+        val upper = math.sqrt(x.toDouble).toInt
+        var result = true
+
+        var i = 3
+        while(i <= upper && result == true){
+          if(x % i == 0){
+            result = false
+          }
+          else{
+            i += 2
+          }
+        }
+
+        (x, result)
+      }
+    }
+    primes.collect()
+    time = System.currentTimeMillis - time
+
+    log("IsPrime", time/1000.0)
+
+
+    // --- Matrix multiplication ---------------------------------------------------
+
+    val matrixSize = System.getenv("MATRIX_SIZE").toInt
+
+    val matrix = new Array[Array[Long]](matrixSize)
+
+    for( row <- 0 until matrixSize ) {
+      matrix(row) = new Array[Long](matrixSize)
+      for( col <- 0 until matrixSize ) {
+        matrix(row)(col) = row + col
+      }
+    }
+
+    time = System.currentTimeMillis
+    val rdd = sc.parallelize(matrix, 1)
+    rdd.mapPartitions { it =>
+      val matrix = it.toArray
+      val size = matrix.size
+
+      val newMatrix = new Array[Array[Long]](size)
+
+      for( row <- 0 until size ) {
+        newMatrix(row) = new Array[Long](size)
+        for( col <- 0 until size ) {
+
+          var result: Long = 0
+          for( i <- 0 until size ) {
+            result += matrix(row)(i) * matrix(col)(i)
+          }
+          newMatrix(row)(col) = result
+        }
+      }
+
+      newMatrix.toIterator
+    }
+    time = System.currentTimeMillis - time
+
+    log("MatrixMultiplication", time/1000.0)
+
+
+    // --- Pi digits ---------------------------------------------------------------
+    // http://rosettacode.org/wiki/Pi#Scala
+
+    val piDigit = System.getenv("PI_DIGIT").toInt
+
+    time = System.currentTimeMillis
+    val piDigits = sc.parallelize(Array(piDigit), 1)
+    piDigits.mapPartitions { it =>
+      var size = it.toArray.asInstanceOf[Array[Int]](0)
+      var result = ""
+
+      var r: BigInt = 0
+      var q, t, k: BigInt = 1
+      var n, l: BigInt = 3
+      var nr, nn: BigInt = 0
+
+      while(size > 0){
+        while((4*q+r-t) >= (n*t)){
+          nr = (2*q+r)*l
+          nn = (q*(7*k)+2+(r*l))/(t*l)
+          q = q * k
+          t = t * l
+          l = l + 2
+          k = k + 1
+          n  = nn
+          r  = nr
+        }
+
+        result += n.toString
+        size -= 1
+        nr = 10*(r-n*t)
+        n  = ((10*(3*q+r))/t)-(10*n)
+        q  = q * 10
+        r  = nr
+      }
+
+      Iterator(result)
+    }
+    time = System.currentTimeMillis - time
+
+    log("PiDigit", time/1000.0)
+
+
+    sc.stop()
+    logFile.close()
+  }
+}