ruby-spark 1.1.0.1-java

data/benchmark/comparison/run-all.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+
+# Current dir
+cd "$(dirname "$0")"
+
+# Exit immediately if a pipeline returns a non-zero status.
+set -e
+
+# Settings
+export WORKERS=2
+export MATRIX_SIZE=100
+export NUMBERS_COUNT=1000000
+export TEXT_FILE=$(mktemp)
+export PI_DIGIT=1000
+export RUBY_BATCH_SIZE=2048
+
+text_file_rows=10
+text_file_per_line=10
+text_file_duplicates=50
+
+mx="4096m"
+ms="4096m"
+
+
+# Parse arguments
+while (( "$#" )); do
+  case $1 in
+    --workers)
+      WORKERS="$2"
+      shift
+      ;;
+    --matrix-size)
+      MATRIX_SIZE="$2"
+      shift
+      ;;
+    --numbers-count)
+      NUMBERS_COUNT="$2"
+      shift
+      ;;
+    --random-file-rows)
+      text_file_rows="$2"
+      shift
+      ;;
+    --text-file-per-line)
+      text_file_per_line="$2"
+      shift
+      ;;
+    --text-file-duplicates)
+      text_file_duplicates="$2"
+      shift
+      ;;
+    --pi-digit)
+      PI_DIGIT="$2"
+      shift
+      ;;
+    --ruby-batch-size)
+      RUBY_BATCH_SIZE="$2"
+      shift
+      ;;
+    --mx)
+      mx="$2"
+      shift
+      ;;
+    --ms)
+      ms="$2"
+      shift
+      ;;
+    *)
+      break
+      ;;
+  esac
+  shift
+done
+
+
+# Generating
+file=$(mktemp)
+
+for (( i=0; i<$text_file_rows; i++ ))
+do
+  shuf -n $text_file_per_line /usr/share/dict/words | tr '\n' ' ' >> $file
+  echo >> $file
+done
+
+for (( i=0; i<$text_file_duplicates; i++ ))
+do
+  cat $file >> $TEXT_FILE
+done
+
+
+# Before run
+if [[ -z "$SPARK_HOME" ]]; then
+  export SPARK_HOME=$(pwd)/spark
+fi
+
+if [[ -z "$RSPARK_HOME" ]]; then
+  export RSPARK_HOME=$(pwd)/rspark
+fi
+
+export SPARK_RUBY_BATCH_SIZE="$RUBY_BATCH_SIZE"
+SPARK_CLASSPATH=$($SPARK_HOME/bin/compute-classpath.sh 2>/dev/null)
+
+export _JAVA_OPTIONS="$_JAVA_OPTIONS -Xms$ms -Xmx$mx"
+
+
+# Log files
+export RUBY_MARSHAL_LOG=$(mktemp)
+export RUBY_OJ_LOG=$(mktemp)
+export PYTHON_LOG=$(mktemp)
+export SCALA_LOG=$(mktemp)
+export R_LOG=$(mktemp)
+
+
+# Run:
+echo "Workers: $WORKERS"
+echo "Matrix size: $MATRIX_SIZE"
+echo "Numbers count: $NUMBERS_COUNT"
+echo "Pi digits: $PI_DIGIT"
+echo "File: rows = $(($text_file_rows * $text_file_duplicates))"
+echo "      per line = $text_file_per_line"
+
+# --- Ruby
+export SPARK_RUBY_SERIALIZER='marshal'
+export RUBY_LOG="$RUBY_MARSHAL_LOG"
+/usr/bin/env ruby ruby.rb &>/dev/null
+
+export SPARK_RUBY_SERIALIZER='oj'
+export RUBY_LOG="$RUBY_OJ_LOG"
+/usr/bin/env ruby ruby.rb &>/dev/null
+
+# # --- Python
+"$SPARK_HOME"/bin/spark-submit --master "local[*]" $(pwd)/python.py &>/dev/null
+
+# # --- Scala
+/usr/bin/env scalac -cp $SPARK_CLASSPATH scala.scala -d scala.jar &>/dev/null
+"$SPARK_HOME"/bin/spark-submit --master "local[*]" $(pwd)/scala.jar &>/dev/null
+
+# --- R
+# "$RSPARK_HOME"/sparkR r.r #&>/dev/null
+
+
+# Parse results
+echo "# Ruby (Marshal)"
+cat $RUBY_MARSHAL_LOG
+echo ""
+
+echo "# Ruby (Oj)"
+cat $RUBY_OJ_LOG
+echo ""
+
+echo "# Python"
+cat $PYTHON_LOG
+echo ""
+
+echo "# Scala"
+cat $SCALA_LOG
+echo ""
+
+echo "# R"
+cat $R_LOG

data/benchmark/comparison/scala.scala

@@ -0,0 +1,181 @@
+import java.io._
+import scala.math
+import scala.io.Source
+import org.apache.spark._
+
+object Scala {
+
+  val logFile = new PrintWriter(new File(System.getenv("SCALA_LOG")))
+
+  def log(args: Any*) {
+    logFile.write(args.mkString(";"))
+    logFile.write("\n")
+  }
+
+  def main(args: Array[String]) {
+    val conf = new SparkConf().setAppName("Scala")
+    val sc = new SparkContext(conf)
+
+    val workers = System.getenv("WORKERS").toInt
+    val numbersCount = System.getenv("NUMBERS_COUNT").toInt
+    val textFile = System.getenv("TEXT_FILE")
+
+    val numbers = 0 until numbersCount
+    val floats = numbers.map(_.toDouble)
+    val strings = Source.fromFile(textFile).mkString.split("\n")
+
+
+    // =============================================================================
+    // Serialization
+    // =============================================================================
+
+    var time: Long = 0
+
+    time = System.currentTimeMillis
+    val rddNumbers = sc.parallelize(numbers, workers)
+    time = System.currentTimeMillis - time
+
+    log("NumbersSerialization", time/1000.0)
+
+
+    time = System.currentTimeMillis
+    val rddFloats = sc.parallelize(floats, workers)
+    time = System.currentTimeMillis - time
+
+    log("FloatsSerialization", time/1000.0)
+
+
+    time = System.currentTimeMillis
+    val rddStrings = sc.parallelize(strings, workers)
+    time = System.currentTimeMillis - time
+
+    log("StringsSerialization", time/1000.0)
+
+
+    // =============================================================================
+    // Computing
+    // =============================================================================
+
+    // --- Is prime? ---------------------------------------------------------------
+
+    time = System.currentTimeMillis
+    val primes = rddNumbers.map{ x =>
+      if(x < 2){
+        (x, false)
+      }
+      else if(x == 2){
+        (x, true)
+      }
+      else if(x % 2 == 0){
+        (x, false)
+      }
+      else{
+        val upper = math.sqrt(x.toDouble).toInt
+        var result = true
+
+        var i = 3
+        while(i <= upper && result == true){
+          if(x % i == 0){
+            result = false
+          }
+          else{
+            i += 2
+          }
+        }
+
+        (x, result)
+      }
+    }
+    primes.collect()
+    time = System.currentTimeMillis - time
+
+    log("IsPrime", time/1000.0)
+
+
+    // --- Matrix multiplication ---------------------------------------------------
+
+    val matrixSize = System.getenv("MATRIX_SIZE").toInt
+
+    val matrix = new Array[Array[Long]](matrixSize)
+
+    for( row <- 0 until matrixSize ) {
+      matrix(row) = new Array[Long](matrixSize)
+      for( col <- 0 until matrixSize ) {
+        matrix(row)(col) = row + col
+      }
+    }
+
+    time = System.currentTimeMillis
+    val rdd = sc.parallelize(matrix, 1)
+    rdd.mapPartitions { it =>
+      val matrix = it.toArray
+      val size = matrix.size
+
+      val newMatrix = new Array[Array[Long]](size)
+
+      for( row <- 0 until size ) {
+        newMatrix(row) = new Array[Long](size)
+        for( col <- 0 until size ) {
+
+          var result: Long = 0
+          for( i <- 0 until size ) {
+            result += matrix(row)(i) * matrix(col)(i)
+          }
+          newMatrix(row)(col) = result
+        }
+      }
+
+      newMatrix.toIterator
+    }
+    time = System.currentTimeMillis - time
+
+    log("MatrixMultiplication", time/1000.0)
+
+
+    // --- Pi digits ---------------------------------------------------------------
+    // http://rosettacode.org/wiki/Pi#Scala
+
+    val piDigit = System.getenv("PI_DIGIT").toInt
+
+    time = System.currentTimeMillis
+    val piDigits = sc.parallelize(Array(piDigit), 1)
+    piDigits.mapPartitions { it =>
+      var size = it.toArray.asInstanceOf[Array[Int]](0)
+      var result = ""
+
+      var r: BigInt = 0
+      var q, t, k: BigInt = 1
+      var n, l: BigInt = 3
+      var nr, nn: BigInt = 0
+
+      while(size > 0){
+        while((4*q+r-t) >= (n*t)){
+          nr = (2*q+r)*l
+          nn = (q*(7*k)+2+(r*l))/(t*l)
+          q = q * k
+          t = t * l
+          l = l + 2
+          k = k + 1
+          n  = nn
+          r  = nr
+        }
+
+        result += n.toString
+        size -= 1
+        nr = 10*(r-n*t)
+        n  = ((10*(3*q+r))/t)-(10*n)
+        q  = q * 10
+        r  = nr
+      }
+
+      Iterator(result)
+    }
+    time = System.currentTimeMillis - time
+
+    log("PiDigit", time/1000.0)
+
+
+    sc.stop()
+    logFile.close()
+  }
+}

data/benchmark/custom_marshal.rb

@@ -0,0 +1,94 @@
+require 'benchmark'
+require 'benchmark/ips'
+
+def pack_int(data)
+  [data].pack('l>')
+end
+
+def pack_long(data)
+  [data].pack('q>')
+end
+
+def pack_doubles(data)
+  data.pack('G*')
+end
+
+module Standard
+  class LabeledPoint
+    def initialize(label, features)
+      @label = label
+      @features = Standard::Vector.new(features)
+    end
+
+    def marshal_dump
+      [@label, @features]
+    end
+
+    def marshal_load(*)
+    end
+  end
+
+  class Vector
+    def initialize(array)
+      @values = array
+    end
+
+    def marshal_dump
+      [@values]
+    end
+
+    def marshal_load(*)
+    end
+  end
+end
+
+module Custom
+  class LabeledPoint
+    def initialize(label, features)
+      @label = label
+      @features = Custom::Vector.new(features)
+    end
+
+    def _dump(*)
+      pack_long(@label) + @features._dump
+    end
+
+    def self._load(*)
+    end
+  end
+
+  class Vector
+    def initialize(array)
+      @values = array
+    end
+
+    def _dump(*)
+      result = 'v'
+      result << pack_int(@values.size)
+      result << pack_doubles(@values)
+      result.encode(Encoding::ASCII_8BIT)
+    end
+
+    def self._load(*)
+    end
+  end
+end
+
+data_size = 10_000
+vector_size = 1_000
+values = Array.new(vector_size) { |x| rand(10_000..100_000) }
+
+@data1 = Array.new(data_size) {|i| Standard::LabeledPoint.new(i, values)}
+@data2 = Array.new(data_size) {|i| Custom::LabeledPoint.new(i, values)}
+
+Benchmark.ips do |r|
+  r.report('standard') do
+    Marshal.dump(@data1)
+  end
+
+  r.report('custom') do
+    Marshal.dump(@data2)
+  end
+
+  r.compare!
+end

data/benchmark/digest.rb

@@ -0,0 +1,150 @@
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+def java?
+  RUBY_PLATFORM =~ /java/
+end
+
+unless java?
+  require 'murmurhash3'
+end
+
+require 'digest'
+require 'benchmark'
+require 'ruby-spark'
+
+TEST = 5_000_000
+WORDS = ["wefwefwef", "rgwefiwefwe", "a", "rujfwgrethrzjrhgawf", "irncrnuggo"]
+
+puts "TEST COUNT = #{TEST*WORDS.size}"
+
+# =================================================================================================
+# Pure ruby mumrumur
+# funny-falcon/murmurhash3-ruby
+
+MASK32 = 0xffffffff
+
+def murmur3_32_rotl(x, r)
+  ((x << r) | (x >> (32 - r))) & MASK32
+end
+
+def murmur3_32_fmix(h)
+  h &= MASK32
+  h ^= h >> 16
+  h = (h * 0x85ebca6b) & MASK32
+  h ^= h >> 13
+  h = (h * 0xc2b2ae35) & MASK32
+  h ^ (h >> 16)
+end
+
+def murmur3_32__mmix(k1)
+  k1 = (k1 * 0xcc9e2d51) & MASK32
+  k1 = murmur3_32_rotl(k1, 15)
+  (k1 * 0x1b873593) & MASK32
+end
+
+def murmur3_32_str_hash(str, seed=0)
+  h1 = seed
+  numbers = str.unpack('V*C*')
+  tailn = str.bytesize % 4
+  tail = numbers.slice!(numbers.size - tailn, tailn)
+  for k1 in numbers
+    h1 ^= murmur3_32__mmix(k1)
+    h1 = murmur3_32_rotl(h1, 13)
+    h1 = (h1*5 + 0xe6546b64) & MASK32
+  end
+
+  unless tail.empty?
+    k1 = 0
+    tail.reverse_each do |c1|
+      k1 = (k1 << 8) | c1
+    end
+    h1 ^= murmur3_32__mmix(k1)
+  end
+
+  h1 ^= str.bytesize
+  murmur3_32_fmix(h1)
+end
+
+
+# =================================================================================================
+# Benchmark
+
+Benchmark.bm(18) do |x|
+
+  x.report("ruby hash"){
+    TEST.times{
+      WORDS.each{ |word|
+        word.hash
+      }
+    }    
+  }
+
+  x.report("ext portable"){
+    TEST.times{
+      WORDS.each{ |word|
+        Spark::Digest.portable_hash(word)
+      }
+    }    
+  }
+
+  x.report("murmur3 32"){
+    TEST.times{
+      WORDS.each{ |word|
+        # MurmurHash3::V128.str_hash(word)
+        # [MurmurHash3::V128.str_hash(word).join.to_i].pack("q>")
+        # MurmurHash3::V128.str_hash(word)
+        # a = MurmurHash3::V32.str_hash(word).to_s
+        # a.slice!(0,8)
+
+        MurmurHash3::V32.str_hash(word)
+      }
+    }    
+  } unless java?
+
+  # Too slow
+  # x.report("murmur3 32 (ruby)"){
+  #   TEST.times{
+  #     WORDS.each{ |word|
+  #       # MurmurHash3::V128.str_hash(word)
+  #       # [MurmurHash3::V128.str_hash(word).join.to_i].pack("q>")
+  #       # MurmurHash3::V128.str_hash(word)
+  #       # a = murmur3_32_str_hash(word).to_s
+  #       # a.slice!(0,8)
+
+  #       murmur3_32_str_hash(word)
+  #     }
+  #   }    
+  # }
+
+  x.report("murmur3 128"){
+    TEST.times{
+      WORDS.each{ |word|
+        # MurmurHash3::V128.str_hash(word)
+        # [MurmurHash3::V128.str_hash(word).join.to_i].pack("q>")
+        # a = MurmurHash3::V128.str_hash(word).to_s
+        # a.slice!(0,8)
+
+        MurmurHash3::V128.str_hash(word)
+      }
+    }    
+  } unless java?
+
+  # x.report("sha256"){
+  #   TEST.times{
+  #     WORDS.each{ |word|
+  #       a = Digest::SHA256.digest(word)
+  #       # a.slice!(0,8)
+  #     }
+  #   }    
+  # }
+
+  # x.report("md5"){
+  #   TEST.times{
+  #     WORDS.each{ |word|
+  #       a = Digest::MD5.digest(word)
+  #       # a.slice!(0,8)
+  #     }
+  #   }    
+  # }
+end