ruby-spark 1.0.0

data/ext/spark/src/main/scala/MarshalDump.scala

@@ -0,0 +1,113 @@
+package org.apache.spark.api.ruby.marshal
+
+import java.io.{DataInputStream, DataOutputStream, ByteArrayInputStream, ByteArrayOutputStream}
+
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.JavaConverters._
+import scala.reflect.{ClassTag, classTag}
+
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.linalg.{Vector, DenseVector, SparseVector}
+
+
+/* =================================================================================================
+ * class MarshalDump
+ * =================================================================================================
+ */
+class MarshalDump(os: DataOutputStream) {
+
+  val NAN_BYTELIST               = "nan".getBytes
+  val NEGATIVE_INFINITY_BYTELIST = "-inf".getBytes
+  val INFINITY_BYTELIST          = "inf".getBytes
+
+  def dump(data: Any) {
+    data match {
+      case null =>
+        os.writeByte('0')
+
+      case item: Boolean =>
+        val char = if(item) 'T' else 'F'
+        os.writeByte(char)
+
+      case item: Int =>
+        os.writeByte('i')
+        dumpInt(item)
+
+      case item: Array[_] =>
+        os.writeByte('[')
+        dumpArray(item)
+
+      case item: Double =>
+        os.writeByte('f')
+        dumpFloat(item)
+
+      case item: ArrayBuffer[Any] => dump(item.toArray)
+    }
+  }
+
+  def dumpInt(data: Int) {
+    if(data == 0){
+      os.writeByte(0)
+    }
+    else if (0 < data && data < 123) {
+      os.writeByte(data + 5)
+    }
+    else if (-124 < data && data < 0) {
+      os.writeByte((data - 5) & 0xff)
+    }
+    else {
+      val buffer = new Array[Byte](4)
+      var value = data
+
+      var i = 0
+      while(i != 4 && value != 0 && value != -1){
+        buffer(i) = (value & 0xff).toByte
+        value = value >> 8
+
+        i += 1
+      }
+      val lenght = i + 1
+      if(value < 0){
+        os.writeByte(-lenght)
+      }
+      else{
+        os.writeByte(lenght)
+      }
+      os.write(buffer, 0, lenght)
+    }
+  }
+
+  def dumpArray(array: Array[_]) {
+    dumpInt(array.size)
+
+    for(item <- array) {
+      dump(item)
+    }
+  }
+
+  def dumpFloat(value: Double) {
+    if(value.isPosInfinity){
+      dumpString(NEGATIVE_INFINITY_BYTELIST)
+    }
+    else if(value.isNegInfinity){
+      dumpString(INFINITY_BYTELIST)
+    }
+    else if(value.isNaN){
+      dumpString(NAN_BYTELIST)
+    }
+    else{
+      // dumpString("%.17g".format(value))
+      dumpString(value.toString)
+    }
+  }
+
+  def dumpString(data: String) {
+    dumpString(data.getBytes)
+  }
+
+  def dumpString(data: Array[Byte]) {
+    dumpInt(data.size)
+    os.write(data)
+  }
+
+}

data/ext/spark/src/main/scala/MarshalLoad.scala

@@ -0,0 +1,220 @@
+package org.apache.spark.api.ruby.marshal
+
+import java.io.{DataInputStream, DataOutputStream, ByteArrayInputStream, ByteArrayOutputStream}
+
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.JavaConverters._
+import scala.reflect.{ClassTag, classTag}
+
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.linalg.{Vector, DenseVector, SparseVector}
+
+
+/* =================================================================================================
+ * class MarshalLoad
+ * =================================================================================================
+ */
+class MarshalLoad(is: DataInputStream) {
+
+  case class WaitForObject()
+
+  val registeredSymbols = ArrayBuffer[String]()
+  val registeredLinks = ArrayBuffer[Any]()
+
+  def load: Any = {
+    load(is.readUnsignedByte.toChar)
+  }
+
+  def load(dataType: Char): Any = {
+    dataType match {
+      case '0' => null
+      case 'T' => true
+      case 'F' => false
+      case 'i' => loadInt
+      case 'f' => loadAndRegisterFloat
+      case ':' => loadAndRegisterSymbol
+      case '[' => loadAndRegisterArray
+      case 'U' => loadAndRegisterUserObject
+      case _ =>
+        throw new IllegalArgumentException(s"Format is not supported: $dataType.")
+    }
+  }
+
+
+  // ----------------------------------------------------------------------------------------------
+  // Load by type
+
+  def loadInt: Int = {
+    var c = is.readByte.toInt
+
+    if (c == 0) {
+      return 0
+    } else if (4 < c && c < 128) {
+      return c - 5
+    } else if (-129 < c && c < -4) {
+      return c + 5
+    }
+
+    var result: Long = 0
+
+    if (c > 0) {
+      result = 0
+      for( i <- 0 until c ) {
+        result |= (is.readUnsignedByte << (8 * i)).toLong
+      }
+    } else {
+      c = -c
+      result = -1
+      for( i <- 0 until c ) {
+        result &= ~((0xff << (8 * i)).toLong)
+        result |= (is.readUnsignedByte << (8 * i)).toLong
+      }
+    }
+
+    result.toInt
+  }
+
+  def loadAndRegisterFloat: Double = {
+    val result = loadFloat
+    registeredLinks += result
+    result
+  }
+
+  def loadFloat: Double = {
+    val string = loadString
+    string match {
+      case "nan"  => Double.NaN
+      case "inf"  => Double.PositiveInfinity
+      case "-inf" => Double.NegativeInfinity
+      case _ => string.toDouble
+    }
+  }
+
+  def loadString: String = {
+    new String(loadStringBytes)
+  }
+
+  def loadStringBytes: Array[Byte] = {
+    val size = loadInt
+    val buffer = new Array[Byte](size)
+
+    var readSize = 0
+    while(readSize < size){
+      val read = is.read(buffer, readSize, size-readSize)
+
+      if(read == -1){
+        throw new IllegalArgumentException("Marshal too short.")
+      }
+
+      readSize += read
+    }
+
+    buffer
+  }
+
+  def loadAndRegisterSymbol: String = {
+    val result = loadString
+    registeredSymbols += result
+    result
+  }
+
+  def loadAndRegisterArray: Array[Any] = {
+    val size = loadInt
+    val array = new Array[Any](size)
+
+    registeredLinks += array
+
+    for( i <- 0 until size ) {
+      array(i) = loadNextObject
+    }
+
+    array
+  }
+
+  def loadAndRegisterUserObject: Any = {
+    val klass = loadNextObject.asInstanceOf[String]
+
+    // Register future class before load the next object
+    registeredLinks += WaitForObject()
+    val index = registeredLinks.size - 1
+
+    val data = loadNextObject
+
+    val result = klass match {
+      case "Spark::Mllib::LabeledPoint" => createLabeledPoint(data)
+      case "Spark::Mllib::DenseVector" => createDenseVector(data)
+      case "Spark::Mllib::SparseVector" => createSparseVector(data)
+      case other =>
+        throw new IllegalArgumentException(s"Object $other is not supported.")
+    }
+
+    registeredLinks(index) = result
+
+    result
+  }
+
+
+  // ----------------------------------------------------------------------------------------------
+  // Other loads
+
+  def loadNextObject: Any = {
+    val dataType = is.readUnsignedByte.toChar
+
+    if(isLinkType(dataType)){
+      readLink(dataType)
+    }
+    else{
+      load(dataType)
+    }
+  }
+
+
+  // ----------------------------------------------------------------------------------------------
+  // To java objects
+
+  def createLabeledPoint(data: Any): LabeledPoint = {
+    val array = data.asInstanceOf[Array[_]]
+    new LabeledPoint(array(0).asInstanceOf[Double], array(1).asInstanceOf[Vector])
+  }
+
+  def createDenseVector(data: Any): DenseVector = {
+    new DenseVector(data.asInstanceOf[Array[_]].map(toDouble(_)))
+  }
+
+  def createSparseVector(data: Any): SparseVector = {
+    val array = data.asInstanceOf[Array[_]]
+    val size = array(0).asInstanceOf[Int]
+    val indices = array(1).asInstanceOf[Array[_]].map(_.asInstanceOf[Int])
+    val values = array(2).asInstanceOf[Array[_]].map(toDouble(_))
+
+    new SparseVector(size, indices, values)
+  }
+
+
+  // ----------------------------------------------------------------------------------------------
+  // Helpers
+
+  def toDouble(data: Any): Double = data match {
+    case x: Int => x.toDouble
+    case x: Double => x
+    case _ => 0.0
+  }
+
+
+  // ----------------------------------------------------------------------------------------------
+  // Cache
+
+  def readLink(dataType: Char): Any = {
+    val index = loadInt
+
+    dataType match {
+      case '@' => registeredLinks(index)
+      case ';' => registeredSymbols(index)
+    }
+  }
+
+  def isLinkType(dataType: Char): Boolean = {
+    dataType == ';' || dataType == '@'
+  }
+
+}

data/ext/spark/src/main/scala/RubyAccumulatorParam.scala

@@ -0,0 +1,69 @@
+package org.apache.spark.api.ruby
+
+import java.io._
+import java.net._
+import java.util.{List, ArrayList}
+
+import scala.collection.JavaConversions._
+import scala.collection.immutable._
+
+import org.apache.spark._
+import org.apache.spark.util.Utils
+
+/**
+ * Internal class that acts as an `AccumulatorParam` for Ruby accumulators. Inside, it
+ * collects a list of pickled strings that we pass to Ruby through a socket.
+ */
+private class RubyAccumulatorParam(serverHost: String, serverPort: Int)
+  extends AccumulatorParam[List[Array[Byte]]] {
+
+  // Utils.checkHost(serverHost, "Expected hostname")
+
+  val bufferSize = SparkEnv.get.conf.getInt("spark.buffer.size", 65536)
+
+  // Socket shoudl not be serialized
+  // Otherwise: SparkException: Task not serializable
+  @transient var socket: Socket = null
+  @transient var socketOutputStream: DataOutputStream = null
+  @transient var socketInputStream:  DataInputStream = null
+
+  def openSocket(){
+    synchronized {
+      if (socket == null || socket.isClosed) {
+        socket = new Socket(serverHost, serverPort)
+
+        socketInputStream  = new DataInputStream(new BufferedInputStream(socket.getInputStream, bufferSize))
+        socketOutputStream = new DataOutputStream(new BufferedOutputStream(socket.getOutputStream, bufferSize))
+      }
+    }
+  }
+
+  override def zero(value: List[Array[Byte]]): List[Array[Byte]] = new ArrayList
+
+  override def addInPlace(val1: List[Array[Byte]], val2: List[Array[Byte]]) : List[Array[Byte]] = synchronized {
+    if (serverHost == null) {
+      // This happens on the worker node, where we just want to remember all the updates
+      val1.addAll(val2)
+      val1
+    } else {
+      // This happens on the master, where we pass the updates to Ruby through a socket
+      openSocket()
+
+      socketOutputStream.writeInt(val2.size)
+      for (array <- val2) {
+        socketOutputStream.writeInt(array.length)
+        socketOutputStream.write(array)
+      }
+      socketOutputStream.flush()
+
+      // Wait for acknowledgement
+      // http://stackoverflow.com/questions/28560133/ruby-server-java-scala-client-deadlock
+      //
+      // if(in.readInt() != RubyConstant.ACCUMULATOR_ACK){
+      //   throw new SparkException("Accumulator was not acknowledged")
+      // }
+
+      new ArrayList
+    }
+  }
+}

data/ext/spark/src/main/scala/RubyBroadcast.scala

@@ -0,0 +1,13 @@
+package org.apache.spark.api.ruby
+
+import org.apache.spark.api.python.PythonBroadcast
+
+/**
+ * An Wrapper for Ruby Broadcast, which is written into disk by Ruby. It also will
+ * write the data into disk after deserialization, then Ruby can read it from disks.
+ *
+ * Class use Python logic - only for semantic
+ */
+class RubyBroadcast(@transient var _path: String, @transient var id: java.lang.Long) extends PythonBroadcast(_path) {
+
+}

data/ext/spark/src/main/scala/RubyConstant.scala

@@ -0,0 +1,13 @@
+package org.apache.spark.api.ruby
+
+object RubyConstant {
+  val DATA_EOF = -2
+  val WORKER_ERROR = -1
+  val WORKER_DONE = 0
+  val CREATE_WORKER = 1
+  val KILL_WORKER = 2
+  val KILL_WORKER_AND_WAIT = 3
+  val SUCCESSFULLY_KILLED = 4
+  val UNSUCCESSFUL_KILLING = 5
+  val ACCUMULATOR_ACK = 6
+}

data/ext/spark/src/main/scala/RubyMLLibAPI.scala

@@ -0,0 +1,55 @@
+package org.apache.spark.mllib.api.ruby
+
+import java.util.ArrayList
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.api.java.JavaRDD
+import org.apache.spark.mllib.linalg._
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.classification.NaiveBayes
+import org.apache.spark.mllib.clustering.GaussianMixtureModel
+import org.apache.spark.mllib.stat.distribution.MultivariateGaussian
+import org.apache.spark.mllib.api.python.MLLibAPI
+
+
+class RubyMLLibAPI extends MLLibAPI {
+  // trainLinearRegressionModelWithSGD
+  // trainLassoModelWithSGD
+  // trainRidgeModelWithSGD
+  // trainLogisticRegressionModelWithSGD
+  // trainLogisticRegressionModelWithLBFGS
+  // trainSVMModelWithSGD
+  // trainKMeansModel
+  // trainGaussianMixture
+
+  // Rjb have a problem with theta: Array[Array[Double]]
+  override def trainNaiveBayes(data: JavaRDD[LabeledPoint], lambda: Double) = {
+    val model = NaiveBayes.train(data.rdd, lambda)
+
+    List(
+      Vectors.dense(model.labels),
+      Vectors.dense(model.pi),
+      model.theta.toSeq
+    ).map(_.asInstanceOf[Object]).asJava
+  }
+
+  // On python is wt just Object
+  def predictSoftGMM(
+      data: JavaRDD[Vector],
+      wt: ArrayList[Object],
+      mu: ArrayList[Object],
+      si: ArrayList[Object]): RDD[Array[Double]] = {
+
+      // val weight = wt.asInstanceOf[Array[Double]]
+      val weight = wt.toArray.map(_.asInstanceOf[Double])
+      val mean = mu.toArray.map(_.asInstanceOf[DenseVector])
+      val sigma = si.toArray.map(_.asInstanceOf[DenseMatrix])
+      val gaussians = Array.tabulate(weight.length){
+        i => new MultivariateGaussian(mean(i), sigma(i))
+      }
+      val model = new GaussianMixtureModel(weight, gaussians)
+      model.predictSoft(data)
+  }
+}