ruby-spark 1.1.0.1-java

data/ext/spark/build.sbt

@@ -0,0 +1,73 @@
+import AssemblyKeys._
+
+assemblySettings
+
+// Default values
+val defaultScalaVersion     = "2.10.4"
+val defaultSparkVersion     = "1.3.0"
+val defaultSparkCoreVersion = "2.10"
+val defaultSparkHome        = "target"
+val defaultHadoopVersion    = "1.0.4"
+
+// Values
+val _scalaVersion     = scala.util.Properties.envOrElse("SCALA_VERSION", defaultScalaVersion)
+val _sparkVersion     = scala.util.Properties.envOrElse("SPARK_VERSION", defaultSparkVersion)
+val _sparkCoreVersion = scala.util.Properties.envOrElse("SPARK_CORE_VERSION", defaultSparkCoreVersion)
+val _sparkHome        = scala.util.Properties.envOrElse("SPARK_HOME", defaultSparkHome)
+val _hadoopVersion    = scala.util.Properties.envOrElse("HADOOP_VERSION", defaultHadoopVersion)
+
+// Project settings
+name := "ruby-spark"
+
+version := "1.0.0"
+
+scalaVersion := _scalaVersion
+
+javacOptions ++= Seq("-source", "1.7", "-target", "1.7")
+
+// Jar target folder
+artifactPath in Compile in packageBin := file(s"${_sparkHome}/ruby-spark.jar")
+outputPath in packageDependency := file(s"${_sparkHome}/ruby-spark-deps.jar")
+
+// Protocol buffer support
+seq(sbtprotobuf.ProtobufPlugin.protobufSettings: _*)
+
+// Additional libraries
+libraryDependencies ++= Seq(
+  "org.apache.spark"  %% "spark-core"    % _sparkVersion excludeAll(ExclusionRule(organization = "org.apache.hadoop")),
+  "org.apache.spark"  %% "spark-graphx"  % _sparkVersion,
+  "org.apache.spark"  %% "spark-mllib"   % _sparkVersion,
+  "org.apache.hadoop" %  "hadoop-client" % _hadoopVersion,
+  "com.github.fommil.netlib" % "all" % "1.1.2",
+  "org.scalatest" % "scalatest_2.10" % "2.2.1" % "test"
+)
+
+// Repositories
+resolvers ++= Seq(
+  "JBoss Repository"     at "http://repository.jboss.org/nexus/content/repositories/releases/",
+  "Spray Repository"     at "http://repo.spray.cc/",
+  "Cloudera Repository"  at "https://repository.cloudera.com/artifactory/cloudera-repos/",
+  "Akka Repository"      at "http://repo.akka.io/releases/",
+  "Twitter4J Repository" at "http://twitter4j.org/maven2/",
+  "Apache HBase"         at "https://repository.apache.org/content/repositories/releases",
+  "Twitter Maven Repo"   at "http://maven.twttr.com/",
+  "scala-tools"          at "https://oss.sonatype.org/content/groups/scala-tools",
+  "Typesafe repository"  at "http://repo.typesafe.com/typesafe/releases/",
+  "Second Typesafe repo" at "http://repo.typesafe.com/typesafe/maven-releases/",
+  "Mesosphere Public Repository" at "http://downloads.mesosphere.io/maven",
+  Resolver.sonatypeRepo("public")
+)
+
+// Merge strategy
+mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
+  {
+    case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
+    case m if m.startsWith("META-INF") => MergeStrategy.discard
+    case PathList("javax", "servlet", xs @ _*) => MergeStrategy.first
+    case PathList("org", "apache", xs @ _*) => MergeStrategy.first
+    case PathList("org", "jboss", xs @ _*) => MergeStrategy.first
+    case "about.html"  => MergeStrategy.rename
+    case "reference.conf" => MergeStrategy.concat
+    case _ => MergeStrategy.first
+  }
+}

data/ext/spark/project/plugins.sbt

@@ -0,0 +1,9 @@
+resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns)
+
+resolvers += "Typesafe Repository" at "http://repo.typesafe.com/typesafe/releases/"
+
+resolvers += "Spray Repository" at "http://repo.spray.cc/"
+
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.10.2")
+
+addSbtPlugin("com.github.gseitz" % "sbt-protobuf" % "0.3.3")

data/ext/spark/sbt/sbt

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# This script launches sbt for this project. If present it uses the system
+# version of sbt. If there is no system version of sbt it attempts to download
+# sbt locally.
+SBT_VERSION=0.13.7
+URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+JAR=sbt/sbt-launch-${SBT_VERSION}.jar
+
+# Download sbt launch jar if it hasn't been downloaded yet
+if [ ! -f ${JAR} ]; then
+  # Download
+  printf "Attempting to fetch sbt\n"
+  JAR_DL=${JAR}.part
+  if hash curl 2>/dev/null; then
+    (curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
+  elif hash wget 2>/dev/null; then
+    (wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR}
+  else
+    printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
+    exit -1
+  fi
+fi
+if [ ! -f ${JAR} ]; then
+  # We failed to download
+  printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n"
+  exit -1
+fi
+printf "Launching sbt from ${JAR}\n"
+java \
+  -Xmx1200m -XX:MaxPermSize=350m -XX:ReservedCodeCacheSize=256m \
+  -jar ${JAR} \
+  "$@"

data/ext/spark/src/main/scala/Exec.scala

@@ -0,0 +1,91 @@
+package org.apache.spark.api.ruby
+
+import java.io.{File, FileOutputStream, InputStreamReader, BufferedReader}
+
+import scala.collection.JavaConversions._
+
+import org.apache.spark.{SparkEnv, Logging}
+import org.apache.spark.util._
+
+
+/* =================================================================================================
+ * class FileCommand
+ * =================================================================================================
+ *
+ * Save command to file and than execute him because from Scala you cannot simply run
+ * something like "bash --norc -i -c 'source .zshrc; ruby master.rb'"
+ */
+
+class FileCommand(command: String) extends Logging {
+
+  var pb: ProcessBuilder = null
+  var file: File = null
+
+  // Command is complete.
+  def this(command: String, env: SparkEnv) = {
+    this(command)
+    create(env)
+  }
+
+  // Template must contains %s which will be replaced for command
+  def this(template: String, command: String, env: SparkEnv, envVars: Map[String, String]) = {
+    this(template.format(command), env)
+    setEnvVars(envVars)
+  }
+
+  private def create(env: SparkEnv) {
+    val dir = new File(env.sparkFilesDir)
+    val ext = if(Utils.isWindows) ".cmd" else ".sh"
+    val shell = if(Utils.isWindows) "cmd" else "bash"
+
+    file = File.createTempFile("command", ext, dir)
+
+    val out = new FileOutputStream(file)
+    out.write(command.getBytes)
+    out.close
+
+    logInfo(s"New FileCommand at ${file.getAbsolutePath}")
+
+    pb = new ProcessBuilder(shell, file.getAbsolutePath)
+  }
+
+  def setEnvVars(vars: Map[String, String]) {
+    pb.environment().putAll(vars)
+  }
+
+  def run = {
+    new ExecutedFileCommand(pb.start)
+  }
+}
+
+
+/* =================================================================================================
+ * class ExecutedFileCommand
+ * =================================================================================================
+ *
+ * Represent process executed from file.
+ */
+
+class ExecutedFileCommand(process: Process) {
+
+  var reader: BufferedReader = null
+
+  def readLine = {
+    openInput
+    reader.readLine.toString.trim
+  }
+
+  def openInput {
+    if(reader != null){
+      return
+    }
+
+    val input = new InputStreamReader(process.getInputStream)
+    reader = new BufferedReader(input)
+  }
+
+  // Delegation
+  def destroy = process.destroy
+  def getInputStream = process.getInputStream
+  def getErrorStream = process.getErrorStream
+}

data/ext/spark/src/main/scala/MLLibAPI.scala

@@ -0,0 +1,4 @@
+package org.apache.spark.mllib.api.python
+
+// PythonMLLibAPI is private for python
+class MLLibAPI extends PythonMLLibAPI {}

data/ext/spark/src/main/scala/Marshal.scala

@@ -0,0 +1,52 @@
+package org.apache.spark.api.ruby.marshal
+
+import java.io.{DataInputStream, DataOutputStream, ByteArrayInputStream, ByteArrayOutputStream}
+
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.JavaConverters._
+
+
+/* =================================================================================================
+ * object Marshal
+ * =================================================================================================
+ */
+object Marshal {
+  def load(bytes: Array[Byte]) = {
+    val is = new DataInputStream(new ByteArrayInputStream(bytes))
+
+    val majorVersion = is.readUnsignedByte // 4
+    val minorVersion = is.readUnsignedByte // 8
+
+    (new MarshalLoad(is)).load
+  }
+
+  def dump(data: Any) = {
+    val aos = new ByteArrayOutputStream
+    val os = new DataOutputStream(aos)
+
+    os.writeByte(4)
+    os.writeByte(8)
+
+    (new MarshalDump(os)).dump(data)
+    aos.toByteArray
+  }
+}
+
+
+/* =================================================================================================
+ * class IterableMarshaller
+ * =================================================================================================
+ */
+class IterableMarshaller(iter: Iterator[Any]) extends Iterator[Array[Byte]] {
+  private val buffer = new ArrayBuffer[Any]
+
+  override def hasNext: Boolean = iter.hasNext
+
+  override def next(): Array[Byte] = {
+    while (iter.hasNext) {
+      buffer += iter.next()
+    }
+
+    Marshal.dump(buffer)
+  }
+}

data/ext/spark/src/main/scala/MarshalDump.scala

@@ -0,0 +1,113 @@
+package org.apache.spark.api.ruby.marshal
+
+import java.io.{DataInputStream, DataOutputStream, ByteArrayInputStream, ByteArrayOutputStream}
+
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.JavaConverters._
+import scala.reflect.{ClassTag, classTag}
+
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.linalg.{Vector, DenseVector, SparseVector}
+
+
+/* =================================================================================================
+ * class MarshalDump
+ * =================================================================================================
+ */
+class MarshalDump(os: DataOutputStream) {
+
+  val NAN_BYTELIST               = "nan".getBytes
+  val NEGATIVE_INFINITY_BYTELIST = "-inf".getBytes
+  val INFINITY_BYTELIST          = "inf".getBytes
+
+  def dump(data: Any) {
+    data match {
+      case null =>
+        os.writeByte('0')
+
+      case item: Boolean =>
+        val char = if(item) 'T' else 'F'
+        os.writeByte(char)
+
+      case item: Int =>
+        os.writeByte('i')
+        dumpInt(item)
+
+      case item: Array[_] =>
+        os.writeByte('[')
+        dumpArray(item)
+
+      case item: Double =>
+        os.writeByte('f')
+        dumpFloat(item)
+
+      case item: ArrayBuffer[Any] => dump(item.toArray)
+    }
+  }
+
+  def dumpInt(data: Int) {
+    if(data == 0){
+      os.writeByte(0)
+    }
+    else if (0 < data && data < 123) {
+      os.writeByte(data + 5)
+    }
+    else if (-124 < data && data < 0) {
+      os.writeByte((data - 5) & 0xff)
+    }
+    else {
+      val buffer = new Array[Byte](4)
+      var value = data
+
+      var i = 0
+      while(i != 4 && value != 0 && value != -1){
+        buffer(i) = (value & 0xff).toByte
+        value = value >> 8
+
+        i += 1
+      }
+      val lenght = i + 1
+      if(value < 0){
+        os.writeByte(-lenght)
+      }
+      else{
+        os.writeByte(lenght)
+      }
+      os.write(buffer, 0, lenght)
+    }
+  }
+
+  def dumpArray(array: Array[_]) {
+    dumpInt(array.size)
+
+    for(item <- array) {
+      dump(item)
+    }
+  }
+
+  def dumpFloat(value: Double) {
+    if(value.isPosInfinity){
+      dumpString(NEGATIVE_INFINITY_BYTELIST)
+    }
+    else if(value.isNegInfinity){
+      dumpString(INFINITY_BYTELIST)
+    }
+    else if(value.isNaN){
+      dumpString(NAN_BYTELIST)
+    }
+    else{
+      // dumpString("%.17g".format(value))
+      dumpString(value.toString)
+    }
+  }
+
+  def dumpString(data: String) {
+    dumpString(data.getBytes)
+  }
+
+  def dumpString(data: Array[Byte]) {
+    dumpInt(data.size)
+    os.write(data)
+  }
+
+}

data/ext/spark/src/main/scala/MarshalLoad.scala

@@ -0,0 +1,220 @@
+package org.apache.spark.api.ruby.marshal
+
+import java.io.{DataInputStream, DataOutputStream, ByteArrayInputStream, ByteArrayOutputStream}
+
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.JavaConverters._
+import scala.reflect.{ClassTag, classTag}
+
+import org.apache.spark.mllib.regression.LabeledPoint
+import org.apache.spark.mllib.linalg.{Vector, DenseVector, SparseVector}
+
+
+/* =================================================================================================
+ * class MarshalLoad
+ * =================================================================================================
+ */
+class MarshalLoad(is: DataInputStream) {
+
+  case class WaitForObject()
+
+  val registeredSymbols = ArrayBuffer[String]()
+  val registeredLinks = ArrayBuffer[Any]()
+
+  def load: Any = {
+    load(is.readUnsignedByte.toChar)
+  }
+
+  def load(dataType: Char): Any = {
+    dataType match {
+      case '0' => null
+      case 'T' => true
+      case 'F' => false
+      case 'i' => loadInt
+      case 'f' => loadAndRegisterFloat
+      case ':' => loadAndRegisterSymbol
+      case '[' => loadAndRegisterArray
+      case 'U' => loadAndRegisterUserObject
+      case _ =>
+        throw new IllegalArgumentException(s"Format is not supported: $dataType.")
+    }
+  }
+
+
+  // ----------------------------------------------------------------------------------------------
+  // Load by type
+
+  def loadInt: Int = {
+    var c = is.readByte.toInt
+
+    if (c == 0) {
+      return 0
+    } else if (4 < c && c < 128) {
+      return c - 5
+    } else if (-129 < c && c < -4) {
+      return c + 5
+    }
+
+    var result: Long = 0
+
+    if (c > 0) {
+      result = 0
+      for( i <- 0 until c ) {
+        result |= (is.readUnsignedByte << (8 * i)).toLong
+      }
+    } else {
+      c = -c
+      result = -1
+      for( i <- 0 until c ) {
+        result &= ~((0xff << (8 * i)).toLong)
+        result |= (is.readUnsignedByte << (8 * i)).toLong
+      }
+    }
+
+    result.toInt
+  }
+
+  def loadAndRegisterFloat: Double = {
+    val result = loadFloat
+    registeredLinks += result
+    result
+  }
+
+  def loadFloat: Double = {
+    val string = loadString
+    string match {
+      case "nan"  => Double.NaN
+      case "inf"  => Double.PositiveInfinity
+      case "-inf" => Double.NegativeInfinity
+      case _ => string.toDouble
+    }
+  }
+
+  def loadString: String = {
+    new String(loadStringBytes)
+  }
+
+  def loadStringBytes: Array[Byte] = {
+    val size = loadInt
+    val buffer = new Array[Byte](size)
+
+    var readSize = 0
+    while(readSize < size){
+      val read = is.read(buffer, readSize, size-readSize)
+
+      if(read == -1){
+        throw new IllegalArgumentException("Marshal too short.")
+      }
+
+      readSize += read
+    }
+
+    buffer
+  }
+
+  def loadAndRegisterSymbol: String = {
+    val result = loadString
+    registeredSymbols += result
+    result
+  }
+
+  def loadAndRegisterArray: Array[Any] = {
+    val size = loadInt
+    val array = new Array[Any](size)
+
+    registeredLinks += array
+
+    for( i <- 0 until size ) {
+      array(i) = loadNextObject
+    }
+
+    array
+  }
+
+  def loadAndRegisterUserObject: Any = {
+    val klass = loadNextObject.asInstanceOf[String]
+
+    // Register future class before load the next object
+    registeredLinks += WaitForObject()
+    val index = registeredLinks.size - 1
+
+    val data = loadNextObject
+
+    val result = klass match {
+      case "Spark::Mllib::LabeledPoint" => createLabeledPoint(data)
+      case "Spark::Mllib::DenseVector" => createDenseVector(data)
+      case "Spark::Mllib::SparseVector" => createSparseVector(data)
+      case other =>
+        throw new IllegalArgumentException(s"Object $other is not supported.")
+    }
+
+    registeredLinks(index) = result
+
+    result
+  }
+
+
+  // ----------------------------------------------------------------------------------------------
+  // Other loads
+
+  def loadNextObject: Any = {
+    val dataType = is.readUnsignedByte.toChar
+
+    if(isLinkType(dataType)){
+      readLink(dataType)
+    }
+    else{
+      load(dataType)
+    }
+  }
+
+
+  // ----------------------------------------------------------------------------------------------
+  // To java objects
+
+  def createLabeledPoint(data: Any): LabeledPoint = {
+    val array = data.asInstanceOf[Array[_]]
+    new LabeledPoint(array(0).asInstanceOf[Double], array(1).asInstanceOf[Vector])
+  }
+
+  def createDenseVector(data: Any): DenseVector = {
+    new DenseVector(data.asInstanceOf[Array[_]].map(toDouble(_)))
+  }
+
+  def createSparseVector(data: Any): SparseVector = {
+    val array = data.asInstanceOf[Array[_]]
+    val size = array(0).asInstanceOf[Int]
+    val indices = array(1).asInstanceOf[Array[_]].map(_.asInstanceOf[Int])
+    val values = array(2).asInstanceOf[Array[_]].map(toDouble(_))
+
+    new SparseVector(size, indices, values)
+  }
+
+
+  // ----------------------------------------------------------------------------------------------
+  // Helpers
+
+  def toDouble(data: Any): Double = data match {
+    case x: Int => x.toDouble
+    case x: Double => x
+    case _ => 0.0
+  }
+
+
+  // ----------------------------------------------------------------------------------------------
+  // Cache
+
+  def readLink(dataType: Char): Any = {
+    val index = loadInt
+
+    dataType match {
+      case '@' => registeredLinks(index)
+      case ';' => registeredSymbols(index)
+    }
+  }
+
+  def isLinkType(dataType: Char): Boolean = {
+    dataType == ';' || dataType == '@'
+  }
+
+}