ruby-spark 1.0.0

data/ext/spark/src/main/scala/RubyUtils.scala

@@ -0,0 +1,15 @@
+package org.apache.spark.api.ruby
+
+import org.apache.spark.util._
+import org.apache.spark.{SparkConf, Logging}
+
+object RubyUtils extends Logging {
+
+  def loadPropertiesFile(conf: SparkConf, path: String): String = {
+    Utils.getPropertiesFromFile(path).foreach {
+      case (key, value) => conf.set(key, value)
+    }
+    path
+  }
+
+}

data/ext/spark/src/main/scala/RubyWorker.scala

@@ -0,0 +1,257 @@
+package org.apache.spark.api.ruby
+
+import java.io.{File, DataInputStream, InputStream, DataOutputStream, FileOutputStream}
+import java.net.{InetAddress, ServerSocket, Socket, SocketException}
+import java.nio.file.Paths
+
+import scala.collection.mutable
+import scala.collection.JavaConversions._
+
+import org.apache.spark._
+import org.apache.spark.api.python.PythonRDD
+import org.apache.spark.util.Utils
+import org.apache.spark.util.RedirectThread
+
+
+/* =================================================================================================
+ * Object RubyWorker
+ * =================================================================================================
+ *
+ * Create and store server for creating workers.
+ */
+
+object RubyWorker extends Logging {
+
+  val PROCESS_WAIT_TIMEOUT = 10000
+
+  private var serverSocket: ServerSocket = null
+  private val serverHost = InetAddress.getByAddress(Array(127, 0, 0, 1))
+  private var serverPort: Int = 0
+
+  private var master: ExecutedFileCommand = null
+  private var masterSocket: Socket = null
+  private var masterOutputStream: DataOutputStream = null
+  private var masterInputStream: DataInputStream = null
+
+  private var workers = new mutable.WeakHashMap[Socket, Long]()
+
+
+  /* ----------------------------------------------------------------------------------------------
+   * Create new worker but first check if exist SocketServer and master process.
+   * If not it will create them. Worker have 2 chance to create.
+   */
+
+  def create(env: SparkEnv): (Socket, Long) = {
+    synchronized {
+      // Create the server if it hasn't been started
+      createServer(env)
+
+      // Attempt to connect, restart and retry once if it fails
+      try {
+        createWorker
+      } catch {
+        case exc: SocketException =>
+          logWarning("Worker unexpectedly quit, attempting to restart")
+          createWorker
+      }
+    }
+  }
+
+  /* ----------------------------------------------------------------------------------------------
+   * Create a worker throught master process. Return new socket and id.
+   * According spark.ruby.worker.type id will be:
+   *   process: PID
+   *   thread: thread object id
+   */
+
+  def createWorker: (Socket, Long) = {
+    synchronized {
+      masterOutputStream.writeInt(RubyConstant.CREATE_WORKER)
+      var socket = serverSocket.accept()
+
+      var id = new DataInputStream(socket.getInputStream).readLong()
+      workers.put(socket, id)
+
+      (socket, id)
+    }
+  }
+
+  /* ----------------------------------------------------------------------------------------------
+   * Create SocketServer and bind it to the localhost. Max numbers of connection on queue
+   * is set to default. If server is created withou exception -> create master.
+   */
+
+  private def createServer(env: SparkEnv){
+    synchronized {
+      // Already running?
+      if(serverSocket != null && masterSocket != null) {
+        return
+      }
+
+      try {
+        // Start Socket Server for comunication
+        serverSocket = new ServerSocket(0, 0, serverHost)
+        serverPort = serverSocket.getLocalPort
+
+        // Create a master for worker creations
+        createMaster(env)
+      } catch {
+        case e: Exception =>
+          throw new SparkException("There was a problem with creating a server", e)
+      }
+    }
+  }
+
+  /* ----------------------------------------------------------------------------------------------
+   * In this point SocketServer must be created. Master process create and kill workers.
+   * Creating workers from Java can be an expensive operation because new process can
+   * get copy of address space.
+   */
+
+  private def createMaster(env: SparkEnv){
+    synchronized {
+      val isDriver = env.executorId == SparkContext.DRIVER_IDENTIFIER
+      val executorOptions = env.conf.get("spark.ruby.executor.options", "")
+      val commandTemplate = env.conf.get("spark.ruby.executor.command")
+      val workerType = env.conf.get("spark.ruby.worker.type")
+
+      // Where is root of ruby-spark
+      var executorLocation = ""
+
+      if(isDriver){
+        // Use worker from current active gem location
+        executorLocation = env.conf.get("spark.ruby.driver_home")
+      }
+      else{
+        // Ruby-spark package uri
+        val uri = env.conf.get("spark.ruby.executor.uri", "")
+
+        if(uri.isEmpty){
+          // Use gem installed on the system
+          try {
+            val homeCommand = new FileCommand(commandTemplate, "ruby-spark home", env, getEnvVars(env))
+
+            executorLocation = homeCommand.run.readLine
+          } catch {
+            case e: java.io.IOException =>
+              throw new SparkException("Ruby-spark gem is not installed.", e)
+          }
+        }
+        else{
+          // Prepare and use gem from uri
+        }
+      }
+
+      // Master and worker are saved in GEM_ROOT/lib/spark/worker
+      executorLocation = Paths.get(executorLocation, "lib", "spark", "worker").toString
+
+      // Create master command
+      // -C: change worker dir before execution
+      val masterRb = s"ruby $executorOptions -C $executorLocation master.rb $workerType $serverPort"
+      val masterCommand = new FileCommand(commandTemplate, masterRb, env, getEnvVars(env))
+
+      // Start master
+      master = masterCommand.run
+
+      // Redirect master stdout and stderr
+      redirectStreamsToStderr(master.getInputStream, master.getErrorStream)
+
+      // Wait for it to connect to our socket
+      serverSocket.setSoTimeout(PROCESS_WAIT_TIMEOUT)
+      try {
+        // Use socket for comunication. Keep stdout and stdin for log
+        masterSocket = serverSocket.accept()
+        masterOutputStream = new DataOutputStream(masterSocket.getOutputStream)
+        masterInputStream  = new DataInputStream(masterSocket.getInputStream)
+
+        PythonRDD.writeUTF(executorOptions, masterOutputStream)
+      } catch {
+        case e: Exception =>
+          throw new SparkException("Ruby master did not connect back in time", e)
+      }
+    }
+  }
+
+  /* ----------------------------------------------------------------------------------------------
+   * Gel all environment variables for executor
+   */
+
+  def getEnvVars(env: SparkEnv): Map[String, String] = {
+    val prefix = "spark.ruby.executor.env."
+    env.conf.getAll.filter{case (k, _) => k.startsWith(prefix)}
+                   .map{case (k, v) => (k.substring(prefix.length), v)}
+                   .toMap
+  }
+
+  /* ------------------------------------------------------------------------------------------- */
+
+  def kill(workerId: Long){
+    masterOutputStream.writeInt(RubyConstant.KILL_WORKER)
+    masterOutputStream.writeLong(workerId)
+  }
+
+  /* ------------------------------------------------------------------------------------------- */
+
+  def killAndWait(workerId: Long){
+    masterOutputStream.writeInt(RubyConstant.KILL_WORKER_AND_WAIT)
+    masterOutputStream.writeLong(workerId)
+
+    // Wait for answer
+    masterInputStream.readInt() match {
+      case RubyConstant.SUCCESSFULLY_KILLED =>
+        logInfo(s"Worker $workerId was successfully killed")
+      case RubyConstant.UNSUCCESSFUL_KILLING =>
+        logInfo(s"Worker $workerId cannot be killed (maybe is already killed)")
+    }
+  }
+
+  /* ----------------------------------------------------------------------------------------------
+   * workers HashMap is week but it avoid long list of workers which cannot be killed (killAndWait)
+   */
+
+  def remove(worker: Socket, workerId: Long){
+    try {
+      workers.remove(worker)
+    } catch {
+      case e: Exception => logWarning(s"Worker $workerId does not exist (maybe is already removed)")
+    }
+  }
+
+  /* ------------------------------------------------------------------------------------------- */
+
+  def stopServer{
+    synchronized {
+      // Kill workers
+      workers.foreach { case (socket, id) => killAndWait(id) }
+
+      // Kill master
+      master.destroy
+
+      // Stop SocketServer
+      serverSocket.close()
+
+      // Clean variables
+      serverSocket = null
+      serverPort = 0
+      master = null
+      masterSocket = null
+      masterOutputStream = null
+      masterInputStream = null
+    }
+  }
+
+  /* ------------------------------------------------------------------------------------------- */
+
+  private def redirectStreamsToStderr(streams: InputStream*) {
+    try {
+      for(stream <- streams) {
+        new RedirectThread(stream, System.err, "stream reader").start()
+      }
+    } catch {
+      case e: Exception =>
+        logError("Exception in redirecting streams", e)
+    }
+  }
+
+  /* ------------------------------------------------------------------------------------------- */
+}

data/ext/spark/src/test/scala/MarshalSpec.scala

@@ -0,0 +1,84 @@
+package org.apache.spark.api.ruby.marshal
+
+import org.scalatest._
+
+
+import org.apache.spark.api.ruby.marshal._
+
+class MarshalSpec extends FunSpec with Matchers {
+
+  // ====================================================================================
+  // Load
+
+  describe("Marshal.load"){
+    describe("single value"){
+      it("int"){
+        val data = 1
+        val serialized = Array[Byte](4, 8, 105, 6)
+
+        Marshal.load(serialized) should equal(data)
+      }
+
+      it("double"){
+        val data = 1.2
+        val serialized = Array[Byte](4, 8, 102, 8, 49, 46, 50)
+
+        Marshal.load(serialized) should equal(data)
+      }
+    }
+
+    describe("array"){
+      it("ints"){
+        val data = Array(1, 2, 3, 4, 5)
+        val serialized = Array[Byte](4, 8, 91, 10, 105, 6, 105, 7, 105, 8, 105, 9, 105, 10)
+
+        Marshal.load(serialized) should equal(data)
+      }
+
+      it("doubles"){
+        val data = Array(1.1, 2.2, 3.3)
+        val serialized = Array[Byte](4, 8, 91, 8, 102, 8, 49, 46, 49, 102, 8, 50, 46, 50, 102, 8, 51, 46, 51)
+
+        Marshal.load(serialized) should equal(data)
+      }
+    }
+  }
+
+  // ====================================================================================
+  // Dump
+
+  describe("Marshal.dump"){
+    describe("single value"){
+      it("int"){
+        val data = 1
+        val serialized = Array(4, 8, 105, 6)
+
+        Marshal.dump(data) should equal(serialized)
+      }
+
+      it("double"){
+        val data = 1.2
+        val serialized = Array(4, 8, 102, 8, 49, 46, 50)
+
+        Marshal.dump(data) should equal(serialized)
+      }
+    }
+
+    describe("array"){
+      it("ints"){
+        val data = Array(1, 2, 3, 4, 5)
+        val serialized = Array(4, 8, 91, 10, 105, 6, 105, 7, 105, 8, 105, 9, 105, 10)
+
+        Marshal.dump(data) should equal(serialized)
+      }
+
+      it("doubles"){
+        val data = Array(1.1, 2.2, 3.3)
+        val serialized = Array(4, 8, 91, 8, 102, 8, 49, 46, 49, 102, 8, 50, 46, 50, 102, 8, 51, 46, 51)
+
+        Marshal.dump(data) should equal(serialized)
+      }
+    }
+  }
+
+}

data/lib/ruby-spark.rb

@@ -0,0 +1 @@
+require_relative 'spark'

data/lib/spark.rb

@@ -0,0 +1,198 @@
+# Gems and libraries
+require 'method_source'
+require 'forwardable'
+require 'sourcify'
+require 'socket'
+require 'tempfile'
+require 'tmpdir'
+
+module Spark
+  autoload :Context,        'spark/context'
+  autoload :Config,         'spark/config'
+  autoload :RDD,            'spark/rdd'
+  autoload :CLI,            'spark/cli'
+  autoload :Build,          'spark/build'
+  autoload :Serializer,     'spark/serializer'
+  autoload :Helper,         'spark/helper'
+  autoload :StorageLevel,   'spark/storage_level'
+  autoload :Command,        'spark/command'
+  autoload :CommandBuilder, 'spark/command_builder'
+  autoload :Sampler,        'spark/sampler'
+  autoload :Logger,         'spark/logger'
+  autoload :JavaBridge,     'spark/java_bridge'
+  autoload :ExternalSorter, 'spark/sort'
+  autoload :Constant,       'spark/constant'
+  autoload :Broadcast,      'spark/broadcast'
+  autoload :Accumulator,    'spark/accumulator'
+  autoload :StatCounter,    'spark/stat_counter'
+  autoload :Mllib,          'spark/mllib'
+
+  include Helper::System
+
+  def self.print_logo(message=nil)
+    puts <<-STRING
+
+    Welcome to
+                  __           ____              __
+        ______ __/ /  __ __   / __/__  ___ _____/ /__
+       / __/ // / _ \\/ // /  _\\ \\/ _ \\/ _ `/ __/  '_/
+      /_/  \\_,_/_.__/\\_, /  /___/ .__/\\_,_/_/ /_/\\_\\   version #{Spark::VERSION}
+                    /___/      /_/
+
+    #{message}
+
+    STRING
+  end
+
+  # Returns current configuration. Configurations can be changed until
+  # context is initialized. In this case config is locked only for reading.
+  #
+  # == Configuration can be changed:
+  #
+  #   Spark.config.set('spark.app.name', 'RubySpark')
+  #
+  #   Spark.config['spark.app.name'] = 'RubySpark'
+  #
+  #   Spark.config do
+  #     set 'spark.app.name', 'RubySpark'
+  #   end
+  #
+  def self.config(&block)
+    @config ||= Spark::Config.new
+
+    if block_given?
+      @config.instance_eval(&block)
+    else
+      @config
+    end
+  end
+
+  # Destroy current configuration. This can be useful for restarting config
+  # to set new. It has no effect if context is already started.
+  def self.clear_config
+    @config = nil
+  end
+
+  # Return a current active context or nil.
+  #
+  # TODO: Run `start` if context is nil?
+  #
+  def self.context
+    @context
+  end
+
+  # Initialize spark context if not already. Config will be automatically
+  # loaded on constructor. From that point `config` will use configuration
+  # from running Spark and will be locked only for reading.
+  def self.start
+    if started?
+      # Already started
+    else
+      @context ||= Spark::Context.new
+    end
+  end
+
+  def self.stop
+    @context.stop
+    RubyWorker.stopServer
+    logger.info('Workers were stopped')
+  rescue
+    nil
+  ensure
+    @context = nil
+    clear_config
+  end
+
+  def self.started?
+    !!@context
+  end
+
+  def self.logger
+    @logger ||= Spark::Logger.new
+  end
+
+  # Root of the gem
+  def self.root
+    @root ||= File.expand_path('..', File.dirname(__FILE__))
+  end
+
+  def self.home
+    root
+  end
+
+  # Default directory for java extensions
+  def self.target_dir
+    @target_dir ||= File.join(root, 'target')
+  end
+
+  # Directory where is worker.rb
+  def self.worker_dir
+    @worker_dir ||= File.join(root, 'lib', 'spark', 'worker')
+  end
+
+  def self.ruby_spark_jar
+    @ruby_spark_jar ||= File.join(target_dir, 'ruby-spark.jar')
+  end
+
+  def self.spark_ext_dir
+    @spark_ext_dir ||= File.join(root, 'ext', 'spark')
+  end
+
+
+  # ===============================================================================
+  # Load JVM and jars
+
+  # Load dependent libraries, can be use once
+  # Cannot load before CLI::install
+  #
+  # == Parameters:
+  # spark_home::
+  #   path to directory where are located sparks .jar files or single Spark jar
+  #
+  def self.load_lib(spark_home=nil)
+    return if @java_bridge
+
+    spark_home ||= Spark.target_dir
+
+    bridge = JavaBridge.get
+    @java_bridge = bridge.new(spark_home)
+    @java_bridge.load
+    nil
+  end
+
+  def self.java_bridge
+    @java_bridge
+  end
+
+
+  # Aliases
+  class << self
+    alias_method :sc, :context
+    alias_method :jb, :java_bridge
+  end
+
+end
+
+# C/Java extensions
+require 'ruby_spark_ext'
+
+# Ruby core extensions
+require 'spark/ext/module'
+require 'spark/ext/object'
+require 'spark/ext/hash'
+require 'spark/ext/string'
+require 'spark/ext/integer'
+require 'spark/ext/ip_socket'
+require 'spark/ext/io'
+
+# Other requirments
+require 'spark/version'
+require 'spark/error'
+
+# Make sure that Spark be always stopped
+Kernel::at_exit do
+  begin
+    Spark.stop
+  rescue
+  end
+end