ruby-spark 1.0.0

data/lib/spark/accumulator.rb

@@ -0,0 +1,260 @@
+module Spark
+  ##
+  # A shared variable that can be accumulated, i.e., has a commutative and associative "add"
+  # operation. Worker tasks on a Spark cluster can add values to an Accumulator with the `+=`
+  # operator, but only the driver program is allowed to access its value, using value.
+  # Updates from the workers get propagated automatically to the driver program.
+  #
+  # == Arguments:
+  # value::
+  #   Initial value for accumulator. This values is stored only on driver process
+  #
+  # accum_param::
+  #   How merge 2 value on worker or driver process.
+  #   Symbol or Proc (or String)
+  #
+  # zero_value::
+  #   Initial value for worker process
+  #
+  #
+  # == Examples:
+  #
+  #   accum1 = $sc.accumulator(1)
+  #   accum2 = $sc.accumulator(2, :*, 1)
+  #   accum3 = $sc.accumulator(3, lambda{|max, val| val > max ? val : max})
+  #
+  #   accum1 += 1
+  #
+  #   accum2.add(2)
+  #   accum2.add(2)
+  #   accum2.add(2)
+  #
+  #   accum3.add(9)
+  #   accum3.add(6)
+  #   accum3.add(7)
+  #
+  #   accum1.value # => 2
+  #   accum2.value # => 16
+  #   accum3.value # => 9
+  #
+  #   func = Proc.new do |_, index|
+  #     accum1.add(1)
+  #     accum2.add(2)
+  #     accum3.add(index * 10)
+  #   end
+  #
+  #   rdd = $sc.parallelize(0..4, 4)
+  #   rdd = rdd.bind(accum1: accum1, accum2: accum2, accum3: accum3)
+  #   rdd = rdd.map_partitions_with_index(func)
+  #   rdd.collect
+  #
+  #   accum1.value # => 6
+  #   accum2.value # => 256
+  #   accum3.value # => 30
+  #
+  class Accumulator
+
+    attr_reader :id, :value, :accum_param, :zero_value
+
+    @@instances = {}
+    @@changed = []
+
+    SUPPORTED_SYMBOLS = [:+, :-, :*, :/, :**]
+
+
+    # =========================================================================
+    # Creating and selecting Spark::Accumulator
+
+    def initialize(value, accum_param=:+, zero_value=0)
+      @id = object_id
+      @value = value
+      @accum_param = accum_param
+      @zero_value = zero_value
+      @driver = true
+
+      valid_accum_param
+
+      @@instances[@id] = self
+    end
+
+    def self.changed
+      @@changed
+    end
+
+    def self.instances
+      @@instances
+    end
+
+    def valid_accum_param
+      if @accum_param.is_a?(Symbol)
+        raise Spark::AccumulatorError, "Unsupported symbol #{@accum_param}" unless SUPPORTED_SYMBOLS.include?(@accum_param)
+        @serialized_accum_param = @accum_param
+        return
+      end
+
+      if @accum_param.is_a?(Proc)
+        begin
+          @serialized_accum_param = @accum_param.to_source
+          return
+        rescue
+          raise Spark::SerializeError, 'Proc can not be serialized. Use String instead.'
+        end
+      end
+
+      if @accum_param.is_a?(String)
+        @serialized_accum_param = @accum_param
+        @accum_param = eval(@accum_param)
+
+        unless @accum_param.is_a?(Proc)
+          raise Spark::SerializeError, 'Yours param is not a Proc.'
+        end
+
+        return
+      end
+
+      raise Spark::AccumulatorError, 'Unsupported param. Use Symbol, Proc or String.'
+    end
+
+    # Driver process or worker
+    def driver?
+      @driver
+    end
+
+
+    # =========================================================================
+    # Operations
+
+    def add(term)
+      if !driver? && !@@changed.include?(self)
+        @@changed << self
+      end
+
+      if @accum_param.is_a?(Proc)
+        @value = @accum_param.call(@value, term)
+      else
+        add_by_symbol(term)
+      end
+    end
+
+    def +(term)
+      add(term)
+      self
+    end
+
+    def add_by_symbol(term)
+      case @accum_param
+      when :+
+        @value += term
+      when :-
+        @value -= term
+      when :*
+        @value *= term
+      when :/
+        @value /= term
+      when :**
+        @value **= term
+      end
+    end
+
+
+    # =========================================================================
+    # Dump and load
+
+    def marshal_dump
+      [@id, @zero_value, @serialized_accum_param]
+    end
+
+    def marshal_load(array)
+      @id, @zero_value, @serialized_accum_param = array
+
+      @value = @zero_value
+      @driver = false
+      load_accum_param
+    end
+
+    def load_accum_param
+      if @serialized_accum_param.is_a?(String)
+        @accum_param = eval(@serialized_accum_param)
+      else
+        @accum_param = @serialized_accum_param
+      end
+    end
+
+  end
+end
+
+# =============================================================================
+# Server for handeling Accumulator update
+#
+module Spark
+  class Accumulator
+    class Server
+
+      attr_reader :server, :host, :port
+
+      def self.start
+        @instance ||= Spark::Accumulator::Server.new
+      end
+
+      def self.stop
+        @instance && @instance.stop
+      end
+
+      def self.host
+        start
+        @instance.host
+      end
+
+      def self.port
+        start
+        @instance.port
+      end
+
+      def initialize
+        @server = TCPServer.new(0)
+        @host = @server.hostname
+        @port = @server.port
+
+        @threads = []
+        handle_accept
+      end
+
+      def stop
+        @threads.each(&:kill)
+      rescue
+        nil
+      end
+
+      def handle_accept
+        @threads << Thread.new do
+          loop {
+            handle_connection(@server.accept)
+          }
+        end
+
+      end
+
+      def handle_connection(socket)
+        @threads << Thread.new do
+          until socket.closed?
+            count = socket.read_int
+            count.times do
+              data = socket.read_data
+              accum = Spark::Accumulator.instances[data[0]]
+              if accum
+                accum.add(data[1])
+              else
+                Spark.logger.warn("Accumulator with id #{data[0]} does not exist.")
+              end
+            end
+
+            # http://stackoverflow.com/questions/28560133/ruby-server-java-scala-client-deadlock
+            # socket.write_int(Spark::Constant::ACCUMULATOR_ACK)
+          end
+
+        end
+      end
+
+    end
+  end
+end

data/lib/spark/broadcast.rb

@@ -0,0 +1,98 @@
+module Spark
+  ##
+  # Broadcast a read-only variable to the cluster, returning a Spark::Broadcast
+  # object for reading it in distributed functions. The variable will
+  # be sent to each cluster only once.
+  #
+  # == Example:
+  #
+  #   broadcast1 = $sc.broadcast('a')
+  #   broadcast2 = $sc.broadcast('b')
+  #   broadcast3 = $sc.broadcast([1,2,3])
+  #
+  #   func = Proc.new do |part, index|
+  #     [
+  #       broadcast1.value * index,
+  #       broadcast2.value * index,
+  #       broadcast3.value.reduce(:+)
+  #     ]
+  #   end
+  #
+  #   rdd = $sc.parallelize(0..5, 4)
+  #   rdd = rdd.bind(broadcast1: broadcast1, broadcast2: broadcast2, broadcast3: broadcast3)
+  #   rdd = rdd.map_partitions_with_index(func)
+  #   rdd.collect
+  #   # => ["", "", 6, "a", "b", 6, "aa", "bb", 6, "aaa", "bbb", 6]
+  #
+  class Broadcast
+
+    LOADED       = 0 # id, value, path
+    NOT_LOADED   = 1 # id, path
+    WITHOUT_PATH = 2 # id
+
+    attr_reader :id, :state, :path, :jbroadcast
+
+    @@registered = {}
+
+    # =========================================================================
+    # Creating broadcast for SparkContext
+
+    # Create new Broadcast and dump value to the disk
+    #
+    #   b = $sc.broadcast('a')
+    #
+    #   b.value # => 'a'
+    #   b.path
+    #   b.jbroadcast
+    #
+    def initialize(sc, value)
+      @id = object_id
+      @value = value
+      @state = LOADED
+
+      file = Tempfile.create('broadcast', sc.temp_dir)
+      file.binmode
+      file.write(Marshal.dump(value))
+      file.close
+
+      @path = file.path
+      @jbroadcast = RubyRDD.readBroadcastFromFile(sc.jcontext, @path, Spark.jb.to_long(@id))
+
+      ObjectSpace.define_finalizer(self, proc { File.unlink(@path) })
+    end
+
+    def self.register(id, path)
+      @@registered[id] = path
+    end
+
+    def value
+      case state
+      when LOADED
+        @value
+      when NOT_LOADED
+        @value = Marshal.load(File.read(@path))
+        @state = LOADED
+        @value
+      when WITHOUT_PATH
+        @path = @@registered[id]
+
+        if @path
+          @state = NOT_LOADED
+          value
+        else
+          raise Spark::BroadcastError, "Broadcast #{@id} do not have registered path."
+        end
+      end
+    end
+
+    def marshal_dump
+      @id
+    end
+
+    def marshal_load(id)
+      @id = id
+      @state = WITHOUT_PATH
+    end
+
+  end
+end

data/lib/spark/build.rb

@@ -0,0 +1,43 @@
+module Spark
+  module Build
+
+    DEFAULT_SCALA_VERSION  = '2.10.4'
+    DEFAULT_CORE_VERSION   = '2.10'
+    DEFAULT_SPARK_VERSION  = '1.3.0'
+    DEFAULT_HADOOP_VERSION = '1.0.4'
+
+    SBT       = 'sbt/sbt'
+    SBT_DEPS  = 'assemblyPackageDependency'
+    SBT_EXT   = 'package'
+    SBT_CLEAN = 'clean'
+
+    def self.build(options)
+      spark_home     = options.spark_home     || Spark.target_dir
+      scala_version  = options.scala_version  || DEFAULT_SCALA_VERSION
+      spark_core     = options.spark_core     || DEFAULT_CORE_VERSION
+      spark_version  = options.spark_version  || DEFAULT_SPARK_VERSION
+      hadoop_version = options.hadoop_version || DEFAULT_HADOOP_VERSION
+      only_ext       = options.only_ext
+
+      env = {
+        'SCALA_VERSION' => scala_version,
+        'SPARK_VERSION' => spark_version,
+        'SPARK_CORE_VERSION' => spark_core,
+        'HADOOP_VERSION' => hadoop_version,
+        'SPARK_HOME' => spark_home
+      }
+
+      cmd = [SBT]
+      cmd << SBT_EXT
+      cmd << SBT_DEPS unless only_ext
+      cmd << SBT_CLEAN unless $debug
+
+      Dir.chdir(Spark.spark_ext_dir) do
+        unless Kernel.system(env, cmd.join(' '))
+          raise Spark::BuildError, 'Spark cannot be assembled.'
+        end
+      end
+    end
+
+  end
+end

data/lib/spark/cli.rb

@@ -0,0 +1,169 @@
+require 'commander'
+
+module Commander
+  module UI
+    # Disable paging
+    # for 'classic' help
+    def self.enable_paging
+    end
+  end
+end
+
+module Spark
+  class CLI
+    include Commander::Methods
+
+    IRB_HISTORY_FILE = File.join(Dir.home, '.irb_spark_history')
+    IRB_HISTORY_SIZE = 100
+
+    def run
+      program :name, 'RubySpark'
+      program :version, Spark::VERSION
+      program :description, 'Ruby wrapper for Spark'
+
+      global_option('-d', '--debug', 'Logging message to stdout'){ $debug = true }
+      default_command :help
+
+
+      # Build ---------------------------------------------------------------
+      command :build do |c|
+        c.syntax = 'build [options]'
+        c.description = 'Build spark and gem extensions'
+        c.option '--hadoop-version STRING', String, 'Version of hadoop which will stored with the SPARK'
+        c.option '--spark-home STRING', String, 'Directory where SPARK will be stored'
+        c.option '--spark-core STRING', String, 'Version of SPARK core'
+        c.option '--spark-version STRING', String, 'Version of SPARK'
+        c.option '--scala-version STRING', String, 'Version of Scala'
+        c.option '--only-ext', 'Start SPARK immediately'
+
+        c.action do |args, options|
+          options.default hadoop_version: Spark::Build::DEFAULT_HADOOP_VERSION,
+                          spark_home: Spark.target_dir,
+                          spark_core: Spark::Build::DEFAULT_CORE_VERSION,
+                          spark_version: Spark::Build::DEFAULT_SPARK_VERSION,
+                          scala_version: Spark::Build::DEFAULT_SCALA_VERSION,
+                          only_ext: false
+
+          Spark::Build.build(options)
+          puts
+          puts 'Everything is OK'
+        end
+      end
+      alias_command :install, :build
+
+
+      # Pry -------------------------------------------------------------------
+      command :pry do |c|
+        c.syntax = 'pry [options]'
+        c.description = 'Start ruby shell for spark'
+        c.option '--spark-home STRING', String, 'Directory where SPARK is stored'
+        c.option '--properties-file STRING', String, 'Path to a file from which to load extra properties'
+        c.option '--[no-]start', 'Start SPARK immediately'
+        c.option '--[no-]logger', 'Enable/disable logger (default: enable)'
+
+        c.action do |args, options|
+          options.default start: true, logger: true
+
+          Spark.load_lib(options.spark_home)
+          Spark::Logger.disable unless options.logger
+
+          Spark.config do
+            set_app_name 'Pry RubySpark'
+          end
+
+          Spark.config.from_file(options.properties_file)
+
+          if options.start
+            # Load Java and Spark
+            Spark.start
+            $sc = Spark.context
+
+            Spark.print_logo('Spark context is loaded as $sc')
+          else
+            Spark.print_logo('You can start Spark with Spark.start')
+          end
+
+          # Load Pry
+          require 'pry'
+          Pry.start
+        end
+      end
+      alias_command :shell, :pry
+
+
+      # IRB -------------------------------------------------------------------
+      command :irb do |c|
+        c.syntax = 'irb [options]'
+        c.description = 'Start ruby shell for spark'
+        c.option '--spark-home STRING', String, 'Directory where SPARK is stored'
+        c.option '--[no-]start', 'Start SPARK immediately'
+        c.option '--[no-]logger', 'Enable/disable logger (default: enable)'
+
+        c.action do |args, options|
+          options.default start: true, logger: true
+
+          Spark.load_lib(options.spark_home)
+          Spark::Logger.disable unless options.logger
+
+          Spark.config do
+            set_app_name 'Pry RubySpark'
+          end
+
+          if options.start
+            # Load Java and Spark
+            Spark.start
+            $sc = Spark.context
+
+            Spark.print_logo('Spark context is loaded as $sc')
+          else
+            Spark.print_logo('You can start Spark with Spark.start')
+          end
+
+          # Load IRB
+          require 'irb'
+          require 'irb/completion'
+          require 'irb/ext/save-history'
+
+          begin
+            file = File.expand_path(IRB_HISTORY_FILE)
+            if File.exists?(file)
+              lines = IO.readlines(file).collect { |line| line.chomp }
+              Readline::HISTORY.push(*lines)
+            end
+            Kernel.at_exit do
+              lines = Readline::HISTORY.to_a.reverse.uniq.reverse
+              lines = lines[-IRB_HISTORY_SIZE, IRB_HISTORY_SIZE] if lines.nitems > IRB_HISTORY_SIZE
+              File.open(IRB_HISTORY_FILE, File::WRONLY | File::CREAT | File::TRUNC) { |io| io.puts lines.join("\n") }
+            end
+          rescue
+          end
+
+          ARGV.clear # Clear Thor ARGV, otherwise IRB will parse it
+          ARGV.concat ['--readline', '--prompt-mode', 'simple']
+          IRB.start
+        end
+      end
+
+
+      # Home ------------------------------------------------------------------
+      command :home do |c|
+        c.action do |args, options|
+          puts Spark.home
+          exit(0)
+        end
+      end
+
+
+      # Ruby spark jar --------------------------------------------------------
+      command :ruby_spark_jar do |c|
+        c.action do |args, options|
+          puts Spark.ruby_spark_jar
+          exit(0)
+        end
+      end
+
+      run!
+    end
+
+  end
+end