ruby-spark 1.1.0.1-java

data/lib/spark/version.rb

@@ -0,0 +1,3 @@
+module Spark
+  VERSION = '1.1.0.1'
+end

data/lib/spark/worker/master.rb

@@ -0,0 +1,144 @@
+#!/usr/bin/env ruby
+
+$PROGRAM_NAME = 'RubySparkMaster'
+
+require 'socket'
+require 'io/wait'
+require 'nio'
+
+require_relative 'worker'
+
+# New process group
+# Otherwise master can be killed from pry console
+Process.setsid
+
+# =================================================================================================
+# Master
+#
+module Master
+
+  def self.create
+    case ARGV[0].to_s.strip
+    when 'thread'
+      Master::Thread.new
+    else
+      Master::Process.new
+    end
+  end
+
+  class Base
+    include Spark::Constant
+
+    def initialize
+      @port = ARGV[1].to_s.strip.to_i
+      @socket = TCPSocket.open('localhost', @port)
+      @worker_arguments = @socket.read_string
+    end
+
+    def run
+      selector = NIO::Selector.new
+      monitor = selector.register(@socket, :r)
+      monitor.value = Proc.new { receive_message }
+      loop {
+        selector.select {|monitor| monitor.value.call}
+      }
+    end
+
+    def receive_message
+      command = @socket.read_int
+
+      case command
+      when CREATE_WORKER
+        create_worker
+      when KILL_WORKER
+        kill_worker
+      when KILL_WORKER_AND_WAIT
+        kill_worker_and_wait
+      end
+    end
+
+    def kill_worker_and_wait
+      if kill_worker
+        @socket.write_int(SUCCESSFULLY_KILLED)
+      else
+        @socket.write_int(UNSUCCESSFUL_KILLING)
+      end
+    end
+  end
+
+  # ===============================================================================================
+  # Worker::Process
+  #
+  class Process < Base
+
+    def create_worker
+      if fork?
+        pid = ::Process.fork do
+          Worker::Process.new(@port).run
+        end
+      else
+        pid = ::Process.spawn("ruby #{@worker_arguments} worker.rb #{@port}")
+      end
+
+      # Detach child from master to avoid zombie process
+      ::Process.detach(pid)
+    end
+
+    def kill_worker
+      worker_id = @socket.read_long
+      ::Process.kill('TERM', worker_id)
+    rescue
+      nil
+    end
+
+    def fork?
+      @can_fork ||= _fork?
+    end
+
+    def _fork?
+      return false if !::Process.respond_to?(:fork)
+
+      pid = ::Process.fork
+      exit unless pid # exit the child immediately
+      true
+    rescue NotImplementedError
+      false
+    end
+
+  end
+
+  # ===============================================================================================
+  # Worker::Thread
+  #
+  class Thread < Base
+
+    def initialize
+      ::Thread.abort_on_exception = true
+
+      # For synchronous access to socket IO
+      $mutex_for_command  = Mutex.new
+      $mutex_for_iterator = Mutex.new
+
+      super
+    end
+
+    def create_worker
+      ::Thread.new do
+        Worker::Thread.new(@port).run
+      end
+    end
+
+    def kill_worker
+      worker_id = @socket.read_long
+
+      thread = ObjectSpace._id2ref(worker_id)
+      thread.kill
+    rescue
+      nil
+    end
+
+  end
+end
+
+# Create proper master by worker_type
+Master.create.run

data/lib/spark/worker/spark_files.rb

@@ -0,0 +1,15 @@
+class SparkFiles
+
+  class << self
+    attr_accessor :root_directory
+  end
+
+  def self.get(file_name)
+    File.join(root_directory, file_name)
+  end
+
+  def self.get_content(file_name)
+    File.read(get(file_name))
+  end
+
+end

data/lib/spark/worker/worker.rb

@@ -0,0 +1,200 @@
+#!/usr/bin/env ruby
+
+# Load root of the gem
+lib = File.expand_path(File.join('..', '..'), File.dirname(__FILE__))
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'ruby-spark.rb'
+require 'socket'
+
+require_relative 'spark_files'
+
+
+# =================================================================================================
+# Worker
+#
+# Iterator is LAZY !!!
+#
+module Worker
+  class Base
+
+    include Spark::Helper::Serialize
+    include Spark::Helper::System
+    include Spark::Constant
+
+    attr_accessor :socket
+
+    def initialize(port)
+      # Open socket to Spark
+      @socket = TCPSocket.open('localhost', port)
+
+      # Send back worker ID
+      socket.write_long(id)
+    end
+
+    def run
+      begin
+        compute
+      rescue => e
+        send_error(e)
+      else
+        successful_finish
+      end
+    end
+
+    private
+
+      def before_start
+        # Should be implemented in sub-classes
+      end
+
+      def before_end
+        # Should be implemented in sub-classes
+      end
+
+      # These methods must be on one method because iterator is Lazy
+      # which mean that exception can be raised at `serializer` or `compute`
+      def compute
+        before_start
+
+        # Load split index
+        @split_index = socket.read_int
+
+        # Load files
+        SparkFiles.root_directory = socket.read_string
+
+        # Load broadcast
+        count = socket.read_int
+        count.times do
+          Spark::Broadcast.register(socket.read_long, socket.read_string)
+        end
+
+        # Load command
+        @command = socket.read_data
+
+        # Load iterator
+        @iterator = @command.deserializer.load_from_io(socket).lazy
+
+        # Compute
+        @iterator = @command.execute(@iterator, @split_index)
+
+        # Result is not iterable
+        @iterator = [@iterator] unless @iterator.respond_to?(:each)
+
+        # Send result
+        @command.serializer.dump_to_io(@iterator, socket)
+      end
+
+      def send_error(e)
+        # Flag
+        socket.write_int(WORKER_ERROR)
+
+        # Message
+        socket.write_string(e.message)
+
+        # Backtrace
+        socket.write_int(e.backtrace.size)
+        e.backtrace.each do |item|
+          socket.write_string(item)
+        end
+
+        socket.flush
+
+        # Wait for spark
+        # Socket is closed before throwing an exception
+        # Singal that ruby exception was fully received
+        until socket.closed?
+          sleep(0.1)
+        end
+
+        # Depend on type of worker
+        kill_worker
+      end
+
+      def successful_finish
+        # Finish
+        socket.write_int(WORKER_DONE)
+
+        # Send changed accumulator
+        changed = Spark::Accumulator.changed
+        socket.write_int(changed.size)
+        changed.each do |accumulator|
+          socket.write_data([accumulator.id, accumulator.value])
+        end
+
+        # Send it
+        socket.flush
+
+        before_end
+      end
+
+      def log(message=nil)
+        return if !$DEBUG
+
+        $stdout.puts %{==> #{Time.now.strftime('%H:%M:%S')} [#{id}] #{message}}
+        $stdout.flush
+      end
+
+  end
+
+  # ===============================================================================================
+  # Worker::Process
+  #
+  class Process < Base
+
+    def id
+      ::Process.pid
+    end
+
+    private
+
+      def before_start
+        $PROGRAM_NAME = 'RubySparkWorker'
+      end
+
+      def kill_worker
+        Process.exit(false)
+      end
+
+  end
+
+  # ===============================================================================================
+  # Worker::Thread
+  #
+  class Thread < Base
+
+    def id
+      ::Thread.current.object_id
+    end
+
+    private
+
+      def load_command
+        $mutex_for_command.synchronize { super }
+      end
+
+      # Threads changing for reading is very slow
+      # Faster way is do it one by one
+      def load_iterator
+        # Wait for incoming connection for preventing deadlock
+        if jruby?
+          socket.io_wait
+        else
+          socket.wait_readable
+        end
+
+        $mutex_for_iterator.synchronize { super }
+      end
+
+      def kill_worker
+        Thread.current.kill
+      end
+
+  end
+end
+
+# Worker is loaded as standalone
+if $PROGRAM_NAME == __FILE__
+  worker = Worker::Process.new(ARGV[0])
+  worker.run
+end

data/ruby-spark.gemspec

@@ -0,0 +1,47 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'spark/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'ruby-spark'
+  spec.version       = Spark::VERSION
+  spec.authors       = ['Ondřej Moravčík']
+  spec.email         = ['moravcik.ondrej@gmail.com']
+  spec.summary       = %q{Ruby wrapper for Apache Spark}
+  spec.description   = %q{}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  if RUBY_PLATFORM =~ /java/
+    spec.platform = 'java'
+
+    extensions = ['ext/ruby_java/extconf.rb']
+  else
+    extensions = ['ext/ruby_c/extconf.rb']
+
+    spec.add_dependency 'rjb'
+  end
+
+  spec.extensions = extensions
+  spec.required_ruby_version = '>= 2.0'
+
+  spec.requirements << 'java, scala'
+
+  spec.add_dependency 'sourcify', '0.6.0.rc4'
+  spec.add_dependency 'method_source'
+  spec.add_dependency 'commander'
+  spec.add_dependency 'pry'
+  spec.add_dependency 'nio4r'
+  spec.add_dependency 'distribution'
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+end

data/spec/generator.rb

@@ -0,0 +1,37 @@
+class Generator
+  def self.numbers(size=1000)
+    Array.new(size){ rand(1..1000) }
+  end
+
+  def self.numbers_with_zero(size=1000)
+    Array.new(size){ rand(0..1000) }
+  end
+
+  def self.words(size=1000)
+    Array.new(size) { word }
+  end
+
+  def self.word(size=10)
+    Array.new(rand(1..size)){(97+rand(26)).chr}.join
+  end
+
+  def self.lines(size=1000, letters=3)
+    Array.new(size) do
+      Array.new(rand(50..100)){
+        (97+rand(letters)).chr + (" " * (rand(10) == 0 ? 1 : 0))
+      }.join
+    end
+  end
+
+  def self.hash(size=1000)
+    Array.new(size) do
+      [word(2), rand(1..10)]
+    end
+  end
+
+  def self.hash_with_values(size=1000, values_count=10)
+    Array.new(size) do
+      [word(2), Array.new(values_count) { rand(1..10) }]
+    end
+  end
+end