ruby-spark 1.0.0

data/lib/spark/sort.rb

@@ -0,0 +1,189 @@
+module Spark
+  module InternalSorter
+    class Base
+      def initialize(key_function)
+        @key_function = key_function
+      end
+    end
+
+    class Ascending < Base
+      def sort(data)
+        data.sort_by!(&@key_function)
+      end
+    end
+
+    class Descending < Ascending
+      def sort(data)
+        super
+        data.reverse!
+      end
+    end
+
+    def self.get(ascending, key_function)
+      if ascending
+        type = Ascending
+      else
+        type = Descending
+      end
+
+      type.new(key_function)
+    end
+  end
+end
+
+
+module Spark
+  class ExternalSorter
+
+    include Spark::Helper::System
+
+    # Items from GC cannot be destroyed so #make_parts need some reserve
+    MEMORY_RESERVE = 50 # %
+
+    # How big will be chunk for adding new memory because GC not cleaning
+    # immediately un-referenced variables
+    MEMORY_FREE_CHUNK = 10 # %
+
+    # How many items will be evaluate from iterator at start
+    START_SLICE_SIZE = 10
+
+    # Maximum of slicing. Memory control can be avoided by large value.
+    MAX_SLICE_SIZE = 10_000
+
+    # How many values will be taken from each enumerator.
+    EVAL_N_VALUES = 10
+
+    # Default key function
+    KEY_FUNCTION = lambda{|item| item}
+
+    attr_reader :total_memory, :memory_limit, :memory_chunk, :serializer
+
+    def initialize(total_memory, serializer)
+      @total_memory = total_memory
+      @memory_limit = total_memory * (100-MEMORY_RESERVE)    / 100
+      @memory_chunk = total_memory * (100-MEMORY_FREE_CHUNK) / 100
+      @serializer   = serializer
+    end
+
+    def add_memory!
+      @memory_limit += memory_chunk
+    end
+
+    def sort_by(iterator, ascending=true, key_function=KEY_FUNCTION)
+      return to_enum(__callee__, iterator, key_function) unless block_given?
+
+      create_temp_folder
+      internal_sorter = Spark::InternalSorter.get(ascending, key_function)
+
+      # Make N sorted enumerators
+      parts = make_parts(iterator, internal_sorter)
+
+      return [] if parts.empty?
+
+      # Need new key function because items have new structure
+      # From: [1,2,3] to [[1, Enumerator],[2, Enumerator],[3, Enumerator]]
+      key_function_with_enum = lambda{|(key, _)| key_function[key]}
+      internal_sorter = Spark::InternalSorter.get(ascending, key_function_with_enum)
+
+      heap  = []
+      enums = []
+
+      # Load first items to heap
+      parts.each do |part|
+        EVAL_N_VALUES.times {
+          begin
+            heap << [part.next, part]
+          rescue StopIteration
+            break
+          end
+        }
+      end
+
+      # Parts can be empty but heap not
+      while parts.any? || heap.any?
+        internal_sorter.sort(heap)
+
+        # Since parts are sorted and heap contains EVAL_N_VALUES method
+        # can add EVAL_N_VALUES items to the result
+        EVAL_N_VALUES.times {
+          break if heap.empty?
+
+          item, enum = heap.shift
+          enums << enum
+
+          yield item
+        }
+
+        # Add new element to heap from part of which was result item
+        while (enum = enums.shift)
+          begin
+            heap << [enum.next, enum]
+          rescue StopIteration
+            parts.delete(enum)
+            enums.delete(enum)
+          end
+        end
+      end
+
+    ensure
+      destroy_temp_folder
+    end
+
+    private
+
+      def create_temp_folder
+        @dir = Dir.mktmpdir
+      end
+
+      def destroy_temp_folder
+        FileUtils.remove_entry_secure(@dir) if @dir
+      end
+
+      # New part is created when current part exceeds memory limit (is variable)
+      # Every new part have more memory because of ruby GC
+      def make_parts(iterator, internal_sorter)
+        slice = START_SLICE_SIZE
+
+        parts = []
+        part  = []
+
+        loop do
+          begin
+            # Enumerator does not have slice method
+            slice.times { part << iterator.next }
+          rescue StopIteration
+            break
+          end
+
+          # Carefully memory_limit is variable
+          if memory_usage > memory_limit
+            # Sort current part with origin key_function
+            internal_sorter.sort(part)
+            # Tempfile for current part
+            # will be destroyed on #destroy_temp_folder
+            file = Tempfile.new("part", @dir)
+            serializer.dump(part, file)
+            # Peek is at the end of file
+            file.seek(0)
+            parts << serializer.load(file)
+
+            # Some memory will be released but not immediately
+            # need some new memory for start
+            part.clear
+            add_memory!
+          else
+            slice = [slice*2, MAX_SLICE_SIZE].min
+          end
+        end
+
+        # Last part which is not in the file
+        if part.any?
+          internal_sorter.sort(part)
+          parts << part.each
+        end
+
+        parts
+      end
+
+  end # ExternalSorter
+end # Spark

data/lib/spark/stat_counter.rb

@@ -0,0 +1,125 @@
+module Spark
+  class StatCounter
+
+    attr_reader :n   # count of our values
+    attr_reader :mu  # mean of our values
+    attr_reader :m2  # variance numerator (sum of (x - mean)^2)
+    attr_reader :max # max of our values
+    attr_reader :min # min of our values
+
+    def initialize(iterator)
+      @n = 0
+      @mu = 0.0
+      @m2 = 0.0
+      @max = -Float::INFINITY
+      @min = Float::INFINITY
+
+      merge(iterator)
+    end
+
+    def merge(other)
+      if other.is_a?(Spark::StatCounter)
+        merge_stat_counter(other)
+      elsif other.respond_to?(:each)
+        merge_array(other)
+      else
+        merge_value(other)
+      end
+
+      self
+    end
+
+    def sum
+      @n * @mu
+    end
+
+    # Return the variance of the values.
+    def variance
+      if @n == 0
+        Float::NAN
+      else
+        @m2 / @n
+      end
+    end
+
+    # Return the sample variance, which corrects for bias in estimating the variance by dividing
+    # by N-1 instead of N.
+    def sample_variance
+      if @n <= 1
+        Float::NAN
+      else
+        @m2 / (@n - 1)
+      end
+    end
+
+    # Return the standard deviation of the values.
+    def stdev
+      Math.sqrt(variance)
+    end
+
+    # Return the sample standard deviation of the values, which corrects for bias in estimating the
+    # variance by dividing by N-1 instead of N.
+    def sample_stdev
+      Math.sqrt(sample_variance)
+    end
+
+    def to_s
+      "(count: #{count}, mean: #{mean}, stdev: #{stdev}, max: #{max}, min: #{min})"
+    end
+
+    alias_method :count, :n
+    alias_method :mean, :mu
+    alias_method :max_value, :max
+    alias_method :min_value, :min
+    alias_method :sampleStdev, :sample_stdev
+    alias_method :sampleVariance, :sample_variance
+
+    private
+
+      def merge_stat_counter(other)
+        if other == self
+          other = self.deep_copy
+        end
+
+        if @n == 0
+          @n = other.n
+          @mu = other.mu
+          @m2 = other.m2
+          @max = other.max
+          @min = other.min
+        elsif other.n != 0
+          delta = other.mu - @mu
+
+          if other.n * 10 < @n
+            @mu = @mu + (delta * other.n) / (@n + other.n)
+          elsif @n * 10 < other.n
+            @mu = other.mu - (delta * @n) / (@n + other.n)
+          else
+            @mu = (@mu * @n + other.mu * other.n) / (@n + other.n)
+          end
+
+          @max = [@max, other.max].max
+          @min = [@min, other.min].min
+
+          @m2 += other.m2 + (delta * delta * @n * other.n) / (@n + other.n)
+          @n += other.n
+        end
+      end
+
+      def merge_array(array)
+        array.each do |item|
+          merge_value(item)
+        end
+      end
+
+      def merge_value(value)
+        delta = value - @mu
+        @n += 1
+        @mu += delta / @n
+        @m2 += delta * (value - @mu)
+        @max = [@max, value].max
+        @min = [@min, value].min
+      end
+
+  end
+end

data/lib/spark/storage_level.rb

@@ -0,0 +1,39 @@
+# Necessary libraries
+Spark.load_lib
+
+module Spark
+  class StorageLevel
+
+    def self.reload
+      return if @reloaded
+      reload!
+      @reloaded = true
+    end
+
+    def self.reload!
+      self.const_set(:NONE,                  JStorageLevel.NONE)
+      self.const_set(:DISK_ONLY,             JStorageLevel.DISK_ONLY)
+      self.const_set(:DISK_ONLY_2,           JStorageLevel.DISK_ONLY_2)
+      self.const_set(:MEMORY_ONLY,           JStorageLevel.MEMORY_ONLY)
+      self.const_set(:MEMORY_ONLY_SER,       JStorageLevel.MEMORY_ONLY_SER)
+      self.const_set(:MEMORY_ONLY_2,         JStorageLevel.MEMORY_ONLY_2)
+      self.const_set(:MEMORY_ONLY_SER_2,     JStorageLevel.MEMORY_ONLY_SER_2)
+      self.const_set(:MEMORY_AND_DISK,       JStorageLevel.MEMORY_AND_DISK)
+      self.const_set(:MEMORY_AND_DISK_2,     JStorageLevel.MEMORY_AND_DISK_2)
+      self.const_set(:MEMORY_AND_DISK_SER,   JStorageLevel.MEMORY_AND_DISK_SER)
+      self.const_set(:MEMORY_AND_DISK_SER_2, JStorageLevel.MEMORY_AND_DISK_SER_2)
+      self.const_set(:OFF_HEAP,              JStorageLevel.OFF_HEAP)
+    end
+
+    def self.java_get(arg)
+      reload
+
+      if arg.is_a?(String)
+        const_get(arg.upcase)
+      else
+        arg
+      end
+    end
+
+  end
+end

data/lib/spark/version.rb

@@ -0,0 +1,3 @@
+module Spark
+  VERSION = '1.0.0'
+end

data/lib/spark/worker/master.rb

@@ -0,0 +1,144 @@
+#!/usr/bin/env ruby
+
+$PROGRAM_NAME = 'RubySparkMaster'
+
+require 'socket'
+require 'io/wait'
+require 'nio'
+
+require_relative 'worker'
+
+# New process group
+# Otherwise master can be killed from pry console
+Process.setsid
+
+# =================================================================================================
+# Master
+#
+module Master
+
+  def self.create
+    case ARGV[0].to_s.strip
+    when 'thread'
+      Master::Thread.new
+    else
+      Master::Process.new
+    end
+  end
+
+  class Base
+    include Spark::Constant
+
+    def initialize
+      @port = ARGV[1].to_s.strip.to_i
+      @socket = TCPSocket.open('localhost', @port)
+      @worker_arguments = @socket.read_string
+    end
+
+    def run
+      selector = NIO::Selector.new
+      monitor = selector.register(@socket, :r)
+      monitor.value = Proc.new { receive_message }
+      loop {
+        selector.select {|monitor| monitor.value.call}
+      }
+    end
+
+    def receive_message
+      command = @socket.read_int
+
+      case command
+      when CREATE_WORKER
+        create_worker
+      when KILL_WORKER
+        kill_worker
+      when KILL_WORKER_AND_WAIT
+        kill_worker_and_wait
+      end
+    end
+
+    def kill_worker_and_wait
+      if kill_worker
+        @socket.write_int(SUCCESSFULLY_KILLED)
+      else
+        @socket.write_int(UNSUCCESSFUL_KILLING)
+      end
+    end
+  end
+
+  # ===============================================================================================
+  # Worker::Process
+  #
+  class Process < Base
+
+    def create_worker
+      if fork?
+        pid = ::Process.fork do
+          Worker::Process.new(@port).run
+        end
+      else
+        pid = ::Process.spawn("ruby #{@worker_arguments} worker.rb #{@port}")
+      end
+
+      # Detach child from master to avoid zombie process
+      ::Process.detach(pid)
+    end
+
+    def kill_worker
+      worker_id = @socket.read_long
+      ::Process.kill('TERM', worker_id)
+    rescue
+      nil
+    end
+
+    def fork?
+      @can_fork ||= _fork?
+    end
+
+    def _fork?
+      return false if !::Process.respond_to?(:fork)
+
+      pid = ::Process.fork
+      exit unless pid # exit the child immediately
+      true
+    rescue NotImplementedError
+      false
+    end
+
+  end
+
+  # ===============================================================================================
+  # Worker::Thread
+  #
+  class Thread < Base
+
+    def initialize
+      ::Thread.abort_on_exception = true
+
+      # For synchronous access to socket IO
+      $mutex_for_command  = Mutex.new
+      $mutex_for_iterator = Mutex.new
+
+      super
+    end
+
+    def create_worker
+      ::Thread.new do
+        Worker::Thread.new(@port).run
+      end
+    end
+
+    def kill_worker
+      worker_id = @socket.read_long
+
+      thread = ObjectSpace._id2ref(worker_id)
+      thread.kill
+    rescue
+      nil
+    end
+
+  end
+end
+
+# Create proper master by worker_type
+Master.create.run