ruby-spark 1.1.0.1-java

data/lib/spark/serializer/oj.rb

@@ -0,0 +1,23 @@
+module Spark
+  module Serializer
+    class Oj < Base
+
+      def dump(data)
+        ::Oj.dump(data)
+      end
+
+      def load(data)
+        ::Oj.load(data)
+      end
+
+    end
+  end
+end
+
+begin
+  # TODO: require only if it is necessary
+  require 'oj'
+
+  Spark::Serializer.register('oj', Spark::Serializer::Oj)
+rescue LoadError
+end

data/lib/spark/serializer/pair.rb

@@ -0,0 +1,41 @@
+module Spark
+  module Serializer
+    class Pair < Base
+
+      def initialize(serializer1, serializer2)
+        @serializer1 = serializer1
+        @serializer2 = serializer2
+      end
+
+      def to_s
+        "#{name}(#{@serializer1}, #{@serializer2})"
+      end
+
+      def aggregate(item1, item2)
+        item1.zip(item2)
+      end
+
+      def load_from_io(io)
+        return to_enum(__callee__, io) unless block_given?
+
+        loop do
+          size = io.read_int_or_eof
+          break if size == Spark::Constant::DATA_EOF
+
+          item1 = @serializer1.load(io.read(size))
+          item2 = @serializer2.load(io.read_string)
+
+          item1 = [item1] unless @serializer1.batched?
+          item2 = [item2] unless @serializer2.batched?
+
+          aggregate(item1, item2).each do |item|
+            yield item
+          end
+        end
+      end
+
+    end
+  end
+end
+
+Spark::Serializer.register('pair', Spark::Serializer::Pair)

data/lib/spark/serializer/text.rb

@@ -0,0 +1,25 @@
+module Spark
+  module Serializer
+    class Text < Base
+
+      attr_reader :encoding
+
+      def initialize(encoding=Encoding::UTF_8)
+        error('Encoding must be an instance of Encoding') unless encoding.is_a?(Encoding)
+
+        @encoding = encoding
+      end
+
+      def load(data)
+        data.to_s.force_encoding(@encoding)
+      end
+
+      def to_s
+        "Text(#{@encoding})"
+      end
+
+    end
+  end
+end
+
+Spark::Serializer.register('string', 'text', Spark::Serializer::Text)

data/lib/spark/sort.rb

@@ -0,0 +1,189 @@
+module Spark
+  module InternalSorter
+    class Base
+      def initialize(key_function)
+        @key_function = key_function
+      end
+    end
+
+    class Ascending < Base
+      def sort(data)
+        data.sort_by!(&@key_function)
+      end
+    end
+
+    class Descending < Ascending
+      def sort(data)
+        super
+        data.reverse!
+      end
+    end
+
+    def self.get(ascending, key_function)
+      if ascending
+        type = Ascending
+      else
+        type = Descending
+      end
+
+      type.new(key_function)
+    end
+  end
+end
+
+
+module Spark
+  class ExternalSorter
+
+    include Spark::Helper::System
+
+    # Items from GC cannot be destroyed so #make_parts need some reserve
+    MEMORY_RESERVE = 50 # %
+
+    # How big will be chunk for adding new memory because GC not cleaning
+    # immediately un-referenced variables
+    MEMORY_FREE_CHUNK = 10 # %
+
+    # How many items will be evaluate from iterator at start
+    START_SLICE_SIZE = 10
+
+    # Maximum of slicing. Memory control can be avoided by large value.
+    MAX_SLICE_SIZE = 10_000
+
+    # How many values will be taken from each enumerator.
+    EVAL_N_VALUES = 10
+
+    # Default key function
+    KEY_FUNCTION = lambda{|item| item}
+
+    attr_reader :total_memory, :memory_limit, :memory_chunk, :serializer
+
+    def initialize(total_memory, serializer)
+      @total_memory = total_memory
+      @memory_limit = total_memory * (100-MEMORY_RESERVE)    / 100
+      @memory_chunk = total_memory * (100-MEMORY_FREE_CHUNK) / 100
+      @serializer   = serializer
+    end
+
+    def add_memory!
+      @memory_limit += memory_chunk
+    end
+
+    def sort_by(iterator, ascending=true, key_function=KEY_FUNCTION)
+      return to_enum(__callee__, iterator, key_function) unless block_given?
+
+      create_temp_folder
+      internal_sorter = Spark::InternalSorter.get(ascending, key_function)
+
+      # Make N sorted enumerators
+      parts = make_parts(iterator, internal_sorter)
+
+      return [] if parts.empty?
+
+      # Need new key function because items have new structure
+      # From: [1,2,3] to [[1, Enumerator],[2, Enumerator],[3, Enumerator]]
+      key_function_with_enum = lambda{|(key, _)| key_function[key]}
+      internal_sorter = Spark::InternalSorter.get(ascending, key_function_with_enum)
+
+      heap  = []
+      enums = []
+
+      # Load first items to heap
+      parts.each do |part|
+        EVAL_N_VALUES.times {
+          begin
+            heap << [part.next, part]
+          rescue StopIteration
+            break
+          end
+        }
+      end
+
+      # Parts can be empty but heap not
+      while parts.any? || heap.any?
+        internal_sorter.sort(heap)
+
+        # Since parts are sorted and heap contains EVAL_N_VALUES method
+        # can add EVAL_N_VALUES items to the result
+        EVAL_N_VALUES.times {
+          break if heap.empty?
+
+          item, enum = heap.shift
+          enums << enum
+
+          yield item
+        }
+
+        # Add new element to heap from part of which was result item
+        while (enum = enums.shift)
+          begin
+            heap << [enum.next, enum]
+          rescue StopIteration
+            parts.delete(enum)
+            enums.delete(enum)
+          end
+        end
+      end
+
+    ensure
+      destroy_temp_folder
+    end
+
+    private
+
+      def create_temp_folder
+        @dir = Dir.mktmpdir
+      end
+
+      def destroy_temp_folder
+        FileUtils.remove_entry_secure(@dir) if @dir
+      end
+
+      # New part is created when current part exceeds memory limit (is variable)
+      # Every new part have more memory because of ruby GC
+      def make_parts(iterator, internal_sorter)
+        slice = START_SLICE_SIZE
+
+        parts = []
+        part  = []
+
+        loop do
+          begin
+            # Enumerator does not have slice method
+            slice.times { part << iterator.next }
+          rescue StopIteration
+            break
+          end
+
+          # Carefully memory_limit is variable
+          if memory_usage > memory_limit
+            # Sort current part with origin key_function
+            internal_sorter.sort(part)
+            # Tempfile for current part
+            # will be destroyed on #destroy_temp_folder
+            file = Tempfile.new("part", @dir)
+            serializer.dump(part, file)
+            # Peek is at the end of file
+            file.seek(0)
+            parts << serializer.load(file)
+
+            # Some memory will be released but not immediately
+            # need some new memory for start
+            part.clear
+            add_memory!
+          else
+            slice = [slice*2, MAX_SLICE_SIZE].min
+          end
+        end
+
+        # Last part which is not in the file
+        if part.any?
+          internal_sorter.sort(part)
+          parts << part.each
+        end
+
+        parts
+      end
+
+  end # ExternalSorter
+end # Spark

data/lib/spark/stat_counter.rb

@@ -0,0 +1,125 @@
+module Spark
+  class StatCounter
+
+    attr_reader :n   # count of our values
+    attr_reader :mu  # mean of our values
+    attr_reader :m2  # variance numerator (sum of (x - mean)^2)
+    attr_reader :max # max of our values
+    attr_reader :min # min of our values
+
+    def initialize(iterator)
+      @n = 0
+      @mu = 0.0
+      @m2 = 0.0
+      @max = -Float::INFINITY
+      @min = Float::INFINITY
+
+      merge(iterator)
+    end
+
+    def merge(other)
+      if other.is_a?(Spark::StatCounter)
+        merge_stat_counter(other)
+      elsif other.respond_to?(:each)
+        merge_array(other)
+      else
+        merge_value(other)
+      end
+
+      self
+    end
+
+    def sum
+      @n * @mu
+    end
+
+    # Return the variance of the values.
+    def variance
+      if @n == 0
+        Float::NAN
+      else
+        @m2 / @n
+      end
+    end
+
+    # Return the sample variance, which corrects for bias in estimating the variance by dividing
+    # by N-1 instead of N.
+    def sample_variance
+      if @n <= 1
+        Float::NAN
+      else
+        @m2 / (@n - 1)
+      end
+    end
+
+    # Return the standard deviation of the values.
+    def stdev
+      Math.sqrt(variance)
+    end
+
+    # Return the sample standard deviation of the values, which corrects for bias in estimating the
+    # variance by dividing by N-1 instead of N.
+    def sample_stdev
+      Math.sqrt(sample_variance)
+    end
+
+    def to_s
+      "(count: #{count}, mean: #{mean}, stdev: #{stdev}, max: #{max}, min: #{min})"
+    end
+
+    alias_method :count, :n
+    alias_method :mean, :mu
+    alias_method :max_value, :max
+    alias_method :min_value, :min
+    alias_method :sampleStdev, :sample_stdev
+    alias_method :sampleVariance, :sample_variance
+
+    private
+
+      def merge_stat_counter(other)
+        if other == self
+          other = self.deep_copy
+        end
+
+        if @n == 0
+          @n = other.n
+          @mu = other.mu
+          @m2 = other.m2
+          @max = other.max
+          @min = other.min
+        elsif other.n != 0
+          delta = other.mu - @mu
+
+          if other.n * 10 < @n
+            @mu = @mu + (delta * other.n) / (@n + other.n)
+          elsif @n * 10 < other.n
+            @mu = other.mu - (delta * @n) / (@n + other.n)
+          else
+            @mu = (@mu * @n + other.mu * other.n) / (@n + other.n)
+          end
+
+          @max = [@max, other.max].max
+          @min = [@min, other.min].min
+
+          @m2 += other.m2 + (delta * delta * @n * other.n) / (@n + other.n)
+          @n += other.n
+        end
+      end
+
+      def merge_array(array)
+        array.each do |item|
+          merge_value(item)
+        end
+      end
+
+      def merge_value(value)
+        delta = value - @mu
+        @n += 1
+        @mu += delta / @n
+        @m2 += delta * (value - @mu)
+        @max = [@max, value].max
+        @min = [@min, value].min
+      end
+
+  end
+end

data/lib/spark/storage_level.rb

@@ -0,0 +1,39 @@
+# Necessary libraries
+Spark.load_lib
+
+module Spark
+  class StorageLevel
+
+    def self.reload
+      return if @reloaded
+      reload!
+      @reloaded = true
+    end
+
+    def self.reload!
+      self.const_set(:NONE,                  JStorageLevel.NONE)
+      self.const_set(:DISK_ONLY,             JStorageLevel.DISK_ONLY)
+      self.const_set(:DISK_ONLY_2,           JStorageLevel.DISK_ONLY_2)
+      self.const_set(:MEMORY_ONLY,           JStorageLevel.MEMORY_ONLY)
+      self.const_set(:MEMORY_ONLY_SER,       JStorageLevel.MEMORY_ONLY_SER)
+      self.const_set(:MEMORY_ONLY_2,         JStorageLevel.MEMORY_ONLY_2)
+      self.const_set(:MEMORY_ONLY_SER_2,     JStorageLevel.MEMORY_ONLY_SER_2)
+      self.const_set(:MEMORY_AND_DISK,       JStorageLevel.MEMORY_AND_DISK)
+      self.const_set(:MEMORY_AND_DISK_2,     JStorageLevel.MEMORY_AND_DISK_2)
+      self.const_set(:MEMORY_AND_DISK_SER,   JStorageLevel.MEMORY_AND_DISK_SER)
+      self.const_set(:MEMORY_AND_DISK_SER_2, JStorageLevel.MEMORY_AND_DISK_SER_2)
+      self.const_set(:OFF_HEAP,              JStorageLevel.OFF_HEAP)
+    end
+
+    def self.java_get(arg)
+      reload
+
+      if arg.is_a?(String)
+        const_get(arg.upcase)
+      else
+        arg
+      end
+    end
+
+  end
+end