ruby-spark 1.0.0

data/lib/spark/command/pair.rb

@@ -0,0 +1,124 @@
+_Base = Spark::Command::Base
+
+# -------------------------------------------------------------------------------------------------
+# CombineByKey
+
+class Spark::Command::CombineByKey
+
+  # ---------------
+
+  class Base < Spark::Command::Base
+    def run(iterator, *)
+      _run(iterator).to_a
+    end
+
+    def lazy_run(iterator, *)
+      _run(iterator).lazy
+    end
+  end
+
+  # ---------------
+
+  class Combine < Base
+    variable :create_combiner
+    variable :merge_value
+
+    def _run(iterator)
+      # Not use combiners[key] ||= ..
+      # it tests nil and not has_key?
+      combiners = {}
+      iterator.each do |key, value|
+        if combiners.has_key?(key)
+          combiners[key] = @merge_value.call(combiners[key], value)
+        else
+          combiners[key] = @create_combiner.call(value)
+        end
+      end
+      combiners
+    end
+  end
+
+  # ---------------
+
+  class Merge < Base
+    variable :merge_combiners
+
+    def _run(iterator, *)
+      combiners = {}
+      iterator.each do |key, value|
+        if combiners.has_key?(key)
+          combiners[key] = @merge_combiners.call(combiners[key], value)
+        else
+          combiners[key] = value
+        end
+      end
+      combiners
+    end
+  end
+
+  # ---------------
+
+  class CombineWithZero < Base
+    variable :zero_value, function: false, type: Object
+    variable :merge_value
+
+    def _run(iterator)
+      # Not use combiners[key] ||= ..
+      # it tests nil and not has_key?
+      combiners = {}
+      iterator.each do |key, value|
+        unless combiners.has_key?(key)
+          combiners[key] = @zero_value
+        end
+
+        combiners[key] = @merge_value.call(combiners[key], value)
+      end
+      combiners
+    end
+  end
+
+
+  # ---------------
+
+end
+
+# -------------------------------------------------------------------------------------------------
+# MapValues
+
+class Spark::Command::MapValues < _Base
+  variable :map_function
+
+  def run(iterator, *)
+    iterator.map! do |item|
+      item[1] = @map_function.call(item[1])
+      item
+    end
+    iterator
+  end
+
+  def lazy_run(iterator, *)
+    iterator.map do |item|
+      item[1] = @map_function.call(item[1])
+      item
+    end
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# FlatMapValues
+
+class Spark::Command::FlatMapValues < _Base
+  variable :map_function
+
+  def run(iterator, *)
+    iterator.map! do |(key, values)|
+      values = @map_function.call(values)
+      values.flatten!(1)
+      values.map! do |value|
+        [key, value]
+      end
+    end
+    iterator.flatten!(1)
+    iterator
+  end
+end

data/lib/spark/command/sort.rb

@@ -0,0 +1,51 @@
+_Base = Spark::Command::Base
+
+# -------------------------------------------------------------------------------------------------
+# Sort
+
+class Spark::Command::SortByKey < _Base
+  variable :key_function
+  variable :ascending,  function: false, type: [TrueClass, FalseClass]
+  variable :spilling,   function: false, type: [TrueClass, FalseClass]
+  variable :memory,     function: false, type: [Numeric, NilClass]
+  variable :serializer, function: false, type: Spark::Serializer::Base
+
+  # Currently disabled
+  def before_run
+    @spilling = false
+  end
+
+  def run(iterator, _)
+    if @spilling
+      iterator = run_with_spilling(iterator.each)
+    else
+      run_without_spilling(iterator)
+    end
+
+    iterator
+  end
+
+  def run_with_enum(iterator, _)
+    if @spilling
+      iterator = run_with_spilling(iterator)
+    else
+      iterator = iterator.to_a
+      run_without_spilling(iterator)
+    end
+
+    iterator
+  end
+
+  private
+
+    def run_with_spilling(iterator)
+      sorter = Spark::ExternalSorter.new(@memory, @serializer)
+      sorter.sort_by(iterator, @ascending, @key_function)
+    end
+
+    def run_without_spilling(iterator)
+      iterator.sort_by!(&@key_function)
+      iterator.reverse! unless @ascending
+    end
+
+end

data/lib/spark/command/statistic.rb

@@ -0,0 +1,144 @@
+_Base = Spark::Command::Base
+
+# -------------------------------------------------------------------------------------------------
+# Sample
+
+class Spark::Command::Sample < _Base
+  variable :with_replacement, function: false, type: [TrueClass, FalseClass]
+  variable :fraction,         function: false, type: Numeric
+  variable :seed,             function: false, type: [NilClass, Numeric]
+
+  def run(iterator, _)
+    sampler.sample(iterator)
+  end
+
+  def lazy_run(iterator, _)
+    sampler.lazy_sample(iterator)
+  end
+
+  def sampler
+    @sampler ||= _sampler
+  end
+
+  def _sampler
+    if @with_replacement
+      sampler = Spark::Sampler::Poisson
+    else
+      sampler = Spark::Sampler::Uniform
+    end
+
+    sampler = sampler.new(@fraction, @seed)
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# Stats
+
+class Spark::Command::Stats < _Base
+
+  def run(iterator, *)
+    [Spark::StatCounter.new(iterator)]
+  end
+
+  def lazy_run(iterator, *)
+    run(iterator)
+  end
+
+end
+
+# -------------------------------------------------------------------------------------------------
+# Histogram
+
+class Spark::Command::Histogram < _Base
+  include Spark::Helper::Statistic
+
+  variable :even,    function: false, type: [TrueClass, FalseClass]
+  variable :buckets, function: false, type: Array
+
+  def run(iterator, *)
+    counters = Array.new(counter_size) { 0 }
+    iterator.each do |item|
+      if item.nil? || (item.is_a?(Float) && !item.finite?) || item > max || item < min
+        next
+      end
+
+      x = bucket_function.call(item)
+      if x.nil?
+        # next
+      else
+        counters[x] += 1
+      end
+    end
+    [counters]
+  end
+
+  def lazy_run(iterator, *)
+    run(iterator)
+  end
+
+  private
+
+    def min
+      @buckets.first
+    end
+
+    def max
+      @buckets.last
+    end
+
+    def counter_size
+      @buckets.size-1
+    end
+
+    def increment
+      @buckets[1]-@buckets[0]
+    end
+
+    # Decide which bucket function to pass. We decide here rather than having
+    # a general function so that the decission need only be made once.
+    def bucket_function
+      @bucket_function ||= _bucket_function
+    end
+
+    def _bucket_function
+      if @even
+        fast_bucket_function
+      else
+        basic_bucket_function
+      end
+    end
+
+    # Determine the bucket function in constant time.
+    # Requires that buckets are evenly spaced
+    def fast_bucket_function
+      Proc.new do |item|
+        if item.is_a?(Float) && item.nan?
+          nil
+        else
+          bucket_number = (item - min)/increment
+          if bucket_number > counter_size || bucket_number < 0
+            nil
+          else
+            [bucket_number.to_i, counter_size-1].min
+          end
+        end
+      end
+    end
+
+    # Basic bucket function. Same as right bisect.
+    def basic_bucket_function
+      Proc.new do |item|
+        bucket_number = bisect_right(@buckets, item) - 1
+
+        # Counters is @buckets.size - 1
+        # [bucket_number, counter_size-1].min
+
+        if bucket_number > counter_size-1
+          counter_size-1
+        else
+          bucket_number
+        end
+      end
+    end
+
+end

data/lib/spark/command_builder.rb

@@ -0,0 +1,141 @@
+require 'spark/command_validator'
+
+module Spark
+  ##
+  # Builder for building correct {Spark::Command}
+  #
+  class CommandBuilder
+
+    extend Forwardable
+
+    include Spark::Helper::Serialize
+    include Spark::Helper::System
+    include Spark::CommandValidator
+
+    attr_reader :command
+
+    def_delegators :@command, :serializer, :serializer=, :deserializer, :deserializer=, :commands,
+                              :commands=, :libraries, :libraries=, :bound_objects, :bound_objects=
+
+    def initialize(serializer, deserializer=nil)
+      create_command
+      self.serializer   = serializer
+      self.deserializer = deserializer || serializer.dup
+    end
+
+    def create_command
+      @command = Spark::Command.new
+    end
+
+    # Do not user Marshal.dump(Marshal.load(self)) because some variables
+    # have marshal_dump prepared for worker.
+    def deep_copy
+      copy = self.dup
+      copy.create_command
+      copy.serializer    = self.serializer.dup
+      copy.deserializer  = self.deserializer.dup
+      copy.commands      = self.commands.dup
+      copy.libraries     = self.libraries.dup
+      copy.bound_objects = self.bound_objects.dup
+      copy
+    end
+
+    # Serialize Command class for worker
+    # Java use signed number
+    def build
+      unpack_chars(Marshal.dump(@command))
+    end
+
+    def add_command(klass, *args)
+      variables = klass.settings.variables
+      validate_size(variables, args)
+
+      built_args = []
+      variables.values.zip(args) do |var, arg|
+        if var[:function]
+          arg = serialize_function(arg)
+        end
+
+        validate(arg, var)
+        built_args << arg
+      end
+
+      comm = klass.new(*built_args)
+      @command.commands << comm
+      self
+    end
+
+    def add_library(*libraries)
+      @command.libraries += libraries
+    end
+
+    def bind(objects)
+      objects.symbolize_keys!
+      @command.bound_objects.merge!(objects)
+    end
+
+    private
+
+        # Serialized can be Proc and Method
+        #
+        # === Func
+        # * *string:* already serialized proc
+        # * *proc:* proc
+        # * *symbol:* name of method
+        # * *method:* Method class
+        #
+        def serialize_function(func)
+          case func
+          when String
+            serialize_function_from_string(func)
+          when Symbol
+            serialize_function_from_symbol(func)
+          when Proc
+            serialize_function_from_proc(func)
+          when Method
+            serialize_function_from_method(func)
+          else
+            raise Spark::CommandError, 'You must enter String, Symbol, Proc or Method.'
+          end
+        end
+
+        def serialize_function_from_string(string)
+          {type: 'proc', content: string}
+        end
+
+        def serialize_function_from_symbol(symbol)
+          {type: 'symbol', content: symbol}
+        end
+
+        # Serialize Proc as String
+        #
+        #   lambda{|x| x*x}.to_source
+        #   # => "proc { |x| (x * x) }"
+        #
+        def serialize_function_from_proc(proc)
+          serialize_function_from_string(proc.to_source)
+        rescue
+          raise Spark::SerializeError, 'Proc can not be serialized. Use String instead.'
+        end
+
+        # Serialize method as string
+        #
+        #   def test(x)
+        #     x*x
+        #   end
+        #   serialize_function_from_method(method(:test))
+        #
+        #   # => "def test(x)\n  x*x\nend\n"
+        #
+        def serialize_function_from_method(meth)
+          if pry?
+            meth = Pry::Method.new(meth)
+          end
+
+          {type: 'method', name: meth.name, content: meth.source}
+        rescue
+          raise Spark::SerializeError, 'Method can not be serialized. Use full path or Proc.'
+        end
+
+  end
+end