ruby-spark 1.0.0

data/lib/spark/command.rb

@@ -0,0 +1,86 @@
+module Spark
+  ##
+  # Container which includes all commands and other things for worker
+  # Every RDD have own copy of Command
+  #
+  class Command
+
+    attr_accessor :serializer, :deserializer, :commands, :libraries, :bound_objects
+
+    def initialize
+      @serializer = nil
+      @deserializer = nil
+      @commands = []
+      @libraries = []
+      @bound_objects = {}
+    end
+
+    def execute(iterator, split_index)
+      # Require necessary libraries
+      libraries.each{|lib| require lib}
+
+      # Prepare bound objects
+      @commands.each do |command|
+        command.__objects__ = bound_objects
+      end
+
+      # Prepare for running
+      @commands.each(&:prepare)
+
+      # Run all task
+      @commands.each do |command|
+        iterator = command.execute(iterator, split_index)
+      end
+
+      # Return changed iterator. This is not be necessary for some tasks
+      # because of using inplace changing but some task can return
+      # only one value (for example reduce).
+      iterator
+    end
+
+    def last
+      @commands.last
+    end
+
+    def bound_objects
+      # Objects from users
+      # Already initialized objects on worker
+      return @bound_objects if @bound_objects
+
+      if @serialized_bound_objects
+        # Still serialized
+        @bound_objects = Marshal.load(@serialized_bound_objects)
+      else
+        # Something else
+        @bound_objects = {}
+      end
+    end
+
+    # Bound objects can depend on library which is loaded during @execute
+    # In that case worker raise "undefined class/module"
+    def marshal_dump
+      [@serializer, @deserializer, @commands, @libraries, serialized_bound_objects]
+    end
+
+    def marshal_load(array)
+      @serializer = array.shift
+      @deserializer = array.shift
+      @commands = array.shift
+      @libraries = array.shift
+      @serialized_bound_objects = array.shift
+    end
+
+    private
+
+      def serialized_bound_objects
+        @serialized_bound_objects ||= Marshal.dump(@bound_objects)
+      end
+
+  end
+end
+
+require 'spark/command/base'
+require 'spark/command/basic'
+require 'spark/command/pair'
+require 'spark/command/statistic'
+require 'spark/command/sort'

data/lib/spark/command/base.rb

@@ -0,0 +1,154 @@
+##
+# Spark::Command::Base
+#
+# Parent for all commands (Map, FlatMap, Sort, ...)
+#
+class Spark::Command::Base
+
+  DEFAULT_VARIABLE_OPTIONS = {
+    type: Hash,
+    function: true
+  }
+
+  def initialize(*args)
+    settings.variables.each do |name, options|
+      instance_variable_set("@#{name}", args.shift)
+    end
+  end
+
+  def self.error(message)
+    raise Spark::CommandError, message
+  end
+
+  def error(message)
+    self.class.error(message)
+  end
+
+  def log(message=nil)
+    $stdout.puts %{==> #{Time.now.strftime("%H:%M:%S")} [#{self.class.name}] #{message}}
+    $stdout.flush
+  end
+
+
+  # ===============================================================================================
+  # Methods called during class loading
+  # This is not nicer way but these methods set/get classes variables for child
+
+  # Settings for command (variables)
+  def self.settings
+    init_settings
+    class_variable_get(:@@settings)
+  end
+
+  def settings
+    self.class.settings
+  end
+
+  # Init empty settings
+  def self.init_settings
+    if !class_variable_defined?(:@@settings)
+      struct = Struct.new(:variables)
+
+      class_variable_set(:@@settings, struct.new)
+      settings.variables = {}
+    end
+  end
+
+  # New variable for command
+  #
+  # == Example:
+  #
+  #   class Map < Spark::Command::Base
+  #     variable :map_function
+  #   end
+  #
+  #   command = Map.new(1)
+  #
+  #   command.instance_variables
+  #   # => [:@map_function]
+  #   command.instance_variable_get(:@map_function)
+  #   # => 1
+  #
+  def self.variable(name, options={})
+    if settings.variables.has_key?(name)
+      error "Function #{name} already exist."
+    end
+
+    settings.variables[name] = DEFAULT_VARIABLE_OPTIONS.merge(options)
+  end
+
+
+  # ===============================================================================================
+  # Executing methods
+
+  # Execute command for data and split index
+  def execute(iterator, split_index)
+    # Implemented on Base but can be override
+    before_run
+
+    # Run has to be implemented on child
+    if iterator.is_a?(Enumerator::Lazy) && respond_to?(:lazy_run)
+      return lazy_run(iterator, split_index)
+    end
+
+    iterator = iterator.to_a
+    run(iterator, split_index)
+  end
+
+  def prepared?
+    !!@prepared
+  end
+
+  # This is called before execution. Executing will be stopped if
+  # some command contains error (e.g. badly serialized lambda).
+  #
+  # == What is doing?
+  # * evaluate lambda
+  # * evaluate method
+  # * make new lambda
+  #
+  def prepare
+    return if prepared?
+
+    to_function = settings.variables.select {|_, options| options[:function]}
+    to_function.each do |name, options|
+      name = "@#{name}"
+      data = instance_variable_get(name)
+
+      case data[:type]
+      when 'proc'
+        result = eval(data[:content])
+      when 'symbol'
+        result = lambda(&data[:content])
+      when 'method'
+        # Method must me added to instance not Class
+        instance_eval(data[:content])
+        # Method will be available as Proc
+        result = lambda(&method(data[:name]))
+      end
+
+      instance_variable_set(name, result)
+    end
+
+    @prepared = true
+  end
+
+  # This method is called before every execution.
+  def before_run
+  end
+
+
+  # ===============================================================================================
+  # Bound objects
+
+  attr_accessor :__objects__
+
+  def method_missing(method, *args, &block)
+    if __objects__ && __objects__.has_key?(method)
+      return __objects__[method]
+    end
+
+    super
+  end
+
+end

data/lib/spark/command/basic.rb

@@ -0,0 +1,345 @@
+_Base = Spark::Command::Base
+
+# -------------------------------------------------------------------------------------------------
+# Map
+
+class Spark::Command::Map < _Base
+  variable :map_function
+
+  def run(iterator, *)
+    iterator.map! do |item|
+      @map_function.call(item)
+    end
+    iterator
+  end
+
+  def lazy_run(iterator, *)
+    iterator.map do |item|
+      @map_function.call(item)
+    end
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# FlatMap
+
+class Spark::Command::FlatMap < Spark::Command::Map
+  def run(iterator, *)
+    iterator = super
+    iterator.flatten!(1)
+    iterator
+  end
+
+  def lazy_run(iterator, *)
+    iterator.flat_map do |item|
+      @map_function.call(item)
+    end
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# MapPartitionsWithIndex
+
+class Spark::Command::MapPartitionsWithIndex < _Base
+  variable :partition_function
+
+  def run(iterator, index)
+    iterator = @partition_function.call(iterator, index)
+    iterator
+  end
+
+  # User should controll if there is Enumerator or not
+  # alias_method :lazy_run, :run
+end
+
+# -------------------------------------------------------------------------------------------------
+# MapPartitions
+
+class Spark::Command::MapPartitions < Spark::Command::MapPartitionsWithIndex
+  def run(iterator, *)
+    # Do not use `super` because `@partition_function` can be method with 1 argument
+    iterator = @partition_function.call(iterator)
+    iterator
+  end
+  # alias_method :lazy_run, :run
+end
+
+# -------------------------------------------------------------------------------------------------
+# Filter
+
+class Spark::Command::Filter < _Base
+  variable :filter_function
+
+  def run(iterator, *)
+    iterator.select! do |item|
+      @filter_function.call(item)
+    end
+    iterator
+  end
+
+  def lazy_run(iterator, *)
+    iterator.select do |item|
+      @filter_function.call(item)
+    end
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# Compact
+
+class Spark::Command::Compact < _Base
+  def run(iterator, *)
+    iterator.compact!
+    iterator
+  end
+
+  def lazy_run(iterator, *)
+    iterator.select do |item|
+      !item.nil?
+    end
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# Glom
+
+class Spark::Command::Glom < _Base
+  def run(iterator, *)
+    [iterator]
+  end
+
+  def lazy_run(iterator, *)
+    run(iterator.to_a)
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# Shuffle
+
+class Spark::Command::Shuffle < _Base
+  variable :seed, function: false, type: Integer
+
+  def run(iterator, *)
+    iterator.shuffle!(random: rng)
+    iterator
+  end
+
+  def rng
+    Random.new(@seed)
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# PartitionBy
+
+class Spark::Command::PartitionBy
+
+  class Base < Spark::Command::Base
+    include Spark::Helper::Serialize
+
+    def prepare
+      super
+
+      # Default. Keep it after super because Sorting has own key_function.
+      @key_function ||= lambda{|x| x[0]}
+    end
+
+    def run(iterator, *)
+      iterator.map! do |item|
+        make_partition_item(item)
+      end
+      iterator.flatten!(1)
+      iterator
+    end
+
+    def lazy_run(iterator, *)
+      iterator.flat_map do |item|
+        make_partition_item(item)
+      end
+    end
+
+    private
+
+      def make_partition_item(item)
+        [
+          pack_long(@partition_func.call(@key_function[item])),
+          item
+        ]
+      end
+  end
+
+  class Basic < Base
+    variable :partition_func
+  end
+
+  class Sorting < Base
+    variable :key_function
+    variable :bounds, function: false, type: Array
+    variable :ascending, function: false, type: [TrueClass, FalseClass]
+    variable :num_partitions, function: false, type: Numeric
+
+    def prepare
+      super
+
+      # Index by bisect alghoritm
+      @partition_func ||= Proc.new do |key|
+        count = 0
+        @bounds.each{|i|
+          break if i >= key
+          count += 1
+        }
+
+        if @ascending
+          count
+        else
+          @num_partitions - 1 - count
+        end
+      end
+    end
+
+  end # Sorting
+end # PartitionBy
+
+# -------------------------------------------------------------------------------------------------
+# Aggregate
+
+class Spark::Command::Aggregate < _Base
+  variable :reduce_func
+  variable :zero_value, function: false, type: Object
+
+  def run(iterator, *)
+    [iterator.reduce(@zero_value, &@reduce_func)]
+  end
+
+  def lazy_run(iterator, *)
+    run(iterator)
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# Reduce
+
+class Spark::Command::Reduce < Spark::Command::Aggregate
+  def run(iterator, *)
+    [iterator.reduce(&@reduce_func)]
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# Foreach
+
+class Spark::Command::Foreach < _Base
+  variable :each_function
+
+  def run(iterator, *)
+    iterator.each do |item|
+      @each_function.call(item)
+    end
+    nil
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# ForeachPartition
+
+class Spark::Command::ForeachPartition < _Base
+  variable :partition_function
+
+  def run(iterator, *)
+    @partition_function.call(iterator)
+    nil
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# KeyBy
+
+class Spark::Command::KeyBy < _Base
+  variable :key_function
+
+  def run(iterator, *)
+    iterator.map! do |item|
+      [@key_function.call(item), item]
+    end
+    iterator
+  end
+
+  def lazy_run(iterator, *)
+    iterator.map do |item|
+      [@key_function.call(item), item]
+    end
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# Take
+
+class Spark::Command::Take < _Base
+  variable :total,     function: false, type: Numeric
+  variable :last_part, function: false, type: Numeric
+
+  def run(iterator, index)
+    if index == @last_part && iterator.size > @total
+      return iterator.slice!(0, @total)
+    end
+
+    iterator
+  end
+end
+
+# -------------------------------------------------------------------------------------------------
+# Pipe
+
+class Spark::Command::Pipe < _Base
+  variable :cmds, function: false, type: Array
+
+  def before_run
+    require 'open3'
+
+    @in, @out, @threads = Open3.pipeline_rw(*@cmds)
+  end
+
+  def run(iterator, *)
+    create_writing_thread(iterator)
+
+    new_iterator = []
+
+    # Read full input
+    begin
+      loop {
+        new_iterator << @out.readline.rstrip
+      }
+    rescue EOFError
+    end
+
+    new_iterator
+  end
+
+  def lazy_run(iterator, *)
+    create_writing_thread(iterator)
+
+    Enumerator::Lazy.new([nil]) do |yielder, _|
+      begin
+        loop {
+          yielder << @out.readline.rstrip
+        }
+      rescue EOFError
+      end
+    end
+  end
+
+  private
+
+    def create_writing_thread(iterator)
+      @writing_thread = Thread.new do
+        # Send complete iterator to the pipe
+        iterator.each do |item|
+          @in.puts(item.to_s.rstrip)
+        end
+
+        # Input must be closed for EOFError
+        @in.close
+      end
+    end
+
+end