ruby-spark 1.0.0

data/lib/spark/error.rb

@@ -0,0 +1,50 @@
+module Spark
+  # Extension cannot be built
+  class BuildError < StandardError
+  end
+
+  # Proc.to_source
+  # Java object cannot be converted
+  class SerializeError < StandardError
+  end
+
+  # Serializer method
+  # Non-existing serializer
+  class NotImplemented < StandardError
+  end
+
+  # Missison app_name or master
+  class ConfigurationError < StandardError
+  end
+
+  # Wrong parameters
+  class RDDError < StandardError
+  end
+
+  # Validations
+  class CommandError < StandardError
+  end
+
+  # Parser helper
+  class ParseError < StandardError
+  end
+
+  # Validation in context
+  class ContextError < StandardError
+  end
+
+  # Broadcasts
+  # Missing path
+  class BroadcastError < StandardError
+  end
+
+  # Accumulators
+  # Existing keys
+  # Wrong ID
+  class AccumulatorError < StandardError
+  end
+
+  # Wrong instances
+  class MllibError < StandardError
+  end
+end

data/lib/spark/ext/hash.rb

@@ -0,0 +1,41 @@
+module Spark
+  module CoreExtension
+    module Hash
+      module ClassMethods
+      end
+
+      module InstanceMethods
+        # Destructively convert all keys to strings.
+        def stringify_keys_with_spark!
+          transform_keys!{ |key| key.to_s }
+        end
+
+        # Destructively convert all keys to symbols, as long as they respond
+        def symbolize_keys_with_spark!
+          transform_keys!{ |key| key.to_sym rescue key }
+        end
+
+        # Destructively convert all keys using the block operations.
+        # Same as transform_keys but modifies +self+.
+        def transform_keys_with_spark!
+          keys.each do |key|
+            self[yield(key)] = delete(key)
+          end
+          self
+        end
+      end
+
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.send(:include, InstanceMethods)
+        base.class_eval do
+          patch_unless_exist :stringify_keys!, :spark
+          patch_unless_exist :symbolize_keys!, :spark
+          patch_unless_exist :transform_keys!, :spark
+        end
+      end
+    end
+  end
+end
+
+Hash.__send__(:include, Spark::CoreExtension::Hash)

data/lib/spark/ext/integer.rb

@@ -0,0 +1,25 @@
+module Spark
+  module CoreExtension
+    module Integer
+      module ClassMethods
+      end
+
+      module InstanceMethods
+      end
+
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.send(:include, InstanceMethods)
+        base.class_eval do
+          const_set :MAX_WITH_SPARK, 1 << (1.size * 8 - 2) - 1
+          const_set :MIN_WITH_SPARK, -const_get(:MAX_WITH_SPARK) - 1
+
+          path_const_unless_exist :MAX, :SPARK
+          path_const_unless_exist :MIN, :SPARK
+        end
+      end
+    end
+  end
+end
+
+Integer.__send__(:include, Spark::CoreExtension::Integer)

data/lib/spark/ext/io.rb

@@ -0,0 +1,57 @@
+module Spark
+  module CoreExtension
+    module IO
+      module ClassMethods
+      end
+
+      module InstanceMethods
+
+        # Reading
+
+        def read_int
+          unpack_int(read(4))
+        end
+
+        def read_long
+          unpack_long(read(8))
+        end
+
+        def read_string
+          read(read_int)
+        end
+
+        def read_data
+          Marshal.load(read_string)
+        end
+
+
+        # Writing
+
+        def write_int(data)
+          write(pack_int(data))
+        end
+
+        def write_long(data)
+          write(pack_long(data))
+        end
+
+        def write_string(data)
+          write_int(data.size)
+          write(data)
+        end
+
+        def write_data(data)
+          write_string(Marshal.dump(data))
+        end
+      end
+
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.send(:include, Spark::Helper::Serialize)
+        base.send(:include, InstanceMethods)
+      end
+    end
+  end
+end
+
+IO.__send__(:include, Spark::CoreExtension::IO)

data/lib/spark/ext/ip_socket.rb

@@ -0,0 +1,29 @@
+module Spark
+  module CoreExtension
+    module IPSocket
+      module ClassMethods
+      end
+
+      module InstanceMethods
+        def port
+          addr[1]
+        end
+
+        def hostname
+          addr(true)[2]
+        end
+
+        def numeric_address
+          addr[3]
+        end
+      end
+
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.send(:include, InstanceMethods)
+      end
+    end
+  end
+end
+
+IPSocket.__send__(:include, Spark::CoreExtension::IPSocket)

data/lib/spark/ext/module.rb

@@ -0,0 +1,58 @@
+module Spark
+  module CoreExtension
+    module Module
+
+      # Patch method to class unless already exist
+      #
+      # == Example:
+      #
+      #   class Hash
+      #     def a
+      #       1
+      #     end
+      #   end
+      #
+      #   module HashExtension
+      #     module InstanceMethods
+      #       def a_with_spark
+      #         2
+      #       end
+      #
+      #       def b_with_spark
+      #         1
+      #       end
+      #     end
+      #
+      #     def self.included(base)
+      #       base.send(:include, InstanceMethods)
+      #       base.class_eval do
+      #         patch_unless_exist :a, :spark
+      #         patch_unless_exist :b, :spark
+      #       end
+      #     end
+      #   end
+      #
+      #   Hash.include(HashExtension)
+      #
+      #   Hash.new.a # => 1
+      #   Hash.new.b # => 1
+      #
+      def patch_unless_exist(target, suffix)
+        unless method_defined?(target)
+          aliased_target, punctuation = target.to_s.sub(/([?!=])$/, ''), $1
+
+          alias_method target, "#{aliased_target}_with_#{suffix}#{punctuation}"
+        end
+      end
+
+      def path_const_unless_exist(target, suffix)
+        unless const_defined?(target)
+          const_set(target, const_get("#{target}_WITH_#{suffix}"))
+        end
+      end
+
+    end
+  end
+end
+
+Module.__send__(:include, Spark::CoreExtension::Module)

data/lib/spark/ext/object.rb

@@ -0,0 +1,24 @@
+module Spark
+  module CoreExtension
+    module Object
+      module ClassMethods
+      end
+
+      module InstanceMethods
+        def deep_copy_with_spark
+          Marshal.load(Marshal.dump(self))
+        end
+      end
+
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.send(:include, InstanceMethods)
+        base.class_eval do
+          patch_unless_exist :deep_copy, :spark
+        end
+      end
+    end
+  end
+end
+
+Object.__send__(:include, Spark::CoreExtension::Object)

data/lib/spark/ext/string.rb

@@ -0,0 +1,24 @@
+module Spark
+  module CoreExtension
+    module String
+      module ClassMethods
+      end
+
+      module InstanceMethods
+        def camelize_with_spark
+          self.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+        end
+      end
+
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.send(:include, InstanceMethods)
+        base.class_eval do
+          patch_unless_exist :camelize, :spark
+        end
+      end
+    end
+  end
+end
+
+String.__send__(:include, Spark::CoreExtension::String)

data/lib/spark/helper.rb

@@ -0,0 +1,10 @@
+module Spark
+  module Helper
+    autoload :System,    "spark/helper/system"
+    autoload :Logger,    "spark/helper/logger"
+    autoload :Statistic, "spark/helper/statistic"
+    autoload :Serialize, "spark/helper/serialize"
+    autoload :Partition, "spark/helper/partition"
+    autoload :Parser,    "spark/helper/parser"
+  end
+end

data/lib/spark/helper/logger.rb

@@ -0,0 +1,40 @@
+module Spark
+  module Helper
+    module Logger
+
+      def self.included(base)
+        base.send :extend,  Methods
+        base.send :include, Methods
+      end
+
+      module Methods
+        def log_info(message)
+          Spark.logger.info(message)
+        end
+
+        def log_debug(message)
+          Spark.logger.debug(message)
+        end
+
+        def log_trace(message)
+          Spark.logger.trace(message)
+        end
+
+        def log_warning(message)
+          Spark.logger.warning(message)
+        end
+
+        def log_error(message)
+          Spark.logger.error(message)
+        end
+
+        alias_method :logInfo,    :log_info
+        alias_method :logDebug,   :log_debug
+        alias_method :logTrace,   :log_trace
+        alias_method :logWarning, :log_warning
+        alias_method :logError,   :log_error
+
+      end # Methods
+    end # Logger
+  end # Helper
+end # Spark

data/lib/spark/helper/parser.rb

@@ -0,0 +1,85 @@
+module Spark
+  module Helper
+    module Parser
+      
+      def self.included(base)
+        base.send :extend,  Methods
+        base.send :include, Methods
+      end
+     
+      module Methods
+        def to_java_hash(hash)
+          hash_map = HashMap.new
+          hash.each_pair do |key, value|
+            begin
+              # RJB raise Object is NULL (but new record is put correctly)
+              hash_map.put(key, value)
+            rescue RuntimeError
+            end
+          end
+          hash_map
+        end
+
+        def convert_to_java_int(data)
+          if data.is_a?(Array)
+            data.map{|x| JInteger.new(x)}
+          else
+            JInteger.new(data)
+          end
+        end
+
+        def to_java_array_list(array)
+          array_list = ArrayList.new
+          array.each do |item|
+            array_list.add(item)
+          end
+          array_list
+        end
+
+        # Parse and convert memory size. Shifting be better but Float doesn't support it.
+        #
+        # == Examples:
+        #   to_memory_size("512mb")
+        #   # => 524288
+        #
+        #   to_memory_size("512 MB")
+        #   # => 524288
+        #
+        #   to_memory_size("512mb", "GB")
+        #   # => 0.5
+        #
+        def to_memory_size(memory, result_unit="KB")
+          match = memory.match(/([\d]+)[\s]*([\w]*)/)
+          if match.nil?
+            raise Spark::ParseError, "Memory has wrong format. Use: 'SIZE UNIT'"
+          end
+
+          size = match[1].to_f
+          unit = match[2]
+
+          size *= memory_multiplier_based_kb(unit)
+          size /= memory_multiplier_based_kb(result_unit)
+          size.round(2)
+        end
+
+        # Based to KB
+        def memory_multiplier_based_kb(type)
+          case type.to_s.upcase
+          when "G", "GB"
+            1048576
+          when "M", "MB"
+            1024
+          when "K", "KB"
+            1
+          else
+            raise Spark::ParseError, "Unsupported type #{type}"
+          end
+        end
+
+      end # Methods
+
+    end # Parser
+  end # Helper
+end # Spark
+
+