ruby-spark 1.0.0 → 1.1.0.1

data/lib/spark/command/base.rb

@@ -16,6 +16,10 @@ class Spark::Command::Base
     end
   end
 
+  def to_s
+    self.class.name.split('::').last
+  end
+
   def self.error(message)
     raise Spark::CommandError, message
   end

data/lib/spark/command_builder.rb

@@ -32,8 +32,8 @@ module Spark
     def deep_copy
       copy = self.dup
       copy.create_command
-      copy.serializer    = self.serializer.dup
-      copy.deserializer  = self.deserializer.dup
+      copy.serializer    = self.serializer.deep_copy
+      copy.deserializer  = self.deserializer.deep_copy
       copy.commands      = self.commands.dup
       copy.libraries     = self.libraries.dup
       copy.bound_objects = self.bound_objects.dup

data/lib/spark/config.rb

@@ -9,7 +9,7 @@ module Spark
 
     TYPES = {
       'spark.shuffle.spill' => :boolean,
-      'spark.ruby.batch_size' => :integer
+      'spark.ruby.serializer.compress' => :boolean
     }
 
     # Initialize java SparkConf and load default configuration.
@@ -55,8 +55,8 @@ module Spark
         errors << 'A master URL must be set in your configuration.'
       end
 
-      if Spark::Serializer.get(get('spark.ruby.serializer')).nil?
-        errors << 'Default serializer must be set in your configuration.'
+      if Spark::Serializer.find(get('spark.ruby.serializer')).nil?
+        errors << 'Unknow serializer.'
       end
 
       scanned = get('spark.ruby.executor.command').scan('%s')
@@ -137,9 +137,9 @@ module Spark
       set_app_name('RubySpark')
       set_master('local[*]')
       set('spark.ruby.driver_home', Spark.home)
-      set('spark.ruby.parallelize_strategy', default_parallelize_strategy)
       set('spark.ruby.serializer', default_serializer)
-      set('spark.ruby.batch_size', default_batch_size)
+      set('spark.ruby.serializer.compress', default_serializer_compress)
+      set('spark.ruby.serializer.batch_size', default_serializer_batch_size)
       set('spark.ruby.executor.uri', default_executor_uri)
       set('spark.ruby.executor.command', default_executor_command)
       set('spark.ruby.executor.options', default_executor_options)
@@ -147,22 +147,16 @@ module Spark
       load_executor_envs
     end
 
-    # How to handle with data in method parallelize.
-    #
-    # == Possible options:
-    # inplace:: data are changed directly to save memory
-    # deep_copy:: data are cloned fist
-    #
-    def default_parallelize_strategy
-      ENV['SPARK_RUBY_PARALLELIZE_STRATEGY'] || 'inplace'
-    end
-
     def default_serializer
       ENV['SPARK_RUBY_SERIALIZER'] || Spark::Serializer::DEFAULT_SERIALIZER_NAME
     end
 
-    def default_batch_size
-      ENV['SPARK_RUBY_BATCH_SIZE'] || Spark::Serializer::DEFAULT_BATCH_SIZE.to_s
+    def default_serializer_compress
+      ENV['SPARK_RUBY_SERIALIZER_COMPRESS'] || Spark::Serializer::DEFAULT_COMPRESS
+    end
+
+    def default_serializer_batch_size
+      ENV['SPARK_RUBY_SERIALIZER_BATCH_SIZE'] || Spark::Serializer::DEFAULT_BATCH_SIZE
     end
 
     # Ruby executor.

data/lib/spark/context.rb

@@ -2,6 +2,7 @@
 Spark.load_lib
 
 module Spark
+  ##
   # Main entry point for Spark functionality. A SparkContext represents the connection to a Spark
   # cluster, and can be used to create RDDs, accumulators and broadcast variables on that cluster.
   #
@@ -57,10 +58,38 @@ module Spark
       sc.defaultParallelism
     end
 
-    def get_serializer(serializer, *args)
-      serializer   = Spark::Serializer.get(serializer)
-      serializer ||= Spark::Serializer.get(config['spark.ruby.serializer'])
-      serializer.new(config['spark.ruby.batch_size']).set(*args)
+    # Default serializer
+    #
+    # Batch -> Compress -> Basic
+    #
+    def default_serializer
+      # Basic
+      serializer = Spark::Serializer.find!(config('spark.ruby.serializer')).new
+
+      # Compress
+      if config('spark.ruby.serializer.compress')
+        serializer = Spark::Serializer.compressed(serializer)
+      end
+
+      # Bactching
+      batch_size = default_batch_size
+      if batch_size == 'auto'
+        serializer = Spark::Serializer.auto_batched(serializer)
+      else
+        serializer = Spark::Serializer.batched(serializer, batch_size)
+      end
+
+      # Finally, "container" contains serializers
+      serializer
+    end
+
+    def default_batch_size
+      size = config('spark.ruby.serializer.batch_size').to_i
+      if size >= 1
+        size
+      else
+        'auto'
+      end
     end
 
     # Set a local property that affects jobs submitted from this thread, such as the
@@ -93,12 +122,11 @@ module Spark
     # be changed at runtime.
     #
     def config(key=nil)
-      # if key
-      #   Spark.config[key]
-      # else
-      #   Spark.config.get_all
-      # end
-      Spark.config
+      if key
+        Spark.config.get(key)
+      else
+        Spark.config
+      end
     end
 
     # Add a file to be downloaded with this Spark job on every node.
@@ -164,10 +192,7 @@ module Spark
     # == Parameters:
     # data:: Range or Array
     # num_slices:: number of slice
-    # options::
-    #   - use
-    #   - serializer
-    #   - batch_size
+    # serializer:: custom serializer (default: serializer based on configuration)
     #
     # == Examples:
     #   $sc.parallelize(["1", "2", "3"]).map(lambda{|x| x.to_i}).collect
@@ -176,33 +201,21 @@ module Spark
     #   $sc.parallelize(1..3).map(:to_s).collect
     #   #=> ["1", "2", "3"]
     #
-    def parallelize(data, num_slices=nil, options={})
+    def parallelize(data, num_slices=nil, serializer=nil)
       num_slices ||= default_parallelism
+      serializer ||= default_serializer
 
-      # use = jruby? ? (options[:use] || :direct) : :file
-      use = :file
-      serializer = get_serializer(options[:serializer], options[:batch_size])
-
-      if data.is_a?(Array) && config['spark.ruby.parallelize_strategy'] == 'deep_copy'
-        data = data.deep_copy
-      else
-        # For enumerator or range
-        data = data.to_a
-      end
+      serializer.check_each(data)
 
-      case use
-      when :direct
-        serializer.dump_to_java(data)
-        jrdd = jcontext.parallelize(data, num_slices)
-      when :file
-        file = Tempfile.new('to_parallelize', temp_dir)
-        serializer.dump(data, file)
-        file.close # not unlink
-        jrdd = RubyRDD.readRDDFromFile(jcontext, file.path, num_slices)
-        file.unlink
-      end
+      # Through file
+      file = Tempfile.new('to_parallelize', temp_dir)
+      serializer.dump_to_io(data, file)
+      file.close # not unlink
+      jrdd = RubyRDD.readRDDFromFile(jcontext, file.path, num_slices)
 
       Spark::RDD.new(jrdd, self, serializer)
+    ensure
+      file && file.unlink
     end
 
     # Read a text file from HDFS, a local file system (available on all nodes), or any
@@ -217,11 +230,12 @@ module Spark
     #   $sc.text_file(f.path).map(lambda{|x| x.to_i}).collect
     #   # => [1, 2]
     #
-    def text_file(path, min_partitions=nil, options={})
+    def text_file(path, min_partitions=nil, encoding=Encoding::UTF_8, serializer=nil)
       min_partitions ||= default_parallelism
-      serializer = get_serializer(options[:serializer], options[:batch_size])
+      serializer     ||= default_serializer
+      deserializer     = Spark::Serializer.build { __text__(encoding) }
 
-      Spark::RDD.new(@jcontext.textFile(path, min_partitions), self, serializer, get_serializer('UTF8'))
+      Spark::RDD.new(@jcontext.textFile(path, min_partitions), self, serializer, deserializer)
     end
 
     # Read a directory of text files from HDFS, a local file system (available on all nodes), or any
@@ -240,10 +254,10 @@ module Spark
     #   $sc.whole_text_files(dir).flat_map(lambda{|key, value| value.split}).collect
     #   # => ["1", "2", "3", "4"]
     #
-    def whole_text_files(path, min_partitions=nil, options={})
+    def whole_text_files(path, min_partitions=nil, serializer=nil)
       min_partitions ||= default_parallelism
-      serializer = get_serializer(options[:serializer], options[:batch_size])
-      deserializer = get_serializer('Pair', get_serializer('UTF8'), get_serializer('UTF8'))
+      serializer     ||= default_serializer
+      deserializer     = Spark::Serializer.build{ __pair__(__text__, __text__) }
 
       Spark::RDD.new(@jcontext.wholeTextFiles(path, min_partitions), self, serializer, deserializer)
     end
@@ -254,7 +268,7 @@ module Spark
     # If partitions is not specified, this will run over all partitions.
     #
     # == Example:
-    #   rdd = $sc.parallelize(0..10, 5, batch_size: 1)
+    #   rdd = $sc.parallelize(0..10, 5)
     #   $sc.run_job(rdd, lambda{|x| x.to_s}, [0,2])
     #   # => ["[0, 1]", "[4, 5]"]
     #
@@ -282,9 +296,13 @@ module Spark
       # Rjb represent Fixnum as Integer but Jruby as Long
       partitions = to_java_array_list(convert_to_java_int(partitions))
 
+      # File for result
+      file = Tempfile.new('collect', temp_dir)
+
       mapped = rdd.new_rdd_from_command(command, *args)
-      iterator = PythonRDD.runJob(rdd.context.sc, mapped.jrdd, partitions, allow_local)
-      mapped.collect_from_iterator(iterator)
+      RubyRDD.runJob(rdd.context.sc, mapped.jrdd, partitions, allow_local, file.path)
+
+      mapped.collect_from_file(file)
     end
 
 

data/lib/spark/ext/io.rb

@@ -12,6 +12,12 @@ module Spark
           unpack_int(read(4))
         end
 
+        def read_int_or_eof
+          bytes = read(4)
+          return Spark::Constant::DATA_EOF if bytes.nil?
+          unpack_int(bytes)
+        end
+
         def read_long
           unpack_long(read(8))
         end
@@ -35,8 +41,11 @@ module Spark
           write(pack_long(data))
         end
 
+        # Size and data can have different encoding
+        # Marshal: both ASCII
+        # Oj: ASCII and UTF-8
         def write_string(data)
-          write_int(data.size)
+          write_int(data.bytesize)
           write(data)
         end
 
@@ -55,3 +64,4 @@ module Spark
 end
 
 IO.__send__(:include, Spark::CoreExtension::IO)
+StringIO.__send__(:include, Spark::CoreExtension::IO)

data/lib/spark/java_bridge/base.rb

@@ -145,8 +145,8 @@ module Spark
             if class_name == 'JavaRDD'
               jrdd = RubyRDD.toRuby(object)
 
-              serializer   = Spark.sc.get_serializer('marshal', nil)
-              deserializer = Spark.sc.get_serializer('marshal', 2) # is fully batched
+              serializer = Spark::Serializer.build { __batched__(__marshal__) }
+              serializer = Spark::Serializer.build { __batched__(__marshal__, 2) }
 
               return Spark::RDD.new(jrdd, Spark.sc, serializer, deserializer)
             end

data/lib/spark/rdd.rb

@@ -34,6 +34,18 @@ module Spark
       @command = Spark::CommandBuilder.new(serializer, deserializer)
     end
 
+    def inspect
+      comms = @command.commands.join(' -> ')
+
+      result  = %{#<#{self.class.name}:0x#{object_id}}
+      result << %{ (#{comms})} unless comms.empty?
+      result << %{\n}
+      result << %{  Serializer: "#{serializer}"\n}
+      result << %{Deserializer: "#{deserializer}"}
+      result << %{>}
+      result
+    end
+
 
     # =============================================================================
     # Operators
@@ -159,7 +171,16 @@ module Spark
     end
 
     def to_java
-      rdd = self.reserialize('Marshal')
+      marshal = Spark::Serializer.marshal
+
+      if deserializer.batched?
+        ser = deserializer.deep_copy
+        ser.serializer = marshal
+      else
+        ser = Spark::Serializer.batched(marshal)
+      end
+
+      rdd = self.reserialize(ser)
       RubyRDD.toJava(rdd.jrdd, rdd.serializer.batched?)
     end
 
@@ -169,20 +190,32 @@ module Spark
 
     # Return an array that contains all of the elements in this RDD.
     # RJB raise an error if stage is killed.
-    def collect
-      collect_from_iterator(jrdd.collect.iterator)
+    def collect(as_enum=false)
+      file = Tempfile.new('collect', context.temp_dir)
+
+      RubyRDD.writeRDDToFile(jrdd.rdd, file.path)
+
+      collect_from_file(file, as_enum)
     rescue => e
       raise Spark::RDDError, e.message
     end
 
-    def collect_from_iterator(iterator)
+    def collect_from_file(file, as_enum=false)
       if self.is_a?(PipelinedRDD)
         klass = @command.serializer
       else
         klass = @command.deserializer
       end
 
-      klass.load_from_iterator(iterator)
+      if as_enum
+        result = klass.load_from_file(file)
+      else
+        result = klass.load_from_io(file).to_a
+        file.close
+        file.unlink
+      end
+
+      result
     end
 
     # Convert an Array to Hash
@@ -198,7 +231,7 @@ module Spark
     # to satisfy the limit.
     #
     # == Example:
-    #   rdd = $sc.parallelize(0..100, 20, batch_size: 1)
+    #   rdd = $sc.parallelize(0..100, 20)
     #   rdd.take(5)
     #   # => [0, 1, 2, 3, 4]
     #
@@ -293,7 +326,7 @@ module Spark
     #   seq = lambda{|x,y| x+y}
     #   com = lambda{|x,y| x*y}
     #
-    #   rdd = $sc.parallelize(1..10, 2, batch_size: 1)
+    #   rdd = $sc.parallelize(1..10, 2)
     #   rdd.aggregate(1, seq, com)
     #   # => 656
     #
@@ -590,7 +623,7 @@ module Spark
     # of the original partition.
     #
     # == Example:
-    #   rdd = $sc.parallelize(0...4, 4, batch_size: 1)
+    #   rdd = $sc.parallelize(0...4, 4)
     #   rdd.map_partitions_with_index(lambda{|part, index| part.first * index}).collect
     #   # => [0, 1, 4, 9]
     #
@@ -623,7 +656,7 @@ module Spark
     # Return an RDD created by coalescing all elements within each partition into an array.
     #
     # == Example:
-    #   rdd = $sc.parallelize(0..10, 3, batch_size: 1)
+    #   rdd = $sc.parallelize(0..10, 3)
     #   rdd.glom.collect
     #   # => [[0, 1, 2], [3, 4, 5, 6], [7, 8, 9, 10]]
     #
@@ -639,8 +672,14 @@ module Spark
     #   # => [[0, 1, 2], [3, 4, 5, 6, 7, 8, 9, 10]]
     #
     def coalesce(num_partitions)
+      if self.is_a?(PipelinedRDD)
+        deser = @command.serializer
+      else
+        deser = @command.deserializer
+      end
+
       new_jrdd = jrdd.coalesce(num_partitions)
-      RDD.new(new_jrdd, context, @command.serializer, @command.deserializer)
+      RDD.new(new_jrdd, context, @command.serializer, deser)
     end
 
     # Return the Cartesian product of this RDD and another one, that is, the
@@ -655,7 +694,8 @@ module Spark
     #   # => [[1, 4], [1, 5], [1, 6], [2, 4], [2, 5], [2, 6], [3, 4], [3, 5], [3, 6]]
     #
     def cartesian(other)
-      _deserializer = Spark::Serializer::Cartesian.new.set(self.deserializer, other.deserializer)
+      _deserializer = Spark::Serializer::Cartesian.new(self.deserializer, other.deserializer)
+
       new_jrdd = jrdd.cartesian(other.jrdd)
       RDD.new(new_jrdd, context, serializer, _deserializer)
     end
@@ -697,7 +737,7 @@ module Spark
     #
     def union(other)
       if self.serializer != other.serializer
-        other = other.reserialize(serializer.name, serializer.batch_size)
+        other = other.reserialize(serializer)
       end
 
       new_jrdd = jrdd.union(other.jrdd)
@@ -713,10 +753,7 @@ module Spark
     #   rdd.reserialize("oj").collect
     #   # => ["1", "2", "3"]
     #
-    def reserialize(new_serializer, new_batch_size=nil)
-      new_batch_size ||= deserializer.batch_size
-      new_serializer = Spark::Serializer.get!(new_serializer).new(new_batch_size)
-
+    def reserialize(new_serializer)
       if serializer == new_serializer
         return self
       end
@@ -906,7 +943,7 @@ module Spark
     #     x+y
     #   end
     #
-    #   rdd = $sc.parallelize(["a","b","c","a","b","c","a","c"], 2, batch_size: 1).map(lambda{|x| [x, 1]})
+    #   rdd = $sc.parallelize(["a","b","c","a","b","c","a","c"], 2).map(lambda{|x| [x, 1]})
     #   rdd.combine_by_key(method(:combiner), method(:merge), method(:merge)).collect_as_hash
     #   # => {"a"=>3, "b"=>2, "c"=>3}
     #
@@ -973,7 +1010,7 @@ module Spark
     #     x*y
     #   end
     #
-    #   rdd = $sc.parallelize([["a", 1], ["b", 2], ["a", 3], ["a", 4], ["c", 5]], 2, batch_size: 1)
+    #   rdd = $sc.parallelize([["a", 1], ["b", 2], ["a", 3], ["a", 4], ["c", 5]], 2)
     #   rdd.aggregate_by_key(1, method(:combine), method(:merge))
     #   # => [["b", 3], ["a", 16], ["c", 6]]
     #
@@ -1064,6 +1101,17 @@ module Spark
       self.sort_by('lambda{|(key, _)| key}')
     end
 
+    # Sort the RDD by value
+    #
+    # == Example:
+    #   rdd = $sc.parallelize([["a", 3], ["b", 1], ["c", 2]])
+    #   rdd.sort_by_value.collect
+    #   # => [["b", 1], ["c", 2], ["a", 3]]
+    #
+    def sort_by_value(ascending=true, num_partitions=nil)
+      self.sort_by('lambda{|(_, value)| value}')
+    end
+
     # Sorts this RDD by the given key_function
     #
     # This is a different implementation than spark. Sort by doesn't use
@@ -1190,6 +1238,7 @@ module Spark
     alias_method :defaultReducePartitions, :default_reduce_partitions
     alias_method :setName, :set_name
     alias_method :addLibrary, :add_library
+    alias_method :require, :add_library
 
     alias_method :flatMap, :flat_map
     alias_method :mapPartitions, :map_partitions