ruby-spark 1.0.0

data/lib/spark/command_validator.rb

@@ -0,0 +1,34 @@
+module Spark
+  module CommandValidator
+
+    def validate(value, options)
+      validate_type(value, options[:type])
+    end
+
+    def valid?(value, options)
+      begin
+        validate(value, options)
+        return true
+      rescue
+        return false
+      end
+    end
+
+    def validate_type(value, types)
+      types = [types] if !types.is_a?(Array)
+
+      types.each do |type|
+        return if value.is_a?(type)
+      end
+
+      error "Value: #{value} should be a #{types.join(' or ')} but is #{value.class}."
+    end
+
+    def validate_size(array1, array2)
+      if array1.size != array2.size
+        error "Wrong number of arguments (#{array1.size} for #{array2.size})"
+      end
+    end
+
+  end
+end

data/lib/spark/config.rb

@@ -0,0 +1,244 @@
+# Necessary libraries
+Spark.load_lib
+
+module Spark
+  # Common configuration for RubySpark and Spark
+  class Config
+
+    include Spark::Helper::System
+
+    TYPES = {
+      'spark.shuffle.spill' => :boolean,
+      'spark.ruby.batch_size' => :integer
+    }
+
+    # Initialize java SparkConf and load default configuration.
+    def initialize
+      @spark_conf = SparkConf.new(true)
+      set_default
+    end
+
+    def from_file(file)
+      check_read_only
+
+      if file && File.exist?(file)
+        file = File.expand_path(file)
+        RubyUtils.loadPropertiesFile(spark_conf, file)
+      end
+    end
+
+    def [](key)
+      get(key)
+    end
+
+    def []=(key, value)
+      set(key, value)
+    end
+
+    def spark_conf
+      if Spark.started?
+        # Get latest configuration
+        Spark.context.jcontext.conf
+      else
+        @spark_conf
+      end
+    end
+
+    def valid!
+      errors = []
+
+      if !contains?('spark.app.name')
+        errors << 'An application name must be set in your configuration.'
+      end
+
+      if !contains?('spark.master')
+        errors << 'A master URL must be set in your configuration.'
+      end
+
+      if Spark::Serializer.get(get('spark.ruby.serializer')).nil?
+        errors << 'Default serializer must be set in your configuration.'
+      end
+
+      scanned = get('spark.ruby.executor.command').scan('%s')
+
+      if scanned.size == 0
+        errors << "Executor command must contain '%s'."
+      end
+
+      if scanned.size > 1
+        errors << "Executor command can contain only one '%s'."
+      end
+
+      if errors.any?
+        errors.map!{|error| "- #{error}"}
+
+        raise Spark::ConfigurationError, "Configuration is not valid:\r\n#{errors.join("\r\n")}"
+      end
+    end
+
+    def read_only?
+      Spark.started?
+    end
+
+    # Rescue from NoSuchElementException
+    def get(key)
+      value = spark_conf.get(key.to_s)
+
+      case TYPES[key]
+      when :boolean
+        parse_boolean(value)
+      when :integer
+        parse_integer(value)
+      else
+        value
+      end
+    rescue
+      nil
+    end
+
+    def get_all
+      Hash[spark_conf.getAll.map{|tuple| [tuple._1, tuple._2]}]
+    end
+
+    def contains?(key)
+      spark_conf.contains(key.to_s)
+    end
+
+    def set(key, value)
+      check_read_only
+      spark_conf.set(key.to_s, value.to_s)
+    end
+
+    def set_app_name(name)
+      set('spark.app.name', name)
+    end
+
+    def set_master(master)
+      set('spark.master', master)
+    end
+
+    def parse_boolean(value)
+      case value
+      when 'true'
+        true
+      when 'false'
+        false
+      end
+    end
+
+    def parse_integer(value)
+      value.to_i
+    end
+
+    # =============================================================================
+    # Defaults
+
+    def set_default
+      set_app_name('RubySpark')
+      set_master('local[*]')
+      set('spark.ruby.driver_home', Spark.home)
+      set('spark.ruby.parallelize_strategy', default_parallelize_strategy)
+      set('spark.ruby.serializer', default_serializer)
+      set('spark.ruby.batch_size', default_batch_size)
+      set('spark.ruby.executor.uri', default_executor_uri)
+      set('spark.ruby.executor.command', default_executor_command)
+      set('spark.ruby.executor.options', default_executor_options)
+      set('spark.ruby.worker.type', default_worker_type)
+      load_executor_envs
+    end
+
+    # How to handle with data in method parallelize.
+    #
+    # == Possible options:
+    # inplace:: data are changed directly to save memory
+    # deep_copy:: data are cloned fist
+    #
+    def default_parallelize_strategy
+      ENV['SPARK_RUBY_PARALLELIZE_STRATEGY'] || 'inplace'
+    end
+
+    def default_serializer
+      ENV['SPARK_RUBY_SERIALIZER'] || Spark::Serializer::DEFAULT_SERIALIZER_NAME
+    end
+
+    def default_batch_size
+      ENV['SPARK_RUBY_BATCH_SIZE'] || Spark::Serializer::DEFAULT_BATCH_SIZE.to_s
+    end
+
+    # Ruby executor.
+    #
+    # == Options:
+    # nil::
+    #   System's gem is loaded (ruby-spark).
+    #
+    # other::
+    #   Path of library which will be used.
+    #   Current ruby-spark gem is used.
+    #   (default)
+    #
+    def default_executor_uri
+      ENV['SPARK_RUBY_EXECUTOR_URI'] || ''
+    end
+
+    # Command template which is applied when scala want create a ruby
+    # process (e.g. master, home request). Command is represented by '%s'.
+    #
+    # == Example:
+    #   bash --norc -i -c "export HOME=/home/user; cd; source .bashrc; %s"
+    #
+    def default_executor_command
+      ENV['SPARK_RUBY_EXECUTOR_COMMAND'] || '%s'
+    end
+
+    # Options for every worker.
+    #
+    # == Examples:
+    #   -J-Xmx512m
+    #
+    def default_executor_options
+      ENV['SPARK_RUBY_EXECUTOR_OPTIONS'] || ''
+    end
+
+    # Type of worker.
+    #
+    # == Options:
+    # process:: (default)
+    # thread:: (experimental)
+    #
+    def default_worker_type
+      ENV['SPARK_RUBY_WORKER_TYPE'] || 'process'
+    end
+
+    # Load environment variables for executor from ENV.
+    #
+    # == Examples:
+    #   SPARK_RUBY_EXECUTOR_ENV_KEY1="1"
+    #   SPARK_RUBY_EXECUTOR_ENV_KEY2="2"
+    #
+    def load_executor_envs
+      prefix = 'SPARK_RUBY_EXECUTOR_ENV_'
+
+      envs = ENV.select{|key, _| key.start_with?(prefix)}
+      envs.each do |key, value|
+        key = key.dup # ENV keys are frozen
+        key.slice!(0, prefix.size)
+
+        set("spark.ruby.executor.env.#{key}", value)
+      end
+    end
+
+    # Aliases
+    alias_method :getAll,     :get_all
+    alias_method :setAppName, :set_app_name
+    alias_method :setMaster,  :set_master
+
+    private
+
+      def check_read_only
+        if read_only?
+          raise Spark::ConfigurationError, 'Configuration is ready only'
+        end
+      end
+
+  end
+end

data/lib/spark/constant.rb

@@ -0,0 +1,14 @@
+module Spark
+  # Commond constant for Ruby and Spark
+  module Constant
+    DATA_EOF = -2
+    WORKER_ERROR = -1
+    WORKER_DONE = 0
+    CREATE_WORKER = 1
+    KILL_WORKER = 2
+    KILL_WORKER_AND_WAIT = 3
+    SUCCESSFULLY_KILLED = 4
+    UNSUCCESSFUL_KILLING = 5
+    ACCUMULATOR_ACK = 6
+  end
+end

data/lib/spark/context.rb

@@ -0,0 +1,304 @@
+# Necessary libraries
+Spark.load_lib
+
+module Spark
+  # Main entry point for Spark functionality. A SparkContext represents the connection to a Spark
+  # cluster, and can be used to create RDDs, accumulators and broadcast variables on that cluster.
+  #
+  class Context
+
+    include Spark::Helper::System
+    include Spark::Helper::Parser
+    include Spark::Helper::Logger
+
+    attr_reader :jcontext, :jaccumulator, :temp_dir
+
+    # Constructor for Ruby context. Configuration is automatically is taken
+    # from Spark. Config will be automatically set to default if user start
+    # context first.
+    #
+    def initialize
+      Spark.config.valid!
+      @jcontext = JavaSparkContext.new(Spark.config.spark_conf)
+      @jcontext.addJar(Spark.ruby_spark_jar)
+
+      # Does not work on 1.2
+      # ui.attachTab(RubyTab.new(ui, to_java_hash(RbConfig::CONFIG)))
+
+      spark_local_dir = JUtils.getLocalDir(sc.conf)
+      @temp_dir = JUtils.createTempDir(spark_local_dir, 'ruby').getAbsolutePath
+
+      accum_server = Spark::Accumulator::Server
+      accum_server.start
+      @jaccumulator = @jcontext.accumulator(ArrayList.new, RubyAccumulatorParam.new(accum_server.host, accum_server.port))
+
+      log_info("Ruby accumulator server is running on port #{accum_server.port}")
+
+      set_call_site('Ruby') # description of stage
+    end
+
+    def stop
+      Spark::Accumulator::Server.stop
+      log_info('Ruby accumulator server was stopped')
+      @jcontext.stop
+    end
+
+    def sc
+      @jcontext.sc
+    end
+
+    def ui
+      sc.ui
+    end
+
+    # Default level of parallelism to use when not given by user (e.g. parallelize and makeRDD)
+    #
+    def default_parallelism
+      sc.defaultParallelism
+    end
+
+    def get_serializer(serializer, *args)
+      serializer   = Spark::Serializer.get(serializer)
+      serializer ||= Spark::Serializer.get(config['spark.ruby.serializer'])
+      serializer.new(config['spark.ruby.batch_size']).set(*args)
+    end
+
+    # Set a local property that affects jobs submitted from this thread, such as the
+    # Spark fair scheduler pool.
+    #
+    def set_local_property(key, value)
+      jcontext.setLocalProperty(key, value)
+    end
+
+    # Get a local property set in this thread, or null if it is missing
+    #
+    def get_local_property(key)
+      jcontext.getLocalProperty(key)
+    end
+
+    # Support function for API backtraces.
+    #
+    def set_call_site(site)
+      set_local_property('externalCallSite', site)
+    end
+
+    # Capture the current user callsite and return a formatted version for printing. If the user
+    # has overridden the call site, this will return the user's version.
+    #
+    def get_call_site
+      jcontext.getCallSite
+    end
+
+    # Return a copy of this SparkContext's configuration. The configuration *cannot*
+    # be changed at runtime.
+    #
+    def config(key=nil)
+      # if key
+      #   Spark.config[key]
+      # else
+      #   Spark.config.get_all
+      # end
+      Spark.config
+    end
+
+    # Add a file to be downloaded with this Spark job on every node.
+    # The path of file passed can be either a local file, a file in HDFS
+    # (or other Hadoop-supported filesystems), or an HTTP, HTTPS or FTP URI.
+    #
+    # To access the file in Spark jobs, use `SparkFiles.get(file_name)` with the
+    # filename to find its download location.
+    #
+    # == Example:
+    #   `echo 10 > test.txt`
+    #
+    #   $sc.add_file('test.txt')
+    #   $sc.parallelize(0..5).map(lambda{|x| x * SparkFiles.get_content('test.txt').to_i}).collect
+    #   # => [0, 10, 20, 30, 40, 50]
+    #
+    def add_file(*files)
+      files.each do |file|
+        sc.addFile(file)
+      end
+    end
+
+    # Broadcast a read-only variable to the cluster, returning a Spark::Broadcast
+    # object for reading it in distributed functions. The variable will
+    # be sent to each cluster only once.
+    #
+    # == Example:
+    #   broadcast1 = $sc.broadcast('a')
+    #   broadcast2 = $sc.broadcast('b')
+    #
+    #   rdd = $sc.parallelize(0..5, 4)
+    #   rdd = rdd.bind(broadcast1: broadcast1, broadcast2: broadcast2)
+    #   rdd = rdd.map_partitions_with_index(lambda{|part, index| [broadcast1.value * index, broadcast2.value * index] })
+    #   rdd.collect
+    #   # => ["", "", "a", "b", "aa", "bb", "aaa", "bbb"]
+    #
+    def broadcast(value)
+      Spark::Broadcast.new(self, value)
+    end
+
+    # Create an Accumulator with the given initial value, using a given
+    # accum_param helper object to define how to add values of the
+    # data type if provided.
+    #
+    # == Example:
+    #   accum = $sc.accumulator(7)
+    #
+    #   rdd = $sc.parallelize(0..5, 4)
+    #   rdd = rdd.bind(accum: accum)
+    #   rdd = rdd.map_partitions(lambda{|_| accum.add(1) })
+    #   rdd = rdd.collect
+    #
+    #   accum.value
+    #   # => 11
+    #
+    def accumulator(value, accum_param=:+, zero_value=0)
+      Spark::Accumulator.new(value, accum_param, zero_value)
+    end
+
+    # Distribute a local Ruby collection to form an RDD
+    # Direct method can be slow so be careful, this method update data inplace
+    #
+    # == Parameters:
+    # data:: Range or Array
+    # num_slices:: number of slice
+    # options::
+    #   - use
+    #   - serializer
+    #   - batch_size
+    #
+    # == Examples:
+    #   $sc.parallelize(["1", "2", "3"]).map(lambda{|x| x.to_i}).collect
+    #   #=> [1, 2, 3]
+    #
+    #   $sc.parallelize(1..3).map(:to_s).collect
+    #   #=> ["1", "2", "3"]
+    #
+    def parallelize(data, num_slices=nil, options={})
+      num_slices ||= default_parallelism
+
+      # use = jruby? ? (options[:use] || :direct) : :file
+      use = :file
+      serializer = get_serializer(options[:serializer], options[:batch_size])
+
+      if data.is_a?(Array) && config['spark.ruby.parallelize_strategy'] == 'deep_copy'
+        data = data.deep_copy
+      else
+        # For enumerator or range
+        data = data.to_a
+      end
+
+      case use
+      when :direct
+        serializer.dump_to_java(data)
+        jrdd = jcontext.parallelize(data, num_slices)
+      when :file
+        file = Tempfile.new('to_parallelize', temp_dir)
+        serializer.dump(data, file)
+        file.close # not unlink
+        jrdd = RubyRDD.readRDDFromFile(jcontext, file.path, num_slices)
+        file.unlink
+      end
+
+      Spark::RDD.new(jrdd, self, serializer)
+    end
+
+    # Read a text file from HDFS, a local file system (available on all nodes), or any
+    # Hadoop-supported file system URI, and return it as an RDD of Strings.
+    #
+    # == Example:
+    #   f = Tempfile.new("test")
+    #   f.puts("1")
+    #   f.puts("2")
+    #   f.close
+    #
+    #   $sc.text_file(f.path).map(lambda{|x| x.to_i}).collect
+    #   # => [1, 2]
+    #
+    def text_file(path, min_partitions=nil, options={})
+      min_partitions ||= default_parallelism
+      serializer = get_serializer(options[:serializer], options[:batch_size])
+
+      Spark::RDD.new(@jcontext.textFile(path, min_partitions), self, serializer, get_serializer('UTF8'))
+    end
+
+    # Read a directory of text files from HDFS, a local file system (available on all nodes), or any
+    # Hadoop-supported file system URI. Each file is read as a single record and returned in a
+    # key-value pair, where the key is the path of each file, the value is the content of each file.
+    #
+    # == Example:
+    #   dir = Dir.mktmpdir
+    #   f1 = Tempfile.new("test1", dir)
+    #   f2 = Tempfile.new("test2", dir)
+    #   f1.puts("1"); f1.puts("2");
+    #   f2.puts("3"); f2.puts("4");
+    #   f1.close
+    #   f2.close
+    #
+    #   $sc.whole_text_files(dir).flat_map(lambda{|key, value| value.split}).collect
+    #   # => ["1", "2", "3", "4"]
+    #
+    def whole_text_files(path, min_partitions=nil, options={})
+      min_partitions ||= default_parallelism
+      serializer = get_serializer(options[:serializer], options[:batch_size])
+      deserializer = get_serializer('Pair', get_serializer('UTF8'), get_serializer('UTF8'))
+
+      Spark::RDD.new(@jcontext.wholeTextFiles(path, min_partitions), self, serializer, deserializer)
+    end
+
+    # Executes the given partition function f on the specified set of partitions,
+    # returning the result as an array of elements.
+    #
+    # If partitions is not specified, this will run over all partitions.
+    #
+    # == Example:
+    #   rdd = $sc.parallelize(0..10, 5, batch_size: 1)
+    #   $sc.run_job(rdd, lambda{|x| x.to_s}, [0,2])
+    #   # => ["[0, 1]", "[4, 5]"]
+    #
+    def run_job(rdd, f, partitions=nil, allow_local=false)
+      run_job_with_command(rdd, partitions, allow_local, Spark::Command::MapPartitions, f)
+    end
+
+    # Execute the given command on specific set of partitions.
+    #
+    def run_job_with_command(rdd, partitions, allow_local, command, *args)
+      if !partitions.nil? && !partitions.is_a?(Array)
+        raise Spark::ContextError, 'Partitions must be nil or Array'
+      end
+
+      partitions_size = rdd.partitions_size
+
+      # Execute all parts
+      if partitions.nil?
+        partitions = (0...partitions_size).to_a
+      end
+
+      # Can happend when you use coalesce
+      partitions.delete_if {|part| part >= partitions_size}
+
+      # Rjb represent Fixnum as Integer but Jruby as Long
+      partitions = to_java_array_list(convert_to_java_int(partitions))
+
+      mapped = rdd.new_rdd_from_command(command, *args)
+      iterator = PythonRDD.runJob(rdd.context.sc, mapped.jrdd, partitions, allow_local)
+      mapped.collect_from_iterator(iterator)
+    end
+
+
+    # Aliases
+    alias_method :textFile, :text_file
+    alias_method :wholeTextFiles, :whole_text_files
+    alias_method :defaultParallelism, :default_parallelism
+    alias_method :setLocalProperty, :set_local_property
+    alias_method :getLocalProperty, :get_local_property
+    alias_method :setCallSite, :set_call_site
+    alias_method :getCallSite, :get_call_site
+    alias_method :runJob, :run_job
+    alias_method :runJobWithCommand, :run_job_with_command
+    alias_method :addFile, :add_file
+
+  end
+end