ruby-spark 1.1.0.1 → 1.2.0

data/lib/spark.rb

@@ -1,5 +1,6 @@
 # Gems and libraries
 require 'method_source'
+require 'securerandom'
 require 'forwardable'
 require 'sourcify'
 require 'socket'
@@ -29,6 +30,8 @@ module Spark
 
   include Helper::System
 
+  DEFAULT_CONFIG_FILE = File.join(Dir.home, '.ruby-spark.conf')
+
   def self.print_logo(message=nil)
     puts <<-STRING
 
@@ -107,6 +110,63 @@ module Spark
     !!@context
   end
 
+
+  # ===============================================================================
+  # Defaults
+
+  # Load default configuration for Spark and RubySpark
+  # By default are values stored at ~/.ruby-spark.conf
+  # File is automatically created
+  def self.load_defaults
+    unless File.exists?(DEFAULT_CONFIG_FILE)
+      save_defaults_to(DEFAULT_CONFIG_FILE)
+    end
+
+    load_defaults_from(DEFAULT_CONFIG_FILE)
+  end
+
+  # Clear prev setting and load new from file
+  def self.load_defaults_from(file_path)
+    # Parse values
+    values = File.readlines(file_path)
+    values.map!(&:strip)
+    values.select!{|value| value.start_with?('gem.')}
+    values.map!{|value| value.split(nil, 2)}
+    values = Hash[values]
+
+    # Clear prev values
+    @target_dir = nil
+    @ruby_spark_jar = nil
+    @spark_home = nil
+
+    # Load new
+    @target_dir = values['gem.target']
+  end
+
+  # Create target dir and new config file
+  def self.save_defaults_to(file_path)
+    dir = File.join(Dir.home, ".ruby-spark.#{SecureRandom.uuid}")
+
+    if Dir.exist?(dir)
+      save_defaults_to(file_path)
+    else
+      Dir.mkdir(dir, 0700)
+      file = File.open(file_path, 'w')
+      file.puts "# Directory where will be Spark saved"
+      file.puts "gem.target   #{dir}"
+      file.puts ""
+      file.puts "# You can also defined spark properties"
+      file.puts "# spark.master                       spark://master:7077"
+      file.puts "# spark.ruby.serializer              marshal"
+      file.puts "# spark.ruby.serializer.batch_size   2048"
+      file.close
+    end
+  end
+
+
+  # ===============================================================================
+  # Global settings and variables
+
   def self.logger
     @logger ||= Spark::Logger.new
   end
@@ -116,10 +176,6 @@ module Spark
     @root ||= File.expand_path('..', File.dirname(__FILE__))
   end
 
-  def self.home
-    root
-  end
-
   # Default directory for java extensions
   def self.target_dir
     @target_dir ||= File.join(root, 'target')
@@ -146,17 +202,16 @@ module Spark
   # Cannot load before CLI::install
   #
   # == Parameters:
-  # spark_home::
+  # target::
   #   path to directory where are located sparks .jar files or single Spark jar
   #
-  def self.load_lib(spark_home=nil)
+  def self.load_lib(target=nil)
     return if @java_bridge
 
-    spark_home ||= Spark.target_dir
+    target ||= Spark.target_dir
 
-    bridge = JavaBridge.get
-    @java_bridge = bridge.new(spark_home)
-    @java_bridge.load
+    @java_bridge = JavaBridge.init(target)
+    @java_bridge.import_all
     nil
   end
 
@@ -169,6 +224,7 @@ module Spark
   class << self
     alias_method :sc, :context
     alias_method :jb, :java_bridge
+    alias_method :home, :root
   end
 
 end
@@ -189,6 +245,9 @@ require 'spark/ext/io'
 require 'spark/version'
 require 'spark/error'
 
+# Load default settings for gem and Spark
+Spark.load_defaults
+
 # Make sure that Spark be always stopped
 Kernel.at_exit do
   begin

data/lib/spark/accumulator.rb

@@ -77,6 +77,14 @@ module Spark
       @@instances[@id] = self
     end
 
+    def inspect
+      result  = %{#<#{self.class.name}:0x#{object_id}\n}
+      result << %{   ID: #{@id}\n}
+      result << %{ Zero: #{@zero_value.to_s[0, 10]}\n}
+      result << %{Value: #{@value.to_s[0, 10]}>}
+      result
+    end
+
     def self.changed
       @@changed
     end

data/lib/spark/broadcast.rb

@@ -61,6 +61,13 @@ module Spark
       ObjectSpace.define_finalizer(self, proc { File.unlink(@path) })
     end
 
+    def inspect
+      result  = %{#<#{self.class.name}:0x#{object_id}\n}
+      result << %{   ID: #{@id}\n}
+      result << %{Value: #{@value.to_s[0, 10]}>}
+      result
+    end
+
     def self.register(id, path)
       @@registered[id] = path
     end

data/lib/spark/build.rb

@@ -3,7 +3,7 @@ module Spark
 
     DEFAULT_SCALA_VERSION  = '2.10.4'
     DEFAULT_CORE_VERSION   = '2.10'
-    DEFAULT_SPARK_VERSION  = '1.3.0'
+    DEFAULT_SPARK_VERSION  = '1.4.0'
     DEFAULT_HADOOP_VERSION = '1.0.4'
 
     SBT       = 'sbt/sbt'
@@ -11,20 +11,20 @@ module Spark
     SBT_EXT   = 'package'
     SBT_CLEAN = 'clean'
 
-    def self.build(options)
-      spark_home     = options.spark_home     || Spark.target_dir
-      scala_version  = options.scala_version  || DEFAULT_SCALA_VERSION
-      spark_core     = options.spark_core     || DEFAULT_CORE_VERSION
-      spark_version  = options.spark_version  || DEFAULT_SPARK_VERSION
-      hadoop_version = options.hadoop_version || DEFAULT_HADOOP_VERSION
-      only_ext       = options.only_ext
+    def self.build(options={})
+      scala_version      = options[:scala_version]      || DEFAULT_SCALA_VERSION
+      spark_core_version = options[:spark_core_version] || DEFAULT_CORE_VERSION
+      spark_version      = options[:spark_version]      || DEFAULT_SPARK_VERSION
+      hadoop_version     = options[:hadoop_version]     || DEFAULT_HADOOP_VERSION
+      target             = options[:target]             || Spark.target_dir
+      only_ext           = options[:only_ext]           || false
 
       env = {
         'SCALA_VERSION' => scala_version,
         'SPARK_VERSION' => spark_version,
-        'SPARK_CORE_VERSION' => spark_core,
+        'SPARK_CORE_VERSION' => spark_core_version,
         'HADOOP_VERSION' => hadoop_version,
-        'SPARK_HOME' => spark_home
+        'TARGET_DIR' => target
       }
 
       cmd = [SBT]

data/lib/spark/cli.rb

@@ -13,8 +13,8 @@ module Spark
   class CLI
     include Commander::Methods
 
-    IRB_HISTORY_FILE = File.join(Dir.home, '.irb_spark_history')
-    IRB_HISTORY_SIZE = 100
+    # IRB_HISTORY_FILE = File.join(Dir.home, '.irb_spark_history')
+    # IRB_HISTORY_SIZE = 100
 
     def run
       program :name, 'RubySpark'
@@ -29,22 +29,15 @@ module Spark
       command :build do |c|
         c.syntax = 'build [options]'
         c.description = 'Build spark and gem extensions'
-        c.option '--hadoop-version STRING', String, 'Version of hadoop which will stored with the SPARK'
-        c.option '--spark-home STRING', String, 'Directory where SPARK will be stored'
-        c.option '--spark-core STRING', String, 'Version of SPARK core'
-        c.option '--spark-version STRING', String, 'Version of SPARK'
+        c.option '--hadoop-version STRING', String, 'Version of hadoop which will assembled with the Spark'
+        c.option '--spark-core-version STRING', String, 'Version of Spark core'
+        c.option '--spark-version STRING', String, 'Version of Spark'
         c.option '--scala-version STRING', String, 'Version of Scala'
-        c.option '--only-ext', 'Start SPARK immediately'
+        c.option '--target STRING', String, 'Directory where Spark will be stored'
+        c.option '--only-ext', 'Build only extension for RubySpark'
 
         c.action do |args, options|
-          options.default hadoop_version: Spark::Build::DEFAULT_HADOOP_VERSION,
-                          spark_home: Spark.target_dir,
-                          spark_core: Spark::Build::DEFAULT_CORE_VERSION,
-                          spark_version: Spark::Build::DEFAULT_SPARK_VERSION,
-                          scala_version: Spark::Build::DEFAULT_SCALA_VERSION,
-                          only_ext: false
-
-          Spark::Build.build(options)
+          Spark::Build.build(options.__hash__)
           puts
           puts 'Everything is OK'
         end
@@ -52,23 +45,23 @@ module Spark
       alias_command :install, :build
 
 
-      # Pry -------------------------------------------------------------------
-      command :pry do |c|
-        c.syntax = 'pry [options]'
+      # Shell -----------------------------------------------------------------
+      command :shell do |c|
+        c.syntax = 'shell [options]'
         c.description = 'Start ruby shell for spark'
-        c.option '--spark-home STRING', String, 'Directory where SPARK is stored'
+        c.option '--target STRING', String, 'Directory where Spark is stored'
         c.option '--properties-file STRING', String, 'Path to a file from which to load extra properties'
-        c.option '--[no-]start', 'Start SPARK immediately'
+        c.option '--[no-]start', 'Start Spark immediately'
         c.option '--[no-]logger', 'Enable/disable logger (default: enable)'
 
         c.action do |args, options|
           options.default start: true, logger: true
 
-          Spark.load_lib(options.spark_home)
-          Spark::Logger.disable unless options.logger
+          Spark.load_lib(options.target)
+          Spark.logger.disable unless options.logger
 
           Spark.config do
-            set_app_name 'Pry RubySpark'
+            set_app_name 'RubySpark'
           end
 
           Spark.config.from_file(options.properties_file)
@@ -88,61 +81,60 @@ module Spark
           Pry.start
         end
       end
-      alias_command :shell, :pry
 
 
-      # IRB -------------------------------------------------------------------
-      command :irb do |c|
-        c.syntax = 'irb [options]'
-        c.description = 'Start ruby shell for spark'
-        c.option '--spark-home STRING', String, 'Directory where SPARK is stored'
-        c.option '--[no-]start', 'Start SPARK immediately'
-        c.option '--[no-]logger', 'Enable/disable logger (default: enable)'
-
-        c.action do |args, options|
-          options.default start: true, logger: true
-
-          Spark.load_lib(options.spark_home)
-          Spark::Logger.disable unless options.logger
-
-          Spark.config do
-            set_app_name 'Pry RubySpark'
-          end
-
-          if options.start
-            # Load Java and Spark
-            Spark.start
-            $sc = Spark.context
-
-            Spark.print_logo('Spark context is loaded as $sc')
-          else
-            Spark.print_logo('You can start Spark with Spark.start')
-          end
-
-          # Load IRB
-          require 'irb'
-          require 'irb/completion'
-          require 'irb/ext/save-history'
-
-          begin
-            file = File.expand_path(IRB_HISTORY_FILE)
-            if File.exists?(file)
-              lines = IO.readlines(file).collect { |line| line.chomp }
-              Readline::HISTORY.push(*lines)
-            end
-            Kernel.at_exit do
-              lines = Readline::HISTORY.to_a.reverse.uniq.reverse
-              lines = lines[-IRB_HISTORY_SIZE, IRB_HISTORY_SIZE] if lines.nitems > IRB_HISTORY_SIZE
-              File.open(IRB_HISTORY_FILE, File::WRONLY | File::CREAT | File::TRUNC) { |io| io.puts lines.join("\n") }
-            end
-          rescue
-          end
-
-          ARGV.clear # Clear Thor ARGV, otherwise IRB will parse it
-          ARGV.concat ['--readline', '--prompt-mode', 'simple']
-          IRB.start
-        end
-      end
+      # # IRB -------------------------------------------------------------------
+      # command :irb do |c|
+      #   c.syntax = 'irb [options]'
+      #   c.description = 'Start ruby shell for spark'
+      #   c.option '--spark-home STRING', String, 'Directory where Spark is stored'
+      #   c.option '--[no-]start', 'Start Spark immediately'
+      #   c.option '--[no-]logger', 'Enable/disable logger (default: enable)'
+      #
+      #   c.action do |args, options|
+      #     options.default start: true, logger: true
+      #
+      #     Spark.load_lib(options.spark_home)
+      #     Spark::Logger.disable unless options.logger
+      #
+      #     Spark.config do
+      #       set_app_name 'Pry RubySpark'
+      #     end
+      #
+      #     if options.start
+      #       # Load Java and Spark
+      #       Spark.start
+      #       $sc = Spark.context
+      #
+      #       Spark.print_logo('Spark context is loaded as $sc')
+      #     else
+      #       Spark.print_logo('You can start Spark with Spark.start')
+      #     end
+      #
+      #     # Load IRB
+      #     require 'irb'
+      #     require 'irb/completion'
+      #     require 'irb/ext/save-history'
+      #
+      #     begin
+      #       file = File.expand_path(IRB_HISTORY_FILE)
+      #       if File.exists?(file)
+      #         lines = IO.readlines(file).collect { |line| line.chomp }
+      #         Readline::HISTORY.push(*lines)
+      #       end
+      #       Kernel.at_exit do
+      #         lines = Readline::HISTORY.to_a.reverse.uniq.reverse
+      #         lines = lines[-IRB_HISTORY_SIZE, IRB_HISTORY_SIZE] if lines.nitems > IRB_HISTORY_SIZE
+      #         File.open(IRB_HISTORY_FILE, File::WRONLY | File::CREAT | File::TRUNC) { |io| io.puts lines.join("\n") }
+      #       end
+      #     rescue
+      #     end
+      #
+      #     ARGV.clear # Clear Thor ARGV, otherwise IRB will parse it
+      #     ARGV.concat ['--readline', '--prompt-mode', 'simple']
+      #     IRB.start
+      #   end
+      # end
 
 
       # Home ------------------------------------------------------------------

data/lib/spark/config.rb

@@ -16,6 +16,7 @@ module Spark
     def initialize
       @spark_conf = SparkConf.new(true)
       set_default
+      from_file(Spark::DEFAULT_CONFIG_FILE)
     end
 
     def from_file(file)
@@ -140,11 +141,11 @@ module Spark
       set('spark.ruby.serializer', default_serializer)
       set('spark.ruby.serializer.compress', default_serializer_compress)
       set('spark.ruby.serializer.batch_size', default_serializer_batch_size)
-      set('spark.ruby.executor.uri', default_executor_uri)
       set('spark.ruby.executor.command', default_executor_command)
       set('spark.ruby.executor.options', default_executor_options)
       set('spark.ruby.worker.type', default_worker_type)
       load_executor_envs
+      # set('spark.ruby.executor.install', default_executor_install)
     end
 
     def default_serializer
@@ -159,21 +160,6 @@ module Spark
       ENV['SPARK_RUBY_SERIALIZER_BATCH_SIZE'] || Spark::Serializer::DEFAULT_BATCH_SIZE
     end
 
-    # Ruby executor.
-    #
-    # == Options:
-    # nil::
-    #   System's gem is loaded (ruby-spark).
-    #
-    # other::
-    #   Path of library which will be used.
-    #   Current ruby-spark gem is used.
-    #   (default)
-    #
-    def default_executor_uri
-      ENV['SPARK_RUBY_EXECUTOR_URI'] || ''
-    end
-
     # Command template which is applied when scala want create a ruby
     # process (e.g. master, home request). Command is represented by '%s'.
     #
@@ -186,13 +172,23 @@ module Spark
 
     # Options for every worker.
     #
-    # == Examples:
+    # == Example:
     #   -J-Xmx512m
     #
     def default_executor_options
       ENV['SPARK_RUBY_EXECUTOR_OPTIONS'] || ''
     end
 
+    # # Install command which is triggered before on start.
+    # # This command using executor command template.
+    # #
+    # # == Example:
+    # #   gem install ruby-spark -v 1.2.0
+    # #
+    # def default_executor_install
+    #   ENV['SPARK_RUBY_EXECUTOR_INSTALL'] || ''
+    # end
+
     # Type of worker.
     #
     # == Options: