wukong-hadoop 0.0.1

data/lib/wukong-hadoop/driver.rb

@@ -0,0 +1,190 @@
+require 'shellwords'
+require_relative("driver/inputs_and_outputs")
+require_relative("driver/map_logic")
+require_relative("driver/reduce_logic")
+require_relative("driver/local_invocation")
+require_relative("driver/hadoop_invocation")
+
+module Wukong
+  module Hadoop
+
+    # The <tt>Hadoop::Driver</tt> class contains the logic to examine
+    # arguments and construct command lines which it will execute to
+    # create the desired behavior.
+    #
+    # The Hadoop::Driver will introspect on its arguments to guess (if
+    # not given) the processors to use as mapper and reducer in a
+    # map/reduce job.  It will also decide whether to run that job in
+    # local or Hadoop mode.  These decisions result in a command which
+    # it will ultimately execute.
+    class Driver < Wukong::Driver
+
+      include InputsAndOutputs
+      include MapLogic
+      include ReduceLogic
+      include HadoopInvocation
+      include LocalInvocation
+
+      # The settings used by this driver.
+      #
+      # @param [Configliere::Param]
+      attr_accessor :settings
+
+      # The (processed) arguments for this driver.
+      #
+      # @param [Array<String, Pathname>]
+      attr_reader   :args
+
+      # Initialize and run a new Wukong::Hadoop::Driver for the given
+      # +settings+.
+      #
+      # Will rescue all Wukong::Error exceptions by printing a nice
+      # message to STDERR and exiting.
+      # 
+      # @param [Configliere::Param] settings
+      # @param [Array<String>] extra_args
+      def self.run(settings, *extra_args)
+        begin
+          new(settings, *extra_args).run!
+        rescue Wukong::Error => e
+          $stderr.puts e.message
+          exit(127)
+        end
+      end
+
+      # Run this driver.
+      def run!
+        if mode == :local
+          # Log.info "Launching local!"
+          execute_command!(local_commandline)
+        else
+          ensure_input_and_output!
+          remove_output_path! if settings[:rm] || settings[:overwrite]
+          Log.info "Launching Hadoop!"
+          execute_command!(hadoop_commandline)
+        end
+      end
+
+      # Initialize a new driver with the given +settings+ and +args+.
+      #
+      # @param [Configliere::Param] settings
+      # @param [Array<String>] args
+      def initialize(settings, *args)
+        @settings = settings
+        self.args = args
+      end
+
+      # Set the +args+ for this driver.
+      #
+      # Arguments can be either (registered) processor names or files.
+      #
+      # An error will be raised on missing files or those which
+      # couldn't be loaded.
+      #
+      # An error will be raised if more than two arguments (mapper and
+      # reducer) are passed.
+      #
+      # @param [Array<String>] args
+      def args= args
+        raise Error.new("Cannot provide more than two arguments") if args.length > 2
+        @args = args.map do |arg|
+          if processor_registered?(arg)
+            arg
+          else
+            begin
+              rp = Pathname.new(arg).realpath
+              load rp
+              rp
+            rescue => e
+              raise Error.new("No such processor or file: #{arg}")
+            end
+          end
+        end
+      end
+
+      # What mode is this driver in?
+      #
+      # @return [:hadoop, :local]
+      def mode
+        settings[:mode].to_s == 'local' ? :local : :hadoop
+      end
+
+      # Were mapper and/or reducer named by a single argument?
+      #
+      # @return [true, false]
+      def single_job_arg?
+        args.size == 1
+      end
+
+      # Were mapper and/or reducer named by separate arguments?
+      #
+      # @return [true, false]
+      def separate_map_and_reduce_args?
+        args.size == 2
+      end
+
+      # Is there a processor registered with the given +name+?
+      #
+      # @param [#to_s] name
+      # @return [true, false]
+      def processor_registered? name
+        Wukong.registry.registered?(name.to_s.to_sym)
+      end
+
+      # Return the guessed name of a processor at the given +path+.
+      #
+      # @param [String] path
+      # @return [String]
+      def processor_name_from_file(path)
+        File.basename(path, '.rb')
+      end
+
+      # Does the given +path+ contain a processor named after itself?
+      #
+      # @param [String] path
+      # @return [true, false]
+      def file_is_processor?(path)
+        processor_registered?(processor_name_from_file(path))
+      end
+
+      # The prefix to insert befor all invocations of the
+      # <tt>wu-local</tt> runner.
+      #
+      # @return [String]
+      def command_prefix
+        settings[:command_prefix]
+      end
+
+      # Returns parameters to pass to an invocation of
+      # <tt>wu-local</tt>.
+      #
+      # Parameters like <tt>--reduce_tasks</tt> which are relevant to
+      # Wukong-Hadoop will be interpreted and *not* passed.  Others
+      # will be passed unmodified.
+      #
+      # @return [String]
+      def params_to_pass
+        s = (Wukong.loaded_deploy_pack? ? Deploy.pre_deploy_settings : settings)
+        s.reject{ |param, val| s.definition_of(param, :wukong_hadoop) }.map{ |param,val| "--#{param}=#{Shellwords.escape(val.to_s)}" }.join(" ")
+      end
+
+      # Execute a command composed of the given parts.
+      #
+      # Will print the command instead of the <tt>--dry_run</tt>
+      # option was given.
+      #
+      # @param [Array<String>] args
+      def execute_command!(*args)
+        command = args.flatten.reject(&:blank?).join(" \\\n    ")
+        if settings[:dry_run]
+          Log.info("Dry run:")
+          puts command
+        else
+          puts `#{command}`
+          raise "Streaming command failed!" unless $?.success?
+        end
+      end
+      
+    end
+  end
+end

data/lib/wukong-hadoop/driver/hadoop_invocation.rb

@@ -0,0 +1,184 @@
+module Wukong
+  module Hadoop
+
+    # Provides methods for executing a map/reduce job on a Hadoop
+    # cluster via {Hadoop
+    # streaming}[http://hadoop.apache.org/docs/r0.15.2/streaming.html].
+    module HadoopInvocation
+
+      # Raise an error unless we have input and output.
+      def ensure_input_and_output!
+        raise Error.new("Explicit --input and --output paths are required to run a job in Hadoop mode.") if input_paths.nil? || input_paths.empty? || output_path.nil? || output_path.empty?
+      end
+
+      # Remove the output path.
+      #
+      # Will not actually do anything if the <tt>--dry_run</tt> option
+      # is also given.
+      def remove_output_path!
+        cmd = %Q{#{settings[:hadoop_runner]} fs -rmr '#{output_path}'}
+        Log.info "Removing output file #{output_path}: #{cmd}"
+        puts `#{cmd}` unless settings[:dry_run]
+      end
+
+      # Return the Hadoop command used to launch this job in a Hadoop
+      # cluster.
+      #
+      # You should be able to copy, paste, and run this command
+      # unmodified when debugging.
+      #
+      # @return [String]
+      def hadoop_commandline
+        [
+         hadoop_runner,
+         "jar #{settings[:hadoop_home]}/contrib/streaming/hadoop-*streaming*.jar",
+         hadoop_jobconf_options,
+         "-D mapred.job.name='#{job_name}'",
+         hadoop_other_args,
+         "-mapper       '#{mapper_commandline}'",
+         "-reducer      '#{reducer_commandline}'",
+         "-input        '#{input_paths}'",
+         "-output       '#{output_path}'",
+         hadoop_files,
+         io_formats,
+         hadoop_recycle_env,
+        ].flatten.compact.join(" \t\\\n  ")
+      end
+
+      # The job name that will be passed to Hadoop.
+      #
+      # Respects the <tt>--job_name</tt> option if given, otherwise
+      # constructs one from the given processors, input, and output
+      # paths.
+      #
+      # @return [String]
+      def job_name
+        return settings[:job_name] if settings[:job_name]
+        relevant_filename = args.compact.uniq.map { |path| File.basename(path, '.rb') }.join('-')
+        "#{relevant_filename}---#{input_paths}---#{output_path}".gsub(%r{[^\w/\.\-\+]+}, '')
+      end
+
+      # The input format to use.
+      #
+      # Respects the value of <tt>--input_format</tt>.
+      #
+      # @return [String]
+      def input_format
+        settings[:input_format]
+      end
+
+      # The output format to use.
+      #
+      # Respects the value of <tt>--output_format</tt>.
+      #
+      # @return [String]
+      def output_format
+        settings[:output_format]
+      end
+
+      # :nodoc:
+      def io_formats
+        input  = "-inputformat  '#{input_format}'"  if input_format
+        output = "-outputformat '#{output_format}'" if output_format
+        [input, output]
+      end
+      
+      # The name of the Hadoop binary to use.
+      #
+      # Respects the value of <tt>--hadoop_runner</tt> if given.
+      #
+      # @return [String]
+      def hadoop_runner
+        settings[:hadoop_runner] || File.join(settings[:hadoop_home], 'bin/hadoop')
+      end
+
+      # Return an array of jobconf (-D) options that will be passed to Hadoop.
+      #
+      # Translates the "friendly" <tt>wu-hadoop</tt> names into the
+      # less-friendly Hadoop names.
+      #
+      # @return [Array<String>]
+      def hadoop_jobconf_options
+        jobconf_options = []
+        settings[:reuse_jvms]          = '-1'    if     (settings[:reuse_jvms] == true)
+        settings[:respect_exit_status] = 'false' if     (settings[:ignore_exit_status] == true)
+        # If no reducer and no reduce_command, then skip the reduce phase
+        settings[:reduce_tasks]        = 0       unless (reduce? || settings[:reduce_tasks].nil?)
+        # Fields hadoop should use to distribute records to reducers
+        unless settings[:partition_fields].blank?
+          jobconf_options += [jobconf(:partition_fields), jobconf(:output_field_separator)]
+        end
+        jobconf_options += [
+                            :io_sort_mb,               :io_sort_record_percent,
+                            :map_speculative,          :map_tasks,
+                            :max_maps_per_cluster,     :max_maps_per_node,
+                            :max_node_map_tasks,       :max_node_reduce_tasks,
+                            :max_reduces_per_cluster,  :max_reduces_per_node,
+                            :max_record_length,        :min_split_size,
+                            :output_field_separator,   :key_field_separator,
+                            :partition_fields,         :sort_fields,
+                            :reduce_tasks,             :respect_exit_status,
+                            :reuse_jvms,               :timeout,
+                            :max_tracker_failures,     :max_map_attempts,
+                            :max_reduce_attempts
+                           ].map do |opt|
+          defn = settings.definition_of(opt, :description)
+          val  = settings[opt]
+          java_opt(defn, val)
+        end
+        jobconf_options.flatten.compact
+      end
+
+      # Returns other arguments used by Hadoop streaming.
+      #
+      # @return [String]
+      def hadoop_other_args
+        extra_str_args  = parsed_java_opts
+        if settings[:split_on_xml_tag]
+          extra_str_args << %Q{-inputreader 'StreamXmlRecordReader,begin=<#{options.split_on_xml_tag}>,end=</#{options.split_on_xml_tag}>'}
+        end
+        extra_str_args   << ' -lazyOutput' if settings[:noempty]  # don't create reduce file if no records
+        extra_str_args   << ' -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner' unless settings[:partition_fields].blank?
+        extra_str_args
+      end
+
+      # :nodoc:
+      #
+      # http://hadoop.apache.org/docs/r0.20.2/streaming.html#Package+Files+With+Job+Submissions
+      def hadoop_files
+        args.find_all { |arg| arg.to_s =~ /\.rb$/ }.map do |arg|
+          "-file         '#{arg}'" 
+        end
+      end
+
+      # :nodoc:
+      def ruby_interpreter_path
+        Pathname.new(File.join(Config::CONFIG['bindir'], Config::CONFIG['RUBY_INSTALL_NAME'] + Config::CONFIG['EXEEXT'])).realpath
+      end
+
+      # :nodoc:
+      def use_alternative_gemfile
+        ENV['BUNDLE_GEMFILE'] = settings[:gemfile]
+      end
+
+      # :nodoc:
+      def hadoop_recycle_env
+        use_alternative_gemfile if settings[:gemfile]
+        %w[BUNDLE_GEMFILE].map{ |var| %Q{-cmdenv       '#{var}=#{ENV[var]}'} if ENV[var] }.compact
+      end
+
+      # :nodoc:
+      def parsed_java_opts
+        settings[:java_opts].map do |java_opt|
+          java_opt.split('-D').reject{ |opt| opt.blank? }.map{ |opt| '-D ' + opt.strip }
+        end.flatten
+      end
+
+      # :nodoc:
+      def java_opt option, value
+        "-D %s=%s" % [option, Shellwords.escape(value.to_s)] if value
+      end
+
+    end
+  end
+end

data/lib/wukong-hadoop/driver/inputs_and_outputs.rb

@@ -0,0 +1,27 @@
+module Wukong
+  module Hadoop
+
+    # Provides methods for determining input and output paths.
+    # Written as a separate module to allow easy overriding from other
+    # plugins.
+    module InputsAndOutputs
+
+      # The input paths to read from.
+      #
+      # @return [String]
+      def input_paths
+        (settings[:input] || [])
+      end
+
+      # The output path to write to.
+      #
+      # @return [String]
+      def output_path
+        settings[:output]
+      end
+
+    end
+  end
+end
+
+  

data/lib/wukong-hadoop/driver/local_invocation.rb

@@ -0,0 +1,48 @@
+require 'shellwords'
+module Wukong
+  module Hadoop
+
+    # Provides methods for executing a map/reduce job locally on the
+    # command-line.
+    module LocalInvocation
+
+      # Returns the full local command used by Wukong-Hadoop when
+      # simulating a map/reduce job on the command-line.
+      #
+      # You should be able to run this commmand directly to simulate
+      # the job yourself.
+      #
+      # @return [String]
+      def local_commandline
+        [
+         [cat_input, mapper_commandline].tap do |pipeline|
+           pipeline.concat([sort_commandline, reducer_commandline]) if reduce?
+         end.flatten.compact.join(' | '),
+         cat_output
+        ].flatten.compact.join(' ')
+      end
+
+      # Returns the sort command used by Wukong-Hadoop when simulating
+      # a map/reduce job on the command-line.
+      #
+      # @return [String]
+      def sort_commandline
+        settings[:sort_command]
+      end
+
+      # :nodoc:
+      def cat_input
+        return unless input_paths && (!input_paths.empty?)
+        paths = Shellwords.join(input_paths.split(','))
+        "cat #{paths}"
+      end
+
+      # :nodoc:
+      def cat_output
+        return unless output_path
+        "> #{output_path}"
+      end
+
+    end
+  end
+end