wukong 1.5.0 → 1.5.1

data/examples/emr/elastic_mapreduce_example.rb

@@ -1,17 +1,7 @@
 #!/usr/bin/env ruby
-$stderr.puts `jar xvf *.jar `
-$stderr.puts `tar xvjf *.tar.bz2 `
-$stderr.puts `ls -lR . /mnt/var/lib/hadoop/mapred/taskTracker/archive `
-
-Dir['/mnt/var/lib/hadoop/mapred/taskTracker/archive/**/lib'].each{|dir| $: << dir }
-Dir['./**/lib'].each{|dir| $: << dir }
+Dir[File.dirname(__FILE__)+'/vendor/**/lib'].each{|dir| $: << dir }
 require 'rubygems'
 require 'wukong'
-begin
-  require 'wukong/script/emr_command'
-rescue
-  nil
-end
 
 class FooStreamer < Wukong::Streamer::LineStreamer
   def initialize *args
@@ -20,16 +10,9 @@ class FooStreamer < Wukong::Streamer::LineStreamer
   end
 
   def process *args
-    yield [@line_no, *args]
+    yield ["%5d" % @line_no, *args]
     @line_no += 1
   end
 end
 
-case
-when ($0 =~ /mapper\.rb/) then Settings[:map] = true
-when ($0 =~ /reducer\.rb/) then Settings[:reduce] = true
-end
-
 Wukong::Script.new(FooStreamer, FooStreamer).run
-puts 'done!'
-puts $0

data/lib/wukong/script.rb

@@ -86,14 +86,14 @@ module Wukong
     #   thus, requiring a working hadoop install), or to run in local mode
     #   (script --map | sort | script --reduce)
     #
-    Settings.define :default_run_mode, :default => 'hadoop',          :description => 'Run mode: local, hadoop, emr (elastic mapreduce)', :wukong => true, :hide_help => false
-    Settings.define :map_command,                                     :description => "shell command to run as mapper, in place of this wukong script", :wukong => true
-    Settings.define :reduce_command,                                  :description => "shell command to run as reducer, in place of this wukong script", :wukong => true
-    Settings.define :run,                                             :description => "run the script's workflow: Specify 'hadoop' to use hadoop streaming; 'local' to run your_script.rb --map | sort | your_script.rb --reduce; 'emr' to launch on the amazon cloud.", :wukong => true
-    Settings.define :map,                                             :description => "run the script's map phase. Reads/writes to STDIN/STDOUT.", :wukong => true
-    Settings.define :reduce,                                          :description => "run the script's reduce phase. Reads/writes to STDIN/STDOUT. You can only choose one of --run, --map or --reduce.", :wukong => true
-    Settings.define :dry_run,                                         :description => "echo the command that will be run, but don't run it", :wukong => true
-    Settings.define :rm,                                              :description => "Recursively remove the destination directory. Only used in hadoop mode.", :wukong => true
+    Settings.define :default_run_mode,   :default => 'hadoop', :description => 'Run mode: local, hadoop, emr (elastic mapreduce)', :wukong => true, :hide_help => false
+    Settings.define :map_command,                              :description => "shell command to run as mapper, in place of this wukong script", :wukong => true
+    Settings.define :reduce_command,                           :description => "shell command to run as reducer, in place of this wukong script", :wukong => true
+    Settings.define :run,      :env_var => 'WUKONG_RUN_MODE',    :description => "run the script's workflow: Specify 'hadoop' to use hadoop streaming; 'local' to run your_script.rb --map | sort | your_script.rb --reduce; 'emr' to launch on the amazon cloud; 'map' or 'reduce' to run that phase.", :wukong => true
+    Settings.define :map,                                      :description => "run the script's map phase. Reads/writes to STDIN/STDOUT.", :wukong => true
+    Settings.define :reduce,                                   :description => "run the script's reduce phase. Reads/writes to STDIN/STDOUT. You can only choose one of --run, --map or --reduce.", :wukong => true
+    Settings.define :dry_run,                                  :description => "echo the command that will be run, but don't run it", :wukong => true
+    Settings.define :rm,                                       :description => "Recursively remove the destination directory. Only used in hadoop mode.", :wukong => true
 
     #
     # Instantiate the Script with the Mapper and the Reducer class (each a
@@ -204,7 +204,7 @@ module Wukong
     # use the running framework to relaunch the script in map and in reduce mode
     #
     def execute_command! *args
-      command = args.flatten.compact.join(" \\\n    ")
+      command = args.flatten.reject(&:blank?).join(" \\\n    ")
       Log.info "Running\n\n#{command}\n"
       if options[:dry_run]
         Log.info '== [Not running preceding command: dry run] =='

data/lib/wukong/script/emr_command.rb

@@ -1,13 +1,14 @@
 require 'right_aws'
 require 'configliere/config_block'
 Settings.read(File.expand_path('~/.wukong/emr.yaml'))
-Settings.define :access_key,        :description => 'AWS Access key', :env_var => 'AWS_ACCESS_KEY_ID'
-Settings.define :secret_access_key, :description => 'AWS Secret Access key', :env_var => 'AWS_SECRET_ACCESS_KEY'
-Settings.define :emr_runner,        :description => 'Path to the elastic-mapreduce command (~ etc will be expanded)'
-Settings.define :emr_root,          :description => 'S3 url to use as the base for Elastic MapReduce storage'
-Settings.define :key_pair_file,     :description => 'AWS Key pair file', :finally => lambda{ Settings.key_pair_file = File.expand_path(Settings.key_pair_file.to_s) }
-Settings.define :key_pair,          :description => "AWS Key pair name. If not specified, it's taken from key_pair_file's basename", :finally => lambda{ Settings.key_pair ||= File.basename(Settings.key_pair_file.to_s, '.pem') }
-Settings.define :instance_type,     :description => 'AWS instance type to use', :default => 'm1.small'
+Settings.define :emr_credentials_file, :description => 'A .json file holding your AWS access credentials. See http://bit.ly/emr_credentials_file for format'
+Settings.define :access_key,           :description => 'AWS Access key',        :env_var => 'AWS_ACCESS_KEY_ID'
+Settings.define :secret_access_key,    :description => 'AWS Secret Access key', :env_var => 'AWS_SECRET_ACCESS_KEY'
+Settings.define :emr_runner,           :description => 'Path to the elastic-mapreduce command (~ etc will be expanded)'
+Settings.define :emr_root,             :description => 'S3 url to use as the base for Elastic MapReduce storage'
+Settings.define :key_pair_file,        :description => 'AWS Key pair file', :finally => lambda{ Settings.key_pair_file = File.expand_path(Settings.key_pair_file.to_s) if Settings.key_pair_file }
+Settings.define :key_pair,             :description => "AWS Key pair name. If not specified, it's taken from key_pair_file's basename", :finally => lambda{ Settings.key_pair ||= File.basename(Settings.key_pair_file.to_s, '.pem') if Settings.key_pair_file }
+Settings.define :instance_type,        :description => 'AWS instance type to use', :default => 'm1.small'
 Settings.define :master_instance_type, :description => 'Overrides the instance type for the master node', :finally => lambda{ Settings.master_instance_type ||= Settings.instance_type }
 Settings.define :jobflow
 module Wukong
@@ -26,57 +27,71 @@ module Wukong
       S3Util.store(this_script_filename, mapper_s3_uri)
       S3Util.store(this_script_filename, reducer_s3_uri)
       S3Util.store(File.expand_path('~/ics/wukong/bin/bootstrap.sh'), bootstrap_s3_uri)
-      S3Util.store(File.expand_path('/tmp/wukong-libs.tar.bz2'), wukong_libs_s3_uri)
-      S3Util.store(File.expand_path('/tmp/wukong-libs.jar'), s3_path('bin', "wukong-libs.jar"))
     end
 
     def execute_emr_runner
-      command_args = [
-        :hadoop_version, :availability_zone, :key_pair, :key_pair_file,
-      ].map{|args| Settings.dashed_flag_for(*args) }
+      command_args = []
+      command_args << Settings.dashed_flags(:hadoop_version, :enable_debugging, :step_action, [:emr_runner_verbose, :verbose], [:emr_runner_debug, :debug]).join(' ')
+      command_args += emr_credentials
+      if Settings.jobflow
+        command_args << Settings.dashed_flag_for(:jobflow)
+      else
+        command_args << Settings.dashed_flag_for(:alive)
+        command_args << "--create --name=#{job_name}"
+        command_args << Settings.dashed_flags(:num_instances, [:instance_type, :slave_instance_type], :master_instance_type).join(' ')
+      end
       command_args += [
-        %Q{--verbose --debug --access-id #{Settings.access_key} --private-key #{Settings.secret_access_key} },
-        "--stream",
-        "--mapper=#{mapper_s3_uri}",
-        "--reducer=#{reducer_s3_uri}",
-        "--input=#{mapper_s3_uri} --output=#{Settings.emr_root+'/foo-out.tsv'}",
-        #"--enable-debugging --log-uri=#{log_s3_uri}",
-        "--cache-archive=#{s3_path('bin', "wukong-libs.jar")}#wukong-libs.jar",
-        "--cache=#{wukong_libs_s3_uri}##{File.basename wukong_libs_s3_uri}",
         "--bootstrap-action=#{bootstrap_s3_uri}",
+        "--log-uri=#{log_s3_uri}",
+        "--stream",
+        "--mapper=#{mapper_s3_uri} ",
+        "--reducer=#{reducer_s3_uri} ",
+        "--input=#{input_paths} --output=#{output_path}",
+        # to specify zero reducers:
+        # "--arg '-D mapred.reduce.tasks=0'"
       ]
-      if Settings.jobflow
-        command_args << "--jobflow=#{Settings[:jobflow]}"
-       else
-        command_args << '--alive --create'
-        command_args << "--name=#{job_name}"
-        command_args += [ [:instance_type, :slave_instance_type] , :master_instance_type, :num_instances, ].map{|args| Settings.dashed_flag_for(*args) }
-      end
+      Log.info 'Follow along at http://localhost:9000/job'
       execute_command!( File.expand_path(Settings.emr_runner), *command_args )
     end
 
+    def emr_ship_jars
+      S3Util.store(File.expand_path('/tmp/wukong-libs.jar'), wukong_libs_s3_uri)
+      # "--cache-archive=#{wukong_libs_s3_uri}#vendor",
+    end
+
+    def emr_credentials
+      command_args = []
+      if Settings.emr_credentials_file
+        command_args << "--credentials #{File.expand_path(Settings.emr_credentials_file)}"
+      else
+        command_args << %Q{--access-id #{Settings.access_key} --private-key #{Settings.secret_access_key} }
+      end
+      command_args << Settings.dashed_flags(:availability_zone, :key_pair, :key_pair_file).join(' ')
+      command_args
+    end
+
     # A short name for this job
     def job_handle
       File.basename($0,'.rb')
     end
 
     def mapper_s3_uri
-      s3_path(job_handle+'-mapper.rb')
+      emr_s3_path(job_handle+'-mapper.rb')
     end
     def reducer_s3_uri
-      s3_path(job_handle+'-reducer.rb')
+      emr_s3_path(job_handle+'-reducer.rb')
     end
     def log_s3_uri
-      s3_path('log', job_handle)
+      emr_s3_path('log', job_handle)
     end
     def bootstrap_s3_uri
-      s3_path('bin', "bootstrap-#{job_handle}.sh")
+      emr_s3_path('bin', "bootstrap-#{job_handle}.sh")
     end
     def wukong_libs_s3_uri
-      s3_path('bin', "wukong-libs.tar.bz2")
+      emr_s3_path('bin', "wukong-libs.jar")
     end
 
-    def s3_path *path_segs
+    def emr_s3_path *path_segs
       File.join(Settings.emr_root, path_segs.flatten.compact)
     end
 

data/lib/wukong/streamer/accumulating_reducer.rb

@@ -76,14 +76,10 @@
          raise "override the finalize method in your subclass"
        end
 
-       #
-       # Must make sure to finalize the last-seen accumulation.
-       #
-       def stream
-         super
-         # don't finalize if we never saw any field at all
+       # Finalize the last-seen group.
+       def after_stream *args
          finalize(){|record| emit record } unless (self.key == :__first_pass__)
-         after_stream
+         super *args
        end
      end
 

data/wukong.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{wukong}
-  s.version = "1.5.0"
+  s.version = "1.5.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer"]
-  s.date = %q{2010-08-10}
+  s.date = %q{2010-08-11}
   s.description = %q{  Treat your dataset like a:
 
       * stream of lines when it's efficient to process by lines

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: wukong
 version: !ruby/object:Gem::Version 
-  hash: 3
+  hash: 1
   prerelease: false
   segments: 
   - 1
   - 5
-  - 0
-  version: 1.5.0
+  - 1
+  version: 1.5.1
 platform: ruby
 authors: 
 - Philip (flip) Kromer
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-10 00:00:00 -05:00
+date: 2010-08-11 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency