wukong 3.0.0.pre → 3.0.0.pre2

data/old/wukong/script/avro_command.rb

@@ -1,5 +0,0 @@
-
-
-#
-# Placeholder -- avro will go here
-#

data/old/wukong/script/cassandra_loader_script.rb

@@ -1,40 +0,0 @@
-Settings.define :cassandra_keyspace,   :required => true, :description => "The keyspace to bulk load"
-Settings.define :cassandra_col_family, :required => true, :description => "The column family to bulk load"
-Settings.define :cassandra_home,  :env_var => 'CASSANDRA_HOME', :default => '/usr/local/share/cassandra'
-
-module Wukong
-  class CassandraScript < Wukong::Script
-    def hadoop_other_args *args
-      opts = super(*args)
-      opts << "-D stream.map.output=\'cassandra_avro_output\'"
-      opts << "-D stream.io.identifier.resolver.class=\'org.apache.cassandra.hadoop.streaming.AvroResolver\'"
-      opts << "-D cassandra.output.keyspace=\'#{Settings.cassandra_keyspace}\'"
-      opts << "-D cassandra.output.columnfamily=\'#{Settings.cassandra_col_family}\'"
-      opts << "-D cassandra.partitioner.class=\'org.apache.cassandra.dht.RandomPartitioner\'"
-      opts << "-D cassandra.thrift.address=\'#{[Settings.cassandra_hosts].flatten.map{|s| s.gsub(/:.*/, '')}.join(",")}\'"
-      opts << "-D cassandra.thrift.port=\'9160\'"
-      # opts << "-D mapreduce.output.columnfamilyoutputformat.batch.threshold=\'1024\'"
-      # ORDER MATTERS
-      opts << "-libjars \'#{cassandra_jars}\'"
-      opts << "-file    \'#{avro_schema}\'"
-      opts << "-outputformat \'org.apache.cassandra.hadoop.ColumnFamilyOutputFormat\'"
-      opts
-    end
-
-    #
-    # Return paths to cassandra jars as a string
-    #
-    def cassandra_jars
-      jars = []
-      Dir["#{Settings.cassandra_home}/build/apache-cassandra*.jar", "#{Settings.cassandra_home}/build/lib/jars/*.jar", "#{Settings.cassandra_home}/lib/*.jar"].each do |jar|
-        jars << jar
-      end
-      jars.join(',')
-    end
-
-    def avro_schema
-      File.join(Settings.cassandra_home, "interface/avro/cassandra.avpr")
-    end
-
-  end
-end

data/old/wukong/script/emr_command.rb

@@ -1,168 +0,0 @@
-require 'right_aws'
-require 'configliere/config_block'
-#
-EMR_CONFIG_DIR = '~/.wukong' unless defined?(EMR_CONFIG_DIR)
-#
-Settings.define :emr_credentials_file, :description => 'A .json file holding your AWS access credentials. See http://bit.ly/emr_credentials_file for format'
-Settings.define :access_key,           :description => 'AWS Access key',        :env_var => 'AWS_ACCESS_KEY_ID'
-Settings.define :secret_access_key,    :description => 'AWS Secret Access key', :env_var => 'AWS_SECRET_ACCESS_KEY'
-Settings.define :emr_runner,           :description => 'Path to the elastic-mapreduce command (~ etc will be expanded)'
-Settings.define :emr_root,             :description => 'S3 bucket and path to use as the base for Elastic MapReduce storage, organized by job name'
-Settings.define :emr_data_root,        :description => 'Optional '
-Settings.define :emr_bootstrap_script, :description => 'Bootstrap actions for Elastic Map Reduce machine provisioning', :default => EMR_CONFIG_DIR+'/emr_bootstrap.sh', :type => :filename, :finally => lambda{ Settings.emr_bootstrap_script = File.expand_path(Settings.emr_bootstrap_script) }
-Settings.define :emr_extra_args,       :description => 'kludge: allows you to stuff extra args into the elastic-mapreduce invocation', :type => Array, :wukong => true
-Settings.define :alive,                :description => 'Whether to keep machine running after job invocation', :type => :boolean
-#
-Settings.define :key_pair_file,        :description => 'AWS Key pair file',                               :type => :filename
-Settings.define :key_pair,             :description => "AWS Key pair name. If not specified, it's taken from key_pair_file's basename", :finally => lambda{ Settings.key_pair ||= File.basename(Settings.key_pair_file.to_s, '.pem') if Settings.key_pair_file }
-Settings.define :instance_type,        :description => 'AWS instance type to use',                        :default => 'm1.small'
-Settings.define :master_instance_type, :description => 'Overrides the instance type for the master node', :finally => lambda{ Settings.master_instance_type ||= Settings.instance_type }
-Settings.define :jobflow,              :description => "ID of an existing EMR job flow. Wukong will create a new job flow"
-#
-Settings.read(File.expand_path(EMR_CONFIG_DIR+'/emr.yaml'))
-
-module Wukong
-  #
-  # EMR Options
-  #
-  module EmrCommand
-
-    def execute_emr_workflow
-      copy_script_to_cloud
-      execute_emr_runner
-    end
-
-    def copy_script_to_cloud
-      Log.info "  Copying this script to the cloud."
-      S3Util.store(this_script_filename, mapper_s3_uri)
-      S3Util.store(this_script_filename, reducer_s3_uri)
-      S3Util.store(File.expand_path(Settings.emr_bootstrap_script), bootstrap_s3_uri)
-    end
-
-    def copy_jars_to_cloud
-      S3Util.store(File.expand_path('/tmp/wukong-libs.jar'), wukong_libs_s3_uri)
-      # "--cache-archive=#{wukong_libs_s3_uri}#vendor",
-    end
-
-    def hadoop_options_for_emr_runner
-      [hadoop_jobconf_options, hadoop_other_args].flatten.compact.uniq.map do |hdp_opt|
-        hdp_opt.split(' ').map {|part| "--arg '#{part}'"}
-      end.flatten
-    end
-
-    def execute_emr_runner
-      # fix_paths!
-      command_args = []
-      if Settings.jobflow
-        command_args << Settings.dashed_flag_for(:jobflow)
-      else
-        command_args << "--create --name=#{job_name}"
-        command_args << Settings.dashed_flag_for(:alive)
-        command_args << Settings.dashed_flags(:num_instances, [:instance_type, :slave_instance_type], :master_instance_type, :hadoop_version).join(' ')
-        command_args << Settings.dashed_flags(:availability_zone, :key_pair, :key_pair_file).join(' ')
-        command_args << "--bootstrap-action=#{bootstrap_s3_uri}"
-      end
-      command_args << Settings.dashed_flags(:enable_debugging, :step_action, [:emr_runner_verbose, :verbose], [:emr_runner_debug, :debug]).join(' ')
-      command_args += emr_credentials
-      command_args += [
-        "--log-uri=#{log_s3_uri}",
-        "--stream",
-        "--mapper=#{mapper_s3_uri} ",
-        "--reducer=#{reducer_s3_uri} ",
-        "--input=#{input_paths.join(",")} --output=#{output_path}",
-      ]
-      # eg to specify zero reducers:
-      # Settings[:emr_extra_args] = "--arg '-D mapred.reduce.tasks=0'"
-      command_args += Settings[:emr_extra_args] unless Settings[:emr_extra_args].blank?
-      command_args += hadoop_options_for_emr_runner
-      Log.info 'Follow along at http://localhost:9000/job'
-      execute_command!( File.expand_path(Settings.emr_runner), *command_args )
-    end
-
-    def emr_credentials
-      command_args = []
-      if Settings.emr_credentials_file
-        command_args << "--credentials #{File.expand_path(Settings.emr_credentials_file)}"
-      else
-        command_args << %Q{--access-id #{Settings.access_key} --private-key #{Settings.secret_access_key} }
-      end
-      command_args
-    end
-
-    # A short name for this job
-    def job_handle
-      File.basename($0,'.rb')
-    end
-
-    # Produces an s3 URI within the Wukong emr sandbox from a set of path
-    # segments
-    #
-    # @example
-    #   Settings.emr_root = 's3://emr.yourmom.com/wukong'
-    #   emr_s3_path('log', 'my_happy_job', 'run-97.log')
-    #   # => "s3://emr.yourmom.com/wukong/log/my_happy_job/run-97.log"
-    #
-    def emr_s3_path *path_segs
-      File.join(Settings.emr_root, path_segs.flatten.compact)
-    end
-
-    def mapper_s3_uri
-      emr_s3_path(job_handle, 'code', job_handle+'-mapper.rb')
-    end
-    def reducer_s3_uri
-      emr_s3_path(job_handle, 'code', job_handle+'-reducer.rb')
-    end
-    def log_s3_uri
-      emr_s3_path(job_handle, 'log', 'emr_jobs')
-    end
-    def bootstrap_s3_uri
-      emr_s3_path(job_handle, 'bin', "emr_bootstrap.sh")
-    end
-    def wukong_libs_s3_uri
-      emr_s3_path(job_handle, 'code', "wukong-libs.jar")
-    end
-
-    ABSOLUTE_URI = %r{^/|^\w+://}
-    #
-    # Walk through the input paths and the output path. Prepends
-    # Settings.emr_data_root to any that does NOT look like
-    # an absolute path ("/foo") or a URI ("s3://yourmom/data")
-    #
-    def fix_paths!
-      return if Settings.emr_data_root.blank?
-      unless input_paths.blank?
-        @input_paths = input_paths.map{|path|   (path =~ ABSOLUTE_URI) ? path : File.join(Settings.emr_data_root, path) }
-      end
-      unless output_path.blank?
-        @output_path = [output_path].map{|path| (path =~ ABSOLUTE_URI) ? path : File.join(Settings.emr_data_root, path) }
-      end
-    end
-
-    #
-    # Simple class to coordinate s3 operations
-    #
-    class S3Util
-      # class methods
-      class << self
-        def s3
-          @s3 ||= RightAws::S3Interface.new(
-            Settings.access_key, Settings.secret_access_key,
-            {:multi_thread => true, :logger => Log, :port => 80, :protocol => 'http' })
-        end
-        def bucket_and_path_from_uri uri
-          uri =~ %r{^s3\w*://([\w\.\-]+)\W*(.*)} and return([$1, $2])
-        end
-        def store filename, uri
-          dest_bucket, dest_key = bucket_and_path_from_uri(uri)
-          Log.debug "    #{filename} => #{dest_bucket} / #{dest_key}"
-          contents = File.read(filename)
-          s3.store_object(:bucket => dest_bucket, :key => dest_key, :data => contents)
-        end
-      end
-    end
-
-  end
-  Script.class_eval do
-    include EmrCommand
-  end
-end

data/old/wukong/script/hadoop_command.rb

@@ -1,237 +0,0 @@
-# -*- coding: utf-8 -*-
-module Wukong
-  module HadoopCommand
-
-    # ===========================================================================
-    #
-    # Hadoop Options
-    #
-    Settings.define :hadoop_home, :default => '/usr/lib/hadoop', :description => "Path to hadoop installation; ENV['HADOOP_HOME'] by default. HADOOP_HOME/bin/hadoop is used to run hadoop.", :env_var => 'HADOOP_HOME', :wukong => true
-    Settings.define :hadoop_runner,                            :description => "Path to hadoop script. Usually set --hadoop_home instead of this.", :wukong => true
-
-    #
-    # Translate simplified args to their hairy hadoop equivalents
-    #
-    Settings.define :io_sort_mb,             :jobconf => true, :description => 'io.sort.mb',                                             :wukong => true
-    Settings.define :io_sort_record_percent, :jobconf => true, :description => 'io.sort.record.percent',                                 :wukong => true
-    Settings.define :job_name,               :jobconf => true, :description => 'mapred.job.name',                                        :wukong => true
-    Settings.define :key_field_separator,    :jobconf => true, :description => 'map.output.key.field.separator',                         :wukong => true
-    Settings.define :map_speculative,        :jobconf => true, :description => 'mapred.map.tasks.speculative.execution',                 :wukong => true
-    Settings.define :map_tasks,              :jobconf => true, :description => 'mapred.map.tasks',                                       :wukong => true
-    Settings.define :max_maps_per_cluster,   :jobconf => true, :description => 'mapred.max.maps.per.cluster',                            :wukong => true
-    Settings.define :max_maps_per_node,      :jobconf => true, :description => 'mapred.max.maps.per.node',                               :wukong => true
-    Settings.define :max_node_map_tasks,     :jobconf => true, :description => 'mapred.tasktracker.map.tasks.maximum',                   :wukong => true
-    Settings.define :max_node_reduce_tasks,  :jobconf => true, :description => 'mapred.tasktracker.reduce.tasks.maximum',                :wukong => true
-    Settings.define :max_record_length,      :jobconf => true, :description => 'mapred.linerecordreader.maxlength',                      :wukong => true # "Safeguards against corrupted data: lines longer than this (in bytes) are treated as bad records."
-    Settings.define :max_reduces_per_cluster,:jobconf => true, :description => 'mapred.max.reduces.per.cluster',                         :wukong => true
-    Settings.define :max_reduces_per_node,   :jobconf => true, :description => 'mapred.max.reduces.per.node',                            :wukong => true
-    Settings.define :max_tracker_failures,   :jobconf => true, :description => 'mapred.max.tracker.failures',                            :wukong => true
-    Settings.define :max_map_attempts,       :jobconf => true, :description => 'mapred.map.max.attempts',                                :wukong => true
-    Settings.define :max_reduce_attempts,    :jobconf => true, :description => 'mapred.reduce.max.attempts',                             :wukong => true
-    Settings.define :min_split_size,         :jobconf => true, :description => 'mapred.min.split.size',                                  :wukong => true
-    Settings.define :output_field_separator, :jobconf => true, :description => 'stream.map.output.field.separator',                      :wukong => true
-    Settings.define :partition_fields,       :jobconf => true, :description => 'num.key.fields.for.partition',                           :wukong => true
-    Settings.define :reduce_tasks,           :jobconf => true, :description => 'mapred.reduce.tasks',                                    :wukong => true
-    Settings.define :respect_exit_status,    :jobconf => true, :description => 'stream.non.zero.exit.is.failure',                        :wukong => true
-    Settings.define :reuse_jvms,             :jobconf => true, :description => 'mapred.job.reuse.jvm.num.tasks',                         :wukong => true
-    Settings.define :sort_fields,            :jobconf => true, :description => 'stream.num.map.output.key.fields',                       :wukong => true
-    Settings.define :timeout,                :jobconf => true, :description => 'mapred.task.timeout',                                    :wukong => true
-    Settings.define :noempty,                                  :description => "don't create zero-byte reduce files (hadoop mode only)", :wukong => true
-    Settings.define :split_on_xml_tag,                         :description => "Parse XML document by specifying the tag name: 'anything found between <tag> and </tag> will be treated as one record for map tasks'", :wukong => true
-
-    # emit a -jobconf hadoop option if the simplified command line arg is present
-    # if not, the resulting nil will be elided later
-    def jobconf option
-      if options[option]
-        # "-jobconf %s=%s" % [options.definition_of(option, :description), options[option]]
-        "-D %s=%s" % [options.definition_of(option, :description), options[option]]
-      end
-    end
-
-    #
-    # Assemble the hadoop command to execute
-    # and launch the hadoop runner to execute the script across all tasktrackers
-    #
-    # FIXME: Should add some simple logic to ensure that commands are in the
-    # right order or hadoop will complain. ie. -D options MUST come before
-    # others
-    #
-    def execute_hadoop_workflow
-      # Input paths join by ','
-      input_paths = @input_paths.join(',')
-      #
-      # Use Settings[:hadoop_home] to set the path your config install.
-      hadoop_commandline = [
-        hadoop_runner,
-        "jar #{options[:hadoop_home]}/contrib/streaming/hadoop-*streaming*.jar",
-        hadoop_jobconf_options,
-        "-D mapred.job.name='#{job_name}'",
-        hadoop_other_args,
-        "-mapper  '#{mapper_commandline(:hadoop)}'",
-        "-reducer '#{reducer_commandline(:hadoop)}'",
-        "-input   '#{input_paths}'",
-        "-output  '#{output_path}'",
-        "-file    '#{this_script_filename}'",
-        hadoop_recycle_env,
-      ].flatten.compact.join(" \t\\\n  ")
-      Log.info "  Launching hadoop!"
-      execute_command!(hadoop_commandline)
-    end
-
-    def hadoop_jobconf_options
-      jobconf_options = []
-      # Fixup these options
-      options[:reuse_jvms] = '-1'             if (options[:reuse_jvms] == true)
-      options[:respect_exit_status] = 'false' if (options[:ignore_exit_status] == true)
-      # If no reducer and no reduce_command, then skip the reduce phase
-      options[:reduce_tasks] = 0 if (! reducer) && (! options[:reduce_command]) && (! options[:reduce_tasks])
-      # Fields hadoop should use to distribute records to reducers
-      unless options[:partition_fields].blank?
-        jobconf_options += [
-          jobconf(:partition_fields),
-          jobconf(:output_field_separator),
-        ]
-      end
-      jobconf_options += [
-        :io_sort_mb,               :io_sort_record_percent,
-        :map_speculative,          :map_tasks,
-        :max_maps_per_cluster,     :max_maps_per_node,
-        :max_node_map_tasks,       :max_node_reduce_tasks,
-        :max_reduces_per_cluster,  :max_reduces_per_node,
-        :max_record_length,        :min_split_size,
-        :output_field_separator,   :key_field_separator,
-        :partition_fields,         :sort_fields,
-        :reduce_tasks,             :respect_exit_status,
-        :reuse_jvms,               :timeout,
-        :max_tracker_failures,     :max_map_attempts,
-        :max_reduce_attempts
-      ].map{|opt| jobconf(opt)}
-      jobconf_options.flatten.compact
-    end
-
-    def hadoop_other_args
-      extra_str_args  = [ options[:extra_args] ]
-      if options.split_on_xml_tag
-        extra_str_args << %Q{-inputreader 'StreamXmlRecordReader,begin=<#{options.split_on_xml_tag}>,end=</#{options.split_on_xml_tag}>'}
-      end
-      extra_str_args   << ' -lazyOutput' if options[:noempty]  # don't create reduce file if no records
-      extra_str_args   << ' -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner' unless options[:partition_fields].blank?
-      extra_str_args
-    end
-
-    def hadoop_recycle_env
-      %w[RUBYLIB].map do |var|
-        %Q{-cmdenv '#{var}=#{ENV[var]}'} if ENV[var]
-      end.compact
-    end
-
-    # The path to the hadoop runner script
-    def hadoop_runner
-      options[:hadoop_runner] || (options[:hadoop_home]+'/bin/hadoop')
-    end
-
-    module ClassMethods
-      #
-      # Via @pskomoroch via @tlipcon,
-      #
-      #  "there is a little known Hadoop Streaming trick buried in this Python
-      #   script. You will notice that the date is not actually in the raw log
-      #   data itself, but is part of the filename. It turns out that Hadoop makes
-      #   job parameters you would fetch in Java with something like
-      #   job.get("mapred.input.file") available as environment variables for
-      #   streaming jobs, with periods replaced with underscores:
-      #
-      #     filepath = os.environ["map_input_file"]
-      #     filename = os.path.split(filepath)[-1]
-      #   Thanks to Todd Lipcon for directing me to that hack.
-      #
-
-      # HDFS pathname to the input file currently being processed.
-      def input_file
-        ENV['map_input_file']
-      end
-
-      # Directory of the input file
-      def input_dir
-        ENV['mapred_input_dir']
-      end
-
-      # Offset of this chunk within the input file
-      def map_input_start_offset
-        ENV['map_input_start']
-      end
-
-      # length of the mapper's input chunk
-      def map_input_length
-        ENV['map_input_length']
-      end
-
-      def attempt_id
-        ENV['mapred_task_id']
-      end
-      def curr_task_id
-        ENV['mapred_tip_id']
-      end
-
-      def script_cmdline_urlenc
-        ENV['stream_map_streamprocessor']
-      end
-    end
-
-    # Standard ClassMethods-on-include trick
-    def self.included base
-      base.class_eval do
-        extend ClassMethods
-      end
-    end
-  end
-end
-
-        # -partitioner                          org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
-        # -D mapred.output.key.comparator.class=org.apache.hadoop.mapred.lib.KeyFieldBasedComparator \
-        # -D mapred.text.key.comparator.options=-k2,2nr\
-        # -D mapred.text.key.partitioner.options=-k1,2\
-        # -D mapred.text.key.partitioner.options=\"-k1,$partfields\"
-        # -D stream.num.map.output.key.fields=\"$sortfields\"
-        #
-        # -D stream.map.output.field.separator=\"'/t'\"
-        # -D    map.output.key.field.separator=. \
-        # -D       mapred.data.field.separator=. \
-        # -D map.output.key.value.fields.spec=6,5,1-3:0- \
-        # -D reduce.output.key.value.fields.spec=0-2:5- \
-
-      # "HADOOP_HOME"                             =>"/usr/lib/hadoop-0.20/bin/..",
-      # "HADOOP_IDENT_STRING"                     =>"hadoop",
-      # "HADOOP_LOGFILE"                          =>"hadoop-hadoop-tasktracker-ip-10-242-14-223.log",
-      # "HADOOP_LOG_DIR"                          =>"/usr/lib/hadoop-0.20/bin/../logs",
-      # "HOME"                                    =>"/var/run/hadoop-0.20",
-      # "JAVA_HOME"                               =>"/usr/lib/jvm/java-6-sun",
-      # "LD_LIBRARY_PATH"                         =>"/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386/client:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/../lib/i386:/mnt/hadoop/mapred/local/taskTracker/jobcache/job_200910221152_0023/attempt_200910221152_0023_m_000000_0/work:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386/client:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/../lib/i386",
-      # "PATH"                                    =>"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games",
-      # "USER"                                    =>"hadoop",
-      #
-      # "dfs_block_size"                          =>"134217728",
-      # "map_input_start"                         =>"0",
-      # "map_input_length"                        =>"125726898",
-      # "mapred_output_key_class"                 =>"org.apache.hadoop.io.Text",
-      # "mapred_output_value_class"               =>"org.apache.hadoop.io.Text",
-      # "mapred_output_format_class"              =>"org.apache.hadoop.mapred.TextOutputFormat",
-      # "mapred_output_compression_codec"         =>"org.apache.hadoop.io.compress.DefaultCodec",
-      # "mapred_output_compression_type"          =>"BLOCK",
-      # "mapred_task_partition"                   =>"0",
-      # "mapred_tasktracker_map_tasks_maximum"    =>"4",
-      # "mapred_tasktracker_reduce_tasks_maximum" =>"2",
-      # "mapred_tip_id"                           =>"task_200910221152_0023_m_000000",
-      # "mapred_task_id"                          =>"attempt_200910221152_0023_m_000000_0",
-      # "mapred_job_tracker"                      =>"ec2-174-129-141-78.compute-1.amazonaws.com:8021",
-      #
-      # "mapred_input_dir"                        =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/ripd/com.tw/com.twitter.search/20090809",
-      # "map_input_file"                          =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/ripd/com.tw/com.twitter.search/20090809/com.twitter.search+20090809233441-56735-womper.tsv.bz2",
-      # "mapred_working_dir"                      =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip",
-      # "mapred_work_output_dir"                  =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/tmp/twsearch-20090809/_temporary/_attempt_200910221152_0023_m_000000_0",
-      # "mapred_output_dir"                       =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/tmp/twsearch-20090809",
-      # "mapred_temp_dir"                         =>"/mnt/tmp/hadoop-hadoop/mapred/temp",
-      # "PWD"                                     =>"/mnt/hadoop/mapred/local/taskTracker/jobcache/job_200910221152_0023/attempt_200910221152_0023_m_000000_0/work",
-      # "TMPDIR"                                  =>"/mnt/hadoop/mapred/local/taskTracker/jobcache/job_200910221152_0023/attempt_200910221152_0023_m_000000_0/work/tmp",
-      # "stream_map_streamprocessor"              =>"%2Fusr%2Fbin%2Fruby1.8+%2Fmnt%2Fhome%2Fflip%2Fics%2Fwuclan%2Fexamples%2Ftwitter%2Fparse%2Fparse_twitter_search_requests.rb+--map+--rm",
-      # "user_name"                               =>"flip",