wukong 0.1.1

data/lib/wukong/script/hadoop_command.rb

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+module Wukong
+  module HadoopCommand
+
+    # ===========================================================================
+    #
+    # Hadoop Options
+    #
+
+    #
+    # Translate the simplified args to their hairy-assed hadoop equivalents
+    #
+    HADOOP_OPTIONS_MAP = {
+      :max_node_map_tasks     => 'mapred.tasktracker.map.tasks.maximum',
+      :max_node_reduce_tasks  => 'mapred.tasktracker.reduce.tasks.maximum',
+      :map_tasks              => 'mapred.map.tasks',
+      :reduce_tasks           => 'mapred.reduce.tasks',
+      :sort_fields            => 'stream.num.map.output.key.fields',
+      :key_field_separator    => 'map.output.key.field.separator',
+      :partition_fields       => 'num.key.fields.for.partition',
+      :output_field_separator => 'stream.map.output.field.separator',
+      :map_speculative        => 'mapred.map.tasks.speculative.execution',
+      :timeout                => 'mapred.task.timeout',
+    }
+
+    # emit a -jobconf hadoop option if the simplified command line arg is present
+    # if not, the resulting nil will be elided later
+    def jobconf option
+      if options[option]
+        "-jobconf %s=%s" % [HADOOP_OPTIONS_MAP[option], options[option]]
+      end
+    end
+
+    # Define what fields hadoop should treat as the keys
+    def hadoop_sort_args
+      [
+        jobconf(:key_field_separator),
+        jobconf(:sort_fields),
+      ]
+    end
+
+    # Define what fields hadoop should use to distribute records to reducers
+    def hadoop_partition_args
+      if options[:partition_fields]
+        [
+          '-partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner',
+          jobconf(:output_field_separator),
+          jobconf(:partition_fields),
+        ]
+      end
+    end
+
+    # Emit options for setting the number of mappers and reducers.
+    def hadoop_num_tasks_args
+      [
+        jobconf(:max_node_map_tasks),
+        jobconf(:max_node_reduce_tasks),
+        jobconf(:map_tasks),
+        jobconf(:reduce_tasks)
+      ]
+    end
+
+    def hadoop_other_args
+      extra_str_args = [ options[:extra_args] ]
+      extra_hsh_args = [:map_speculative, :timeout].map{|opt| jobconf(opt)  }
+      extra_str_args + extra_hsh_args
+    end
+
+    #
+    # Assemble the hadoop command to execute
+    #
+    def hadoop_command input_path, output_path
+      # If this is wrong, create a config/wukong-site.rb or
+      # otherwise set Wukong::CONFIG[:hadoop_home] to the
+      # root of your config install.
+      hadoop_program = Wukong::CONFIG[:hadoop_home]+'/bin/hadoop'
+      [
+        hadoop_program,
+        "jar #{Wukong::CONFIG[:hadoop_home]}/contrib/streaming/hadoop-*-streaming.jar",
+        hadoop_partition_args,
+        hadoop_sort_args,
+        hadoop_num_tasks_args,
+        "-mapper  '#{map_command}'",
+        "-reducer '#{reduce_command}'",
+        "-input   '#{input_path}'",
+        "-output  '#{output_path}'",
+        hadoop_other_args,
+      ].flatten.compact.join(" \t\\\n  ")
+    end
+
+  end
+end
+
+
+# -inputformat     <name of inputformat (class)> (“auto” by default)
+# -input           <additional DFS input path>
+# -python          <python command to use on nodes> (“python” by default)
+# -name            <job name> (“program.py” by default)
+# -numMapTasks     <number>
+# -numReduceTasks  <number> (no sorting or reducing will take place if this is 0)
+# -priority        <priority value> (“NORMAL” by default)
+# -libjar          <path to jar> (this jar gets put in the class path)
+# -libegg          <path to egg> (this egg gets put in the Python path)
+# -file            <local file> (this file will be put in the dir where the python program gets executed)
+# -cacheFile       hdfs://<host>:<fs_port>/<path to file>#<link name> (a link ”<link name>” to the given file will be in the dir)
+# -cacheArchive    hdfs://<host>:<fs_port>/<path to jar>#<link name> (link points to dir that contains files from given jar)
+# -cmdenv          <env var name>=<value>
+# -jobconf         <property name>=<value>
+# -addpath         yes (replace each input key by a tuple consisting of the path of the corresponding input file and the original key)
+# -fake            yes (fake run, only prints the underlying shell commands but does not actually execute them)
+# -memlimit        <number of bytes> (set an upper limit on the amount of memory that can be used)

data/lib/wukong/script/local_command.rb

@@ -0,0 +1,14 @@
+module Wukong
+  module LocalCommand
+
+    # ===========================================================================
+    #
+    # Local execution Options
+    #
+
+    def local_command input_path, output_path
+      %Q{ cat #{input_path} | #{map_command} | sort | #{reduce_command} > '#{output_path}'}
+    end
+
+  end
+end

data/lib/wukong/streamer.rb

@@ -0,0 +1,13 @@
+module Wukong
+  module Streamer
+    autoload :Base,                 'wukong/streamer/base'
+    autoload :LineStreamer,         'wukong/streamer/line_streamer'
+    autoload :StructStreamer,       'wukong/streamer/struct_streamer'
+    #
+    autoload :Filter,               'wukong/streamer/filter'
+    #
+    autoload :AccumulatingReducer,  'wukong/streamer/accumulating_reducer'
+    autoload :ListReducer,          'wukong/streamer/list_reducer'
+    autoload :UniqByLastReducer,    'wukong/streamer/uniq_by_last_reducer'
+  end
+end

data/lib/wukong/streamer/accumulating_reducer.rb

@@ -0,0 +1,89 @@
+ module Wukong
+   module Streamer
+
+     #
+     # AccumulatingReducer makes it easy to apply one operation across all
+     # occurrences of each key
+     #
+     # On each occurrence of a given key, AccumulatingReducer calls
+     # accumulate, and at the final occurrence calls finalize.
+     #
+     # See ListAccumulatingReducer and KeyCountingReducer for examples
+     #
+     # Make sure you don't have the bad luck, bad judgement or bad approach to
+     # accumulate more data than your box can hold before finalizing.
+     #
+     class AccumulatingReducer < Wukong::Streamer::Base
+       attr_accessor :key
+       def initialize options
+         super options
+         self.key = :__first_pass__
+       end
+
+       #
+       # override for multiple-field keys, etc.
+       #
+       # Note that get_key is called by +process+ -- so the arguments have
+       # already been +recordize+d. In particular, if you are using
+       # StructRecordizer (or StructStreamer), you can write this as
+       #
+       #   def get_key(thing) thing.id.to_i ; end
+       #
+       # or whatever
+       def get_key *record
+         record.first
+       end
+
+       #
+       # Accumulate all records for a given key.
+       #
+       # When the last record for the key is seen, finalize processing and adopt the
+       # new key.
+       #
+       def process *args, &block
+         this_key = get_key(*args)
+         if this_key != self.key      # if this is a new key,
+           unless self.key == :__first_pass__
+             finalize(&block)         # process what we've collected so far
+           end
+           self.key = this_key        # adopt the new key
+           start! *args               # and set up for the next accumulation
+         end
+         # collect the current record
+         accumulate *args, &block
+       end
+
+       #
+       # start! is called on the the first record of the new key
+       #
+       def start! *args
+         raise %Q{start! is the new reset! -- it has args now, namely the first
+         record of the new key.  It doesn\'t want #super either}
+       end
+
+       #
+       # Override this to accumulate each record for the given key in turn.
+       #
+       def accumulate *args, &block
+         raise "override the accumulate method in your subclass"
+       end
+
+       #
+       #
+       # You must override this method.
+       #
+       def finalize
+         raise "override the finalize method in your subclass"
+       end
+
+       #
+       # Must make sure to finalize the last-seen accumulation.
+       #
+       def stream
+         super
+         finalize(){|record| emit record }
+       end
+     end
+
+   end
+end

data/lib/wukong/streamer/base.rb

@@ -0,0 +1,76 @@
+module Wukong
+  module Streamer
+    class Base
+
+      # Options, initially set from the command-line args -- see
+      # Script#process_argv!
+      attr_accessor :options
+
+      #
+      # Accepts option hash from script runner
+      #
+      def initialize options={}
+        self.options = options
+      end
+
+      #
+      # Pass each record to +#process+
+      #
+      def stream
+        before_stream
+        $stdin.each do |line|
+          record = recordize(line.chomp)
+          next unless record
+          process(*record) do |output_record|
+            emit output_record
+          end
+        end
+        after_stream
+      end
+
+      # Called exactly once, before streaming begins
+      def before_stream
+      end
+
+      # Called exactly once, after streaming completes
+      def after_stream
+      end
+
+      #
+      # Default recordizer: returns array of fields by splitting at tabs
+      #
+      def recordize line
+        line.split("\t") rescue nil
+      end
+
+      #
+      # Serializes the record to output.
+      #
+      # Emits a single line of tab-separated fields created by calling #to_flat
+      # on the record and joining with "\t".
+      #
+      # Does no escaping or processing of the record -- that's to_flat's job, or
+      # yours if you override this method.
+      #
+      def emit record
+        puts record.to_flat.join("\t")
+      end
+
+      #
+      # Process each record in turn, yielding the records to emit
+      #
+      def process *args, &block
+        raise "override the process method in your implementation: it should process each record."
+      end
+
+      #
+      # To track processing errors inline,
+      # pass the line back to bad_record!
+      #
+      def bad_record! key, *args
+        warn "Bad record #{args.inspect[0..400]}"
+        puts ["bad_record-"+key, *args].join("\t")
+      end
+    end
+  end
+end

data/lib/wukong/streamer/count_keys.rb

@@ -0,0 +1,30 @@
+module Wukong
+  module Streamer
+    #
+    # Emit each unique key and the count of its occurrences
+    #
+    class CountKeys < Wukong::Streamer::AccumulatingReducer
+      attr_accessor :key_count
+
+      def formatted_key_count
+        "%10d"%key_count.to_i
+      end
+
+      # reset the counter to zero
+      def start! *args
+        self.key_count = 0
+      end
+
+      # record one more for this key
+      def accumulate *vals
+        self.key_count += 1
+      end
+
+      # emit each key field and the count, tab-separated.
+      def finalize
+        yield [key, formatted_key_count]
+      end
+    end
+
+  end
+end

data/lib/wukong/streamer/count_lines.rb

@@ -0,0 +1,26 @@
+module Wukong
+  module Streamer
+    #
+    # For each identical line in the map phase output, emit one representative
+    # line followed by the count of occrrences (separated by a tab).
+    #
+    # (This is the functional equivalent of +'uniq -c'+)
+    #
+    class CountLines < Wukong::Streamer::Base
+      def formatted_count item, key_count
+        "%s\t%10d" % [item, key_count.to_i]
+      end
+
+      #
+      # Delegate to +uniq -c+, but put the count last for idempotence.
+      #
+      def stream
+        %x{/usr/bin/uniq -c}.split("\n").each do |line|
+          key_count, item = line.chomp.strip.split(/\s+/, 2)
+          puts formatted_count(item, key_count)
+        end
+      end
+    end
+
+  end
+end

data/lib/wukong/streamer/filter.rb

@@ -0,0 +1,20 @@
+module Wukong
+  module Streamer
+    #
+    # emit only some records, as dictated by the #emit? method
+    #
+    # This is a mixin: including this module in your streamer
+    # implements its +#process+ method.
+    #
+    module Filter
+      #
+      # Filter out a subset of record/lines
+      #
+      # Subclass and re-define the emit? method
+      #
+      def process *record, &block
+        yield record if emit?(record)
+      end
+    end
+  end
+end

data/lib/wukong/streamer/line_streamer.rb

@@ -0,0 +1,12 @@
+module Wukong
+  module Streamer
+    class LineStreamer < Wukong::Streamer::Base
+      #
+      # Turns a flat line into a record for +#process+ to consume
+      #
+      def recordize line
+        [line]
+      end
+    end
+  end
+end

data/lib/wukong/streamer/list_reducer.rb

@@ -0,0 +1,20 @@
+module Wukong
+  module Streamer
+    #
+    # Emit each unique key and the count of its occurrences
+    #
+    class ListReducer < Wukong::Streamer::AccumulatingReducer
+      attr_accessor :values
+
+      # reset the counter to zero
+      def start! *args
+        self.values = []
+      end
+
+      # record one more for this key
+      def accumulate *record
+        self.values << record
+      end
+    end
+  end
+end

data/lib/wukong/streamer/preprocess_with_pipe_streamer.rb

@@ -0,0 +1,22 @@
+module Wukong
+  module Streamer
+    module PreprocessWithPipeStreamer
+      #
+      # Runs STDIN through a shell command and then begins processing.
+      #
+      # If you don't need to do anything to the output of the command, just
+      # inherit from Wukong::Script and override the #map_command.
+      #
+      # You must provide a @preprocess_pipe_command@ method that returns a shell
+      # command to run the input through.
+      #
+      def stream
+        #
+        `#{preprocess_pipe_command}`.each do |line|
+          item = itemize(line) ; next if item.blank?
+          process(*item)
+        end
+      end
+    end
+  end
+end