wukong 0.1.1

data/examples/package-local.rb

@@ -0,0 +1,100 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+
+require 'wukong'
+
+#
+# This is so very very kludgey
+#
+# Input is an 'ls' file, listing files to .bz2 package.
+#
+# Mapper takes each in turn and creates, within a parallel directory tree under
+# ~/pkgd on the HDFS, a .bz2 compressed version of the file.
+#
+# So, the file
+#   /user/me/fixd/all-20090103
+# is packaged onto the DFS as
+#   /user/me/pkgd/user/me/fixd/all-20090103
+#
+#   listing=tmp/fixd-all-package-listing
+#   hdp-rm $listing
+#   hadoop dfs -lsr fixd | egrep '(part-|\.tsv$)' | hdp-put - $listing ;
+#
+#   ./package.rb --run --rm --map_tasks=1 $listing $pkgd_log
+#
+module ExportPackager
+  PKGD_DIR = '/workspace/flip/pkgd'
+
+  #
+  #
+  class Reducer < Wukong::Streamer::Base
+    def announce *args
+      $stdout.puts *args
+      $stderr.puts *args
+    end
+
+    def handle_existing_target output_filename
+      return true unless File.exist?(output_filename)
+      #   announce "Exists! #{output_filename}"
+      #   return false
+      announce "Removing target file #{output_filename}"
+      begin announce `rm #{output_filename}`
+      rescue Exception => e ; announce e ; end
+      true
+    end
+
+    def mkdir_target_safely output_filename
+      output_dir = File.dirname(output_filename)
+      announce "Ensuring directory #{output_dir} exists"
+      begin announce `mkdir -p #{output_dir}`
+      rescue Exception => e ; announce e ; end
+    end
+
+    def bzip_into_pkgd_file input_filename, output_filename
+      announce "bzip'ing into #{output_filename}"
+      announce `( hadoop dfs -cat #{input_filename}/[^_]\** ) | bzip2 -c > #{output_filename}`
+    end
+
+    def gen_output_filename input_filename
+      input_filename += '.tsv' unless input_filename =~ /.*\.\w{2,}/
+      "%s/%s.bz2" % [PKGD_DIR, input_filename.gsub(/^\//, '')]
+    end
+
+    def rsync host, local_path, remote_path=nil
+      remote_path ||= local_path
+      announce `/usr/bin/rsync -Cuvrtlp #{local_path} #{host}:#{remote_path}`
+      sleep 5
+    end
+
+    def process input_filename
+      output_filename = gen_output_filename(input_filename)
+      handle_existing_target(output_filename) or return
+      mkdir_target_safely    output_filename
+      bzip_into_pkgd_file    input_filename, output_filename
+      rsync :lab3, output_filename
+      #
+    end
+
+    def recordize line
+      # handle ls or straight file list, either
+      line.split(/\s/).last
+    end
+
+    def stream
+      super
+      rsync :lab3, PKGD_DIR+'/'
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :map_tasks => 1,
+        :max_node_reduce_tasks => 1, # only one reducer per local filesystem
+        :timeout => 40 * 60 * 1000   # timeout in ms
+    end
+  end
+  # Execute the script
+  Script.new(nil, Reducer).run
+end
+
+

data/examples/package.rb

@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+$: << ENV['WUKONG_PATH'] if ENV['WUKONG_PATH']
+require 'wukong'
+
+#
+# This is so very very kludgey
+#
+# Input is an 'ls' file, listing files to .bz2 package.
+#
+# Reducer takes each in turn and creates, within a parallel directory tree under
+# ~/pkgd on the HDFS, a .bz2 compressed version of the file.
+#
+# So, the file
+#   /user/me/fixd/all-20090103
+# is packaged onto the DFS as
+#   /user/me/pkgd/user/me/fixd/all-20090103
+#
+#   listing=tmp/fixd-all-package-listing
+#   hdp-rm $listing
+#   hadoop dfs -lsr fixd | egrep '(part-|\.tsv$)' | hdp-put - $listing ;
+#
+#   ./package.rb --run --rm --map_tasks=1 $listing $pkgd_log
+#
+module ExportPackager
+  PKGD_DIR = 'pkgd'
+
+  #
+  #
+  class Reducer < Wukong::Streamer::Base
+    def announce str
+      return if str.blank?
+      $stderr.puts str
+      $stdout.puts str
+    end
+
+    def remove_target_filename output_filename
+      begin announce "rm\t#{"%-70s"%output_filename}\t" +
+          `( hadoop dfs -rmr #{output_filename} ) 2>&1`
+      rescue ; nil ; end
+    end
+
+    def mkdir_target_safely output_filename
+      output_dir = File.dirname(output_filename)
+      begin announce "mkdir\t#{"%-70s"%output_dir}\t" +
+          `( hadoop dfs -mkdir #{output_dir} ) 2>&1`
+      rescue ; nil ; end
+    end
+
+    def bzip_into_pkgd_file input_filename, output_filename
+      announce "cat|bz\t#{"%-70s"%input_filename}\t" +
+        `( hadoop dfs -cat #{input_filename}/[^_]\\* | bzip2 -c | hadoop dfs -put - #{output_filename} ) 2>&1`
+    end
+
+    def verify input_filename, output_filename
+      announce "sha1sum\t#{"%-70s"%output_filename}\t" +
+        `( hadoop dfs -cat #{output_filename}        | bzcat - | sha1sum ) 2>&1`
+      announce "sha1sum\t#{"%-70s"%input_filename}\t" +
+        `( hadoop dfs -cat #{input_filename}/[^_]\\*           | sha1sum ) 2>&1`
+    end
+
+    def gen_output_filename input_filename
+      "%s/%s.bz2" % [PKGD_DIR, input_filename.gsub(%r{^/},"")]
+    end
+
+    def process input_filename, output_filename
+      # remove_target_filename output_filename
+      # mkdir_target_safely    output_filename
+      bzip_into_pkgd_file    input_filename, output_filename
+      verify                 input_filename, output_filename
+    end
+
+    def stream
+      announce `hostname`
+      $stdin.each do |input_filename|
+        # handle ls or straight file list, either
+        input_filename = input_filename.chomp.strip.split(/\s/).last
+        output_filename = gen_output_filename input_filename
+        announce "********************************************************"
+        announce "Packing\t#{"%-70s"%input_filename}\t#{output_filename}"
+        process input_filename, output_filename
+        announce "Done\t#{"%-70s"%input_filename}\t#{output_filename}\n\n"
+      end
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :timeout => (24 * 60 * 60 * 1000)  # milliseconds in one day
+    end
+  end
+end
+
+#
+# Execute the script
+#
+ExportPackager::Script.new(nil, ExportPackager::Reducer, :reduce_tasks => 1000).run

data/examples/pagerank/README.textile

@@ -0,0 +1,6 @@
+
+Calculate pagerank for a tab-separated adjacency list.
+
+See 
+* Heretrix pagerank util
+* http://www.umiacs.umd.edu/~jimmylin/cloud9/docs/exercises/pagerank.html

data/examples/pagerank/gen_initial_pagerank_graph.pig

@@ -0,0 +1,57 @@
+
+-- ===========================================================================
+--
+-- Load Graph
+-- 
+AFollowsB               = LOAD    'twnew/all/a_follows_b' AS (rsrc: chararray, user_a_id: int, user_b_id: int) ;
+FollEdges_0             = FOREACH AFollowsB GENERATE user_a_id AS src, user_b_id AS dest ;
+
+InitPagerankFoll_0 	= GROUP FollEdges_0 BY src ;
+InitPagerankFoll_1 	= FOREACH InitPagerankFoll_0 GENERATE
+  group                   AS src,
+  1.0F                    AS pagerank:float,
+  FollEdges_0.(dest)  	  AS dests
+  ;
+rmf                            twnew/pagerank-foll/pagerank_graph_000 ;
+STORE InitPagerankFoll_1 INTO 'twnew/pagerank-foll/pagerank_graph_000';
+
+
+-- MultiEdge               = LOAD    'twnew/all/multi_edge' AS (
+--   rsrc: chararray, src: int, dest: int,
+--   fo:     int, fr:    int,
+--   re_out: int, re_in: int,
+--   at_out: int, at_in: int,
+--   rt_out: int, rt_in: int,
+--   fv_out: int, fv_in: int) ;
+-- 
+-- SymmEdges_0   = FOREACH MultiEdge GENERATE src, dest, fo, fr ;
+-- SymmEdges_1   = FILTER  SymmEdges_0 BY (fo >= 1.0) AND (fr >= 1.0) ;
+-- SymmEdges     = FOREACH SymmEdges_1 GENERATE src, dest ;
+-- -- rm twnew/graphs/symm_edges; STORE SymmEdges INTO 'twnew/graphs/symm_edges' ;
+-- SymmEdges = LOAD 'twnew/graphs/symm_edges' AS (src:int , dest:int);
+-- 
+-- AnyoutEdges_0 = FOREACH MultiEdge GENERATE src, dest, fo, re_out, fv_out ;
+-- AnyoutEdges_1 = FILTER AnyoutEdges_0 BY (fo >= 1.0) OR (re_out >= 1.0) OR (fv_out >= 1.0) ;
+-- AnyoutEdges   = FOREACH AnyoutEdges_1 GENERATE src, dest ;
+-- -- rm twnew/graphs/anyout_edges; STORE AnyoutEdges INTO 'twnew/graphs/anyout_edges' ;
+-- AnyoutEdges = LOAD 'twnew/graphs/anyout_edges' AS (src:int , dest:int);
+-- 
+-- 
+-- InitPagerankSymm_0 = GROUP SymmEdges BY src ;
+-- InitPagerankSymm_1 = FOREACH InitPagerankSymm_0 GENERATE
+--   group                 AS src,
+--   1.0F                  AS pagerank:float,
+--   SymmEdges.(dest)  AS dests
+--   ;
+-- rm                             twnew/pagerank-symm/pagerank_graph_000 ;
+-- STORE InitPagerankSymm_1 INTO 'twnew/pagerank-symm/pagerank_graph_000';
+-- 
+-- 
+-- InitPagerankAnyout_0 = GROUP   AnyoutEdges BY src ;
+-- InitPagerankAnyout_1 = FOREACH InitPagerankAnyout_0 GENERATE
+--   group                 AS src,
+--   1.0F                  AS pagerank:float,
+--   AnyoutEdges.(dest)  AS dests
+--   ;
+-- rm                               twnew/pagerank-anyout/pagerank_graph_000 ;
+-- STORE InitPagerankAnyout_1 INTO 'twnew/pagerank-anyout/pagerank_graph_000';

data/examples/pagerank/pagerank.rb

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong'
+
+#
+#
+#
+
+module PageRank
+  #
+  # Damping factor (prob. of a 'random' jump)
+  # 0.85 works well in practice. See http://en.wikipedia.org/wiki/Pagerank
+  #
+  DAMPING_FACTOR = 0.85
+
+  #
+  # Each user's line looks like
+  #
+  #   user_a    pagerank        id1,id2,...,idN
+  #
+  # we need to disperse this user's pagerank to each of id1..idN, and
+  # rendezvous the list of outbound links at user_a's reducer as well.
+  #
+  module Iterating
+    class Mapper < Wukong::Streamer::Base
+      #
+      # Send pagerank to each page, and send the dests list back to self
+      #
+      def process src, pagerank, dests_str, &block
+        # This lets us use Pig to generate the input
+        dests_str = dests_str.gsub(/[\(\{\}\)]/, '')
+        dests     = dests_str.split(",")
+        yield_pagerank_shares src, pagerank, dests, &block
+        yield_own_dest_list   src, dests_str,       &block
+      end
+
+      #
+      # Take the source node's pagerank and distribute it among all the out-nodes
+      #
+      def yield_pagerank_shares src, pagerank, dests
+        pagerank_share = pagerank.to_f / dests.length
+        dests.each do |dest|
+          yield [dest, 'p', pagerank_share]
+        end
+      end
+
+      #
+      # Dispatch this user's out-node list to rendezvous with itself.
+      #
+      def yield_own_dest_list src, dests_str
+        yield [src, 'd', dests_str]
+      end
+    end
+
+    class Reducer < Wukong::Streamer::AccumulatingReducer
+      attr_accessor :node_id, :pagerank, :dests_str
+      # Begin reduction with 0 accumulated pagerank and no dests as yet
+      def start! node_id, *args
+        self.node_id   = node_id
+        self.pagerank  = 0.0
+        self.dests_str = nil
+      end
+      # We'll receive fractional pagerank from all incoming edges,
+      # and the destination list from this node's map stage
+      def accumulate node_id, what, val
+        case what
+        when 'p' then self.pagerank += val.to_f
+        when 'd' then self.dests_str = val
+        else     raise "Don't know how to accumulate #{[node_id, what, val].inspect}"
+        end
+      end
+      # To finalize, dump the damped pagerank and dest list
+      # in a form that can be fed back into this script
+      def finalize
+        damped_pagerank = (self.pagerank * DAMPING_FACTOR) + (1 - DAMPING_FACTOR)
+        self.dests_str = 'dummy' if self.dests_str.blank?
+        yield [node_id, damped_pagerank, dests_str]
+      end
+    end
+
+    class Script < Wukong::Script
+      def default_options
+        super.merge :extra_args => ' -jobconf io.sort.record.percent=0.25 '
+      end
+    end
+    Script.new(Mapper, Reducer).run
+  end
+end

data/examples/pagerank/pagerank_initialize.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong'
+require 'wukong/streamer/set_reducer'
+
+module PageRank
+  class Script < Wukong::Script
+    #
+    # Input format is
+    #
+    #   rsrc    src_id  dest_id  [... junk ...]
+    #
+    # All we want from the line are its src and dest IDs.
+    #
+    def map_command
+      %Q{/usr/bin/cut -d"\t" -f2,3}
+    end
+
+    def default_options
+      super.merge :extra_args => ' -jobconf io.sort.record.percent=0.25 '
+    end
+  end
+
+  #
+  # Accumulate the dests list in memory, dump as a whole. Multiple edges between
+  # any two nodes are permitted, and will accumulate pagerank according to the
+  # edge's multiplicity.
+  #
+  class Reducer < Wukong::Streamer::ListReducer
+    def accumulate src, dest
+      self.values << dest
+    end
+
+    # Emit src, initial pagerank, and flattened dests list
+    def finalize
+      self.values = ['dummy'] if self.values.blank?
+      yield [key, 1.0, self.values.to_a.join(",")]
+    end
+  end
+
+  # Execute the script
+  Script.new(nil, PageRank::Reducer).run
+end
+
+
+

data/examples/pagerank/run_pagerank.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Directory to pagerank on.
+work_dir=$1 ; shift
+if [ "$work_dir" == '' ] ; then echo "Please specify the parent of the directory made by gen_initial_pagerank" ; exit ; fi
+
+
+# How many rounds to run
+max_iter=10
+# this directory
+script_dir="`dirname $0`"
+
+for (( curr=0 , next=1 ; "$curr" < "$max_iter" ; curr++ , next++ )) ; do
+  curr_str=`printf "%03d" ${curr}`
+  next_str=`printf "%03d" ${next}`
+  curr_dir=$work_dir/pagerank_graph_${curr_str}
+  next_dir=$work_dir/pagerank_graph_${next_str}
+  $script_dir/pagerank.rb --rm --run $curr_dir $next_dir
+done

data/examples/rank_and_bin.rb

@@ -0,0 +1,173 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+require 'wukong'
+require 'wukong/streamer/rank_and_bin_reducer'
+
+#
+# This example uses the classes from http://github.com/mrflip/twitter_friends
+# (That's sloppy, and I apologize. I'm building this script for that, but it
+# seems broadly useful and I'm not maintaining two copies. Once this script is
+# more worky we'll make it standalone.  Anyway you should get the picture.)
+#
+$: << File.dirname(__FILE__)+'/../../projects/twitter_friends/lib'
+require 'twitter_friends';
+require 'twitter_friends/struct_model' ; include TwitterFriends::StructModel
+
+
+#
+# attrs to bin
+#
+BINNABLE_ATTRS = {
+  :twitter_user => [
+    [:followers_count,  :fo   ],
+    [:friends_count,    :fr   ],
+    [:statuses_count,   :st   ],
+    [:favourites_count, :fv   ],
+    [:created_at,       :crat ]
+    ]
+
+}
+RESOURCE_ALIASES = {
+  :twitter_user  => :u,
+  :user_metrics  => :um,
+}
+#
+# KLUDGE This is not DRY at all but  let's get it working first
+#
+BinUserMetrics = TypedStruct.new(
+  [:id, Integer],
+  *BINNABLE_ATTRS[:user_metrics].map{|attr, attr_abbr| [attr_abbr, Integer] }
+  )
+BINNED_RESOURCE_ALIASES = {
+  :u   => BinTwitterUser,
+}
+
+module RankAndBinAttrs
+  class ExplodeResourceMapper < Wukong::Streamer::StructStreamer
+    def get_and_format_attr thing, attr
+      val = thing.send(attr)
+      case thing.members_types[attr].to_s.to_sym
+      when :Integer     then "%010d"   % val.to_i
+      when :Float       then "%020.7f" % val.to_f
+      when :Bignum      then "%020d"   % val.to_i
+      else
+        raise [val, thing.members_types[attr].to_s.to_sym].inspect
+      end
+    end
+
+    #
+    # The data expansion of this mapper is large enough that it makes sense to
+    # be a little responsible with what we emit.  We'll use the RESOURCE_ALIASES
+    # and BINNABLE_ATTRS hashes, above, to dump a more parsimonious
+    # representation.
+    #
+    def process thing, *args, &block
+      attr_abbrs = BINNABLE_ATTRS[thing.class.resource_name]
+      return unless attr_abbrs
+      attr_abbrs.each do |attr, abbr|
+        yield [
+          RESOURCE_ALIASES[thing.class.resource_name],
+          abbr,
+          get_and_format_attr(thing, attr),
+          thing.id.to_i
+        ]
+      end
+    end
+  end
+
+  class BinAttrReducer < Wukong::Streamer::RankAndBinReducer
+    attr_accessor :last_rsrc_attr
+    #
+    # Note that we might get several different resources at the same reducer
+    #
+    def get_key rsrc, attr, val, *args
+      if [rsrc, attr] != self.last_rsrc_attr
+        # Note: since each partition has the same cardinality, we don't need to
+        # fiddle around with the bin_size, etc -- just reset the order
+        # parameters' state.
+        reset_order_params!
+        self.last_rsrc_attr = [rsrc, attr]
+      end
+      val
+    end
+
+    #
+    # Note well -- we are rearranging the field order to
+    #
+    #   resource_abbr id  attr_abbr  bin
+    #
+    # for proper sorting to the re-assembler
+    #
+    def emit record
+      rsrc, attr, val, id, numbering, rank, bin = record
+      super [rsrc, id, attr, bin]
+    end
+  end
+
+  class ReassembleObjectReducer < Wukong::Streamer::AccumulatingReducer
+    attr_accessor :thing
+    def klass_from_abbr rsrc_abbr
+      BINNED_RESOURCE_ALIASES[rsrc_abbr.to_sym]
+    end
+    def get_key rsrc_abbr, id, *args
+      [rsrc_abbr, id.to_i]
+    end
+
+    def start! rsrc_abbr, id, *args
+      klass = klass_from_abbr(rsrc_abbr)
+      self.thing = klass.new id.to_i
+    end
+
+    def accumulate rsrc, id, attr, bin
+      thing.send("#{attr}=", bin)
+    end
+
+    def finalize
+      yield thing
+    end
+  end
+
+  #
+  # Two-phase script
+  #
+  # FIXME -- We need a runner class to manage this.
+  #
+  class Script < Wukong::Script
+    attr_accessor :phase
+    # KLUDGE !!
+    def initialize
+      case
+      when ARGV.detect{|arg| arg =~ /--phase=1/}
+        # Phase 1 -- Steal underpants.  Also, disassemble each object, and find
+        # the bin for each binnable attribute's value
+        self.phase = 1
+        self.mapper_klass, self.reducer_klass = [ExplodeResourceMapper, BinAttrReducer]
+      when ARGV.detect{|arg| arg =~ /--phase=2/}
+        # Phase 2 -- ????
+        raise "Phase 2 : ????"
+      when ARGV.detect{|arg| arg =~ /--phase=3/}
+        # Phase 3 -- profit. In this case, put records back together.
+        self.phase = 3
+        self.mapper_klass, self.reducer_klass = [nil, ReassembleObjectReducer]
+      else
+        raise "Please run me with a --phase= option"
+      end
+      super mapper_klass, reducer_klass
+    end
+
+    def default_options
+      extra_options =
+        case self.phase
+        # partition on [rsrc, attr]; sort on [rsrc, attr, val]
+        when 1  then { :sort_fields => 3, :partition_fields => 2 }
+        # sort on [rsrc, id]
+        when 3  then { :sort_fields => 2 }
+        else { }
+        end
+      super.merge extra_options
+    end
+  end
+
+  # execute script
+  Script.new.run
+end