mrflip-wukong 0.1.0

data/examples/pagerank/README.textile

@@ -0,0 +1,6 @@
+
+Calculate pagerank for a tab-separated adjacency list.
+
+See 
+* Heretrix pagerank util
+* http://www.umiacs.umd.edu/~jimmylin/cloud9/docs/exercises/pagerank.html

data/examples/pagerank/gen_initial_pagerank_graph.pig

@@ -0,0 +1,57 @@
+
+-- ===========================================================================
+--
+-- Load Graph
+-- 
+AFollowsB               = LOAD    'twnew/all/a_follows_b' AS (rsrc: chararray, user_a_id: int, user_b_id: int) ;
+FollEdges_0             = FOREACH AFollowsB GENERATE user_a_id AS src, user_b_id AS dest ;
+
+InitPagerankFoll_0 	= GROUP FollEdges_0 BY src ;
+InitPagerankFoll_1 	= FOREACH InitPagerankFoll_0 GENERATE
+  group                   AS src,
+  1.0F                    AS pagerank:float,
+  FollEdges_0.(dest)  	  AS dests
+  ;
+rmf                            twnew/pagerank-foll/pagerank_graph_000 ;
+STORE InitPagerankFoll_1 INTO 'twnew/pagerank-foll/pagerank_graph_000';
+
+
+-- MultiEdge               = LOAD    'twnew/all/multi_edge' AS (
+--   rsrc: chararray, src: int, dest: int,
+--   fo:     int, fr:    int,
+--   re_out: int, re_in: int,
+--   at_out: int, at_in: int,
+--   rt_out: int, rt_in: int,
+--   fv_out: int, fv_in: int) ;
+-- 
+-- SymmEdges_0   = FOREACH MultiEdge GENERATE src, dest, fo, fr ;
+-- SymmEdges_1   = FILTER  SymmEdges_0 BY (fo >= 1.0) AND (fr >= 1.0) ;
+-- SymmEdges     = FOREACH SymmEdges_1 GENERATE src, dest ;
+-- -- rm twnew/graphs/symm_edges; STORE SymmEdges INTO 'twnew/graphs/symm_edges' ;
+-- SymmEdges = LOAD 'twnew/graphs/symm_edges' AS (src:int , dest:int);
+-- 
+-- AnyoutEdges_0 = FOREACH MultiEdge GENERATE src, dest, fo, re_out, fv_out ;
+-- AnyoutEdges_1 = FILTER AnyoutEdges_0 BY (fo >= 1.0) OR (re_out >= 1.0) OR (fv_out >= 1.0) ;
+-- AnyoutEdges   = FOREACH AnyoutEdges_1 GENERATE src, dest ;
+-- -- rm twnew/graphs/anyout_edges; STORE AnyoutEdges INTO 'twnew/graphs/anyout_edges' ;
+-- AnyoutEdges = LOAD 'twnew/graphs/anyout_edges' AS (src:int , dest:int);
+-- 
+-- 
+-- InitPagerankSymm_0 = GROUP SymmEdges BY src ;
+-- InitPagerankSymm_1 = FOREACH InitPagerankSymm_0 GENERATE
+--   group                 AS src,
+--   1.0F                  AS pagerank:float,
+--   SymmEdges.(dest)  AS dests
+--   ;
+-- rm                             twnew/pagerank-symm/pagerank_graph_000 ;
+-- STORE InitPagerankSymm_1 INTO 'twnew/pagerank-symm/pagerank_graph_000';
+-- 
+-- 
+-- InitPagerankAnyout_0 = GROUP   AnyoutEdges BY src ;
+-- InitPagerankAnyout_1 = FOREACH InitPagerankAnyout_0 GENERATE
+--   group                 AS src,
+--   1.0F                  AS pagerank:float,
+--   AnyoutEdges.(dest)  AS dests
+--   ;
+-- rm                               twnew/pagerank-anyout/pagerank_graph_000 ;
+-- STORE InitPagerankAnyout_1 INTO 'twnew/pagerank-anyout/pagerank_graph_000';

data/examples/pagerank/pagerank.rb

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong'
+
+#
+#
+#
+
+module PageRank
+  #
+  # Damping factor (prob. of a 'random' jump)
+  # 0.85 works well in practice. See http://en.wikipedia.org/wiki/Pagerank
+  #
+  DAMPING_FACTOR = 0.85
+
+  #
+  # Each user's line looks like
+  #
+  #   user_a    pagerank        id1,id2,...,idN
+  #
+  # we need to disperse this user's pagerank to each of id1..idN, and
+  # rendezvous the list of outbound links at user_a's reducer as well.
+  #
+  module Iterating
+    class Mapper < Wukong::Streamer::Base
+      #
+      # Send pagerank to each page, and send the dests list back to self
+      #
+      def process src, pagerank, dests_str, &block
+        # This lets us use Pig to generate the input
+        dests_str = dests_str.gsub(/[\(\{\}\)]/, '')
+        dests     = dests_str.split(",")
+        yield_pagerank_shares src, pagerank, dests, &block
+        yield_own_dest_list   src, dests_str,       &block
+      end
+
+      #
+      # Take the source node's pagerank and distribute it among all the out-nodes
+      #
+      def yield_pagerank_shares src, pagerank, dests
+        pagerank_share = pagerank.to_f / dests.length
+        dests.each do |dest|
+          yield [dest, 'p', pagerank_share]
+        end
+      end
+
+      #
+      # Dispatch this user's out-node list to rendezvous with itself.
+      #
+      def yield_own_dest_list src, dests_str
+        yield [src, 'd', dests_str]
+      end
+    end
+
+    class Reducer < Wukong::Streamer::AccumulatingReducer
+      attr_accessor :node_id, :pagerank, :dests_str
+      # Begin reduction with 0 accumulated pagerank and no dests as yet
+      def start! node_id, *args
+        self.node_id   = node_id
+        self.pagerank  = 0.0
+        self.dests_str = nil
+      end
+      # We'll receive fractional pagerank from all incoming edges,
+      # and the destination list from this node's map stage
+      def accumulate node_id, what, val
+        case what
+        when 'p' then self.pagerank += val.to_f
+        when 'd' then self.dests_str = val
+        else     raise "Don't know how to accumulate #{[node_id, what, val].inspect}"
+        end
+      end
+      # To finalize, dump the damped pagerank and dest list
+      # in a form that can be fed back into this script
+      def finalize
+        damped_pagerank = (self.pagerank * DAMPING_FACTOR) + (1 - DAMPING_FACTOR)
+        self.dests_str = 'dummy' if self.dests_str.blank?
+        yield [node_id, damped_pagerank, dests_str]
+      end
+    end
+
+    class Script < Wukong::Script
+      def default_options
+        super.merge :extra_args => ' -jobconf io.sort.record.percent=0.25 '
+      end
+    end
+    Script.new(Mapper, Reducer).run
+  end
+end

data/examples/pagerank/pagerank_initialize.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong'
+require 'wukong/streamer/set_reducer'
+
+module PageRank
+  class Script < Wukong::Script
+    #
+    # Input format is
+    #
+    #   rsrc    src_id  dest_id  [... junk ...]
+    #
+    # All we want from the line are its src and dest IDs.
+    #
+    def map_command
+      %Q{/usr/bin/cut -d"\t" -f2,3}
+    end
+
+    def default_options
+      super.merge :extra_args => ' -jobconf io.sort.record.percent=0.25 '
+    end
+  end
+
+  #
+  # Accumulate the dests list in memory, dump as a whole. Multiple edges between
+  # any two nodes are permitted, and will accumulate pagerank according to the
+  # edge's multiplicity.
+  #
+  class Reducer < Wukong::Streamer::ListReducer
+    def accumulate src, dest
+      self.values << dest
+    end
+
+    # Emit src, initial pagerank, and flattened dests list
+    def finalize
+      self.values = ['dummy'] if self.values.blank?
+      yield [key, 1.0, self.values.to_a.join(",")]
+    end
+  end
+
+  # Execute the script
+  Script.new(nil, PageRank::Reducer).run
+end
+
+
+

data/examples/pagerank/run_pagerank.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Directory to pagerank on.
+work_dir=$1 ; shift
+if [ "$work_dir" == '' ] ; then echo "Please specify the parent of the directory made by gen_initial_pagerank" ; exit ; fi
+
+
+# How many rounds to run
+max_iter=10
+# this directory
+script_dir="`dirname $0`"
+
+for (( curr=0 , next=1 ; "$curr" < "$max_iter" ; curr++ , next++ )) ; do
+  curr_str=`printf "%03d" ${curr}`
+  next_str=`printf "%03d" ${next}`
+  curr_dir=$work_dir/pagerank_graph_${curr_str}
+  next_dir=$work_dir/pagerank_graph_${next_str}
+  $script_dir/pagerank.rb --rm --run $curr_dir $next_dir
+done

data/examples/rank_and_bin.rb

@@ -0,0 +1,173 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+require 'wukong'
+require 'wukong/streamer/rank_and_bin_reducer'
+
+#
+# This example uses the classes from http://github.com/mrflip/twitter_friends
+# (That's sloppy, and I apologize. I'm building this script for that, but it
+# seems broadly useful and I'm not maintaining two copies. Once this script is
+# more worky we'll make it standalone.  Anyway you should get the picture.)
+#
+$: << File.dirname(__FILE__)+'/../../projects/twitter_friends/lib'
+require 'twitter_friends';
+require 'twitter_friends/struct_model' ; include TwitterFriends::StructModel
+
+
+#
+# attrs to bin
+#
+BINNABLE_ATTRS = {
+  :twitter_user => [
+    [:followers_count,  :fo   ],
+    [:friends_count,    :fr   ],
+    [:statuses_count,   :st   ],
+    [:favourites_count, :fv   ],
+    [:created_at,       :crat ]
+    ]
+
+}
+RESOURCE_ALIASES = {
+  :twitter_user  => :u,
+  :user_metrics  => :um,
+}
+#
+# KLUDGE This is not DRY at all but  let's get it working first
+#
+BinUserMetrics = TypedStruct.new(
+  [:id, Integer],
+  *BINNABLE_ATTRS[:user_metrics].map{|attr, attr_abbr| [attr_abbr, Integer] }
+  )
+BINNED_RESOURCE_ALIASES = {
+  :u   => BinTwitterUser,
+}
+
+module RankAndBinAttrs
+  class ExplodeResourceMapper < Wukong::Streamer::StructStreamer
+    def get_and_format_attr thing, attr
+      val = thing.send(attr)
+      case thing.members_types[attr].to_s.to_sym
+      when :Integer     then "%010d"   % val.to_i
+      when :Float       then "%020.7f" % val.to_f
+      when :Bignum      then "%020d"   % val.to_i
+      else
+        raise [val, thing.members_types[attr].to_s.to_sym].inspect
+      end
+    end
+
+    #
+    # The data expansion of this mapper is large enough that it makes sense to
+    # be a little responsible with what we emit.  We'll use the RESOURCE_ALIASES
+    # and BINNABLE_ATTRS hashes, above, to dump a more parsimonious
+    # representation.
+    #
+    def process thing, *args, &block
+      attr_abbrs = BINNABLE_ATTRS[thing.class.resource_name]
+      return unless attr_abbrs
+      attr_abbrs.each do |attr, abbr|
+        yield [
+          RESOURCE_ALIASES[thing.class.resource_name],
+          abbr,
+          get_and_format_attr(thing, attr),
+          thing.id.to_i
+        ]
+      end
+    end
+  end
+
+  class BinAttrReducer < Wukong::Streamer::RankAndBinReducer
+    attr_accessor :last_rsrc_attr
+    #
+    # Note that we might get several different resources at the same reducer
+    #
+    def get_key rsrc, attr, val, *args
+      if [rsrc, attr] != self.last_rsrc_attr
+        # Note: since each partition has the same cardinality, we don't need to
+        # fiddle around with the bin_size, etc -- just reset the order
+        # parameters' state.
+        reset_order_params!
+        self.last_rsrc_attr = [rsrc, attr]
+      end
+      val
+    end
+
+    #
+    # Note well -- we are rearranging the field order to
+    #
+    #   resource_abbr id  attr_abbr  bin
+    #
+    # for proper sorting to the re-assembler
+    #
+    def emit record
+      rsrc, attr, val, id, numbering, rank, bin = record
+      super [rsrc, id, attr, bin]
+    end
+  end
+
+  class ReassembleObjectReducer < Wukong::Streamer::AccumulatingReducer
+    attr_accessor :thing
+    def klass_from_abbr rsrc_abbr
+      BINNED_RESOURCE_ALIASES[rsrc_abbr.to_sym]
+    end
+    def get_key rsrc_abbr, id, *args
+      [rsrc_abbr, id.to_i]
+    end
+
+    def start! rsrc_abbr, id, *args
+      klass = klass_from_abbr(rsrc_abbr)
+      self.thing = klass.new id.to_i
+    end
+
+    def accumulate rsrc, id, attr, bin
+      thing.send("#{attr}=", bin)
+    end
+
+    def finalize
+      yield thing
+    end
+  end
+
+  #
+  # Two-phase script
+  #
+  # FIXME -- We need a runner class to manage this.
+  #
+  class Script < Wukong::Script
+    attr_accessor :phase
+    # KLUDGE !!
+    def initialize
+      case
+      when ARGV.detect{|arg| arg =~ /--phase=1/}
+        # Phase 1 -- Steal underpants.  Also, disassemble each object, and find
+        # the bin for each binnable attribute's value
+        self.phase = 1
+        self.mapper_klass, self.reducer_klass = [ExplodeResourceMapper, BinAttrReducer]
+      when ARGV.detect{|arg| arg =~ /--phase=2/}
+        # Phase 2 -- ????
+        raise "Phase 2 : ????"
+      when ARGV.detect{|arg| arg =~ /--phase=3/}
+        # Phase 3 -- profit. In this case, put records back together.
+        self.phase = 3
+        self.mapper_klass, self.reducer_klass = [nil, ReassembleObjectReducer]
+      else
+        raise "Please run me with a --phase= option"
+      end
+      super mapper_klass, reducer_klass
+    end
+
+    def default_options
+      extra_options =
+        case self.phase
+        # partition on [rsrc, attr]; sort on [rsrc, attr, val]
+        when 1  then { :sort_fields => 3, :partition_fields => 2 }
+        # sort on [rsrc, id]
+        when 3  then { :sort_fields => 2 }
+        else { }
+        end
+      super.merge extra_options
+    end
+  end
+
+  # execute script
+  Script.new.run
+end

data/examples/run_all.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+src_path="tmp/README.textile"
+out_root="tmp/test"
+hdp_opts="--map_tasks=1 --reduce_tasks=1"
+
+# ---------------------------------------------------------------------------
+#
+# Set up directories and copy over sample input
+#
+
+# hdp-rm ${src_path}
+# hdp-put `dirname $0`/../README.textile tmp/
+# hdp-mkdir $out_root 
+
+# ---------------------------------------------------------------------------
+#
+# Run scripts
+#
+
+cmd="word_count"
+    # hdp-rm -r ${out_root}/${cmd}
+    # ./examples/${cmd}.rb  	--run $hdp_opts $src_path ${out_root}/${cmd}
+    # hdp-catd ${out_root}/${cmd} | head -n 20
+    word_count=${out_root}/${cmd}
+
+cmd="sample_records"
+    # hdp-rm -r ${out_root}/${cmd}
+    # ./examples/${cmd}.rb  	--sampling_fraction=0.8 \
+    #     			--run $hdp_opts $src_path ${out_root}/${cmd}
+    # hdp-catd ${out_root}/${cmd} | head -n 200 | tail -n 20
+    sample_records=${out_root}/${cmd}
+
+
+# cmd="size"
+#     hdp-rm -r ${out_root}/${cmd}
+#     ./examples/${cmd}.rb  	--run $hdp_opts $src_path ${out_root}/${cmd}
+#     hdp-catd ${out_root}/${cmd} 
+#     size=${out_root}/${cmd}
+
+
+cmd="count_keys"
+    hdp-rm -r ${out_root}/${cmd}
+    ./examples/${cmd}.rb  	--run $hdp_opts $word_count ${out_root}/${cmd}
+    hdp-catd ${out_root}/${cmd}  | head -n 200 | tail -n 20
+    count_keys=${out_root}/${cmd}
+

data/examples/sample_records.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+require 'wukong'
+
+#
+# Probabilistically emit some fraction of record/lines
+#
+# Set the sampling fraction at the command line using the
+#   --sampling_fraction=
+# option: for example, to take a random 1/1000th of the lines in huge_files,
+#  ./examples/sample_records.rb --sampling_fraction=0.001 --go huge_files sampled_files
+#
+class Mapper < Wukong::Streamer::LineStreamer
+  include Wukong::Streamer::Filter
+
+  #
+  # floating-point number between 0 and 1 giving the fraction of lines to emit:
+  # at sampling_fraction=1 all records are emitted, at 0 none are.
+  #
+  # Takes its value from a mandatory command-line option
+  #
+  def sampling_fraction
+    @sampling_fraction ||= ( options[:sampling_fraction] && options[:sampling_fraction].to_f ) or
+      raise "Please supply a --sampling_fraction= argument, a decimal number between 0 and 1"
+  end
+
+  #
+  # randomly decide to emit +sampling_fraction+ fraction of lines
+  #
+  def emit? line
+    rand < self.sampling_fraction
+  end
+end
+
+class Script < Wukong::Script
+  def default_options
+    super.merge :reduce_tasks => 0
+  end
+end
+
+#
+# Executes the script
+#
+Script.new( Mapper, nil ).run