wukong 3.0.0.pre → 3.0.0.pre2

data/old/examples/network_graph/gen_2paths.rb

@@ -1,68 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong/script'
-
-class Edge < Struct.new(:src, :dest)
-end
-
-class MultiEdge < Struct.new(
-    :src,           :dest,
-    :a_follows_b,   :b_follows_a,
-    :a_replies_b,   :b_replies_a,
-    :a_favorites_b, :b_favorites_a
-    )
-end
-
-module Gen1HoodEdges
-  class Mapper < Wukong::Streamer::Base
-    def process rsrc, src, dest
-      # next if (src.to_i == 0) || (dest.to_i == 0)
-      yield [ dest, 'i', src ]
-      yield [ src,  'o', dest]
-    end
-  end
-
-  #
-  # Accumulate ( !!in memory!!) all inbound links onto middle node
-  #
-  # Then for each outbound link, loop over those inbound links and emit the
-  # triple (in, mid,out)
-  #
-  class Reducer < Wukong::Streamer::AccumulatingReducer
-    attr_accessor :ins
-    def start! *args
-      self.ins  = []
-    end
-    def accumulate mid, dir, node
-      case dir
-      when 'i'
-        self.ins << node
-        if (self.ins.length % 1000 == 0) && (self.ins.length > 10000)
-          $stderr.puts ["Accumulating:", mid, self.ins.length].join("\t")
-        end
-      when 'o'
-        ins.each do |inn|
-          yield ['path_2', inn, mid, node]
-        end
-      end
-    end
-    def finalize
-    end
-    def get_key mid, *_
-      mid
-    end
-  end
-
-  class Script < Wukong::Script
-    def default_options
-      super.merge :sort_fields => 2, :partition_fields => 1
-    end
-  end
-
-end
-
-# Execute the script
-Gen1HoodEdges::Script.new(
-  Gen1HoodEdges::Mapper,
-  Gen1HoodEdges::Reducer
-  ).run

data/old/examples/network_graph/gen_multi_edge.rb

@@ -1,112 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong'
-
-#
-# Takes any number of flavors of directed edge with the form
-#
-#   a_relatesto_b src_id dest_id [optional fields]
-#
-# and prepares a combined adjacency list.  You need to supply a model named
-# "MultiEdge" with members for each edge type.
-#
-# For instance, suppose you have a social network with edges like
-#
-#   a_follows_b   user_a_id  user_b_id
-#   a_messages_b  user_a_id  user_b_id  message_id date
-#   a_favorites_b user_a_id  user_b_id  message_id date
-#
-# Your MultiEdge class might look like
-#
-#   class MultiEdge < Struct(
-#     :src, :dest,
-#     :a_follows_b,   :b_follows_a,
-#     :a_messages_b,   :b_messages_a,
-#     :a_favorites_b, :b_favorites_a
-#     )
-#   end
-#
-# The row for a user pair who follows each other; with user_a #24601 messaging b
-# 57 times and favoriting 5 of user_b's messages; and user_b #8675309 messaging
-# 62 times and favoriting none, will emerge as (tab separated, with [blank]
-# indicating there is no text in that slot):
-#
-#   ...
-#   24601       8675309 1       1       57      62      5       [blank]
-#   ...
-#
-module GenMultiEdge
-  #
-  # Emit each relation as
-  #
-  #   src       dest    rel
-  #
-  # Canonicalizes the src and dest ids to 10-character, zero-padded strings.
-  # (Ten chars fits a 32-bit up-to-4-billion-and-change unsigned integer.)
-  # Discards all the ancillary crap except +src+, +dest+ and +rel+
-  #
-  class Mapper < Wukong::Streamer::Base
-    def process rsrc, src, dest, *_
-      # note that a_retweets_b_id matches here
-      m = /^a_([a-z]+)_b.*/.match(rsrc) or return
-      rel = m.captures.first
-      src = src.to_i ; dest = dest.to_i
-      return if ((src == 0) || (dest == 0))
-      yield [src,  dest, "a_#{rel}_b"]
-      yield [dest, src,  "b_#{rel}_a"]
-    end
-  end
-
-  #
-  # Aggregate all sightings of relations for each pair into
-  # a single combined
-  #
-  # Note that [a,b] and [b,a] /each/ have a listing, with the a->b and b<-a
-  # relations repeated for each.  That is, if there is an "a_messages_b"
-  # relation, you'll have edges
-  #
-  #    x        y       ...     a_messages_b(x,y)       b_messages_a(y,x)  ...
-  #    y        x       ...     a_messages_b(y,x)       b_messages_a(x,y)  ...
-  #
-  #
-  class Reducer < Wukong::Streamer::AccumulatingReducer
-    attr_accessor :multi_edge
-    def get_key src, dest, rel
-      [src, dest]
-    end
-    def start! *args
-      self.multi_edge = MultiEdge.new
-    end
-    def accumulate src, dest, rel
-      self.multi_edge[rel] ||= 0
-      self.multi_edge[rel]  += 1
-    end
-    def finalize
-      multi_edge.src, multi_edge.dest = key
-      yield self.multi_edge
-    end
-  end
-end
-
-Edge = TypedStruct.new(
-  [:src,              Integer],
-  [:dest,             Integer]
-  )
-
-MultiEdge = TypedStruct.new(
-  [:src,              Integer],
-  [:dest,             Integer],
-  [:a_follows_b,      Integer],
-  [:b_follows_a,      Integer],
-  [:a_replies_b,      Integer],
-  [:b_replies_a,      Integer],
-  [:a_atsigns_b,      Integer],
-  [:b_atsigns_a,      Integer],
-  [:a_retweets_b,     Integer],
-  [:b_retweets_a,     Integer],
-  [:a_favorites_b,    Integer],
-  [:b_favorites_a,    Integer]
-  )
-
-# Execute the script
-Script.new(Mapper, Reducer, :sort_fields => 2).run

data/old/examples/network_graph/gen_symmetric_links.rb

@@ -1,64 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-$: << File.dirname(__FILE__)+'/../../lib'
-require 'wukong'
-
-class Edge < Struct.new(:src, :dest)
-end
-
-class ASymmetricB < Edge
-end
-
-module Wukong::Streamer
-  class EdgeStreamer < Wukong::Streamer::Base
-    def recordize line
-      rsrc, src, dest, *_ = super(line)
-      [ASymmetricB.new(src.to_i, dest.to_i)]
-    end
-  end
-end
-
-#
-# Find symmetric links
-#
-# Takes adjacency list for a directed graph and emits only edges where
-#   A->B and B->A
-#
-# The output will list each such symmetric edge exactly once as
-#    a_symmetric_b   node1    node2
-# where node1 is lexicographically less than node2.
-#
-module FindSymmetricLinks
-
-  class Mapper < Wukong::Streamer::EdgeStreamer
-    def process edge
-      yield edge.to_flat(false)
-      yield ASymmetricB.new(edge.dest, edge.src).to_flat(false)
-    end
-  end
-
-  #
-  #
-  class Reducer < Wukong::Streamer::Base
-    def stream
-      %x{/usr/bin/uniq -c}.split("\n").each do |line|
-        key_count, rsrc, src, dest, data = line.chomp.strip.split(/\s+/, 4)
-        next unless key_count.to_i == 2
-        next unless src.to_i < dest.to_i
-        emit [src, dest, data].compact
-      end
-    end
-  end
-
-  class Script < Wukong::Script
-    def default_options
-      super.merge :sort_fields => 3
-    end
-  end
-end
-
-# Execute the script
-Wukong::Script.new(
-  FindSymmetricLinks::Mapper,
-  FindSymmetricLinks::Reducer
-  ).run

data/old/examples/pagerank/README.textile

@@ -1,6 +0,0 @@
-
-Calculate pagerank for a tab-separated adjacency list.
-
-See 
-* Heretrix pagerank util
-* http://www.umiacs.umd.edu/~jimmylin/cloud9/docs/exercises/pagerank.html

data/old/examples/pagerank/gen_initial_pagerank_graph.pig

@@ -1,57 +0,0 @@
-
--- ===========================================================================
---
--- Load Graph
--- 
-AFollowsB               = LOAD    'twnew/all/a_follows_b' AS (rsrc: chararray, user_a_id: int, user_b_id: int) ;
-FollEdges_0             = FOREACH AFollowsB GENERATE user_a_id AS src, user_b_id AS dest ;
-
-InitPagerankFoll_0 	= GROUP FollEdges_0 BY src ;
-InitPagerankFoll_1 	= FOREACH InitPagerankFoll_0 GENERATE
-  group                   AS src,
-  1.0F                    AS pagerank:float,
-  FollEdges_0.(dest)  	  AS dests
-  ;
-rmf                            twnew/pagerank-foll/pagerank_graph_000 ;
-STORE InitPagerankFoll_1 INTO 'twnew/pagerank-foll/pagerank_graph_000';
-
-
--- MultiEdge               = LOAD    'twnew/all/multi_edge' AS (
---   rsrc: chararray, src: int, dest: int,
---   fo:     int, fr:    int,
---   re_out: int, re_in: int,
---   at_out: int, at_in: int,
---   rt_out: int, rt_in: int,
---   fv_out: int, fv_in: int) ;
--- 
--- SymmEdges_0   = FOREACH MultiEdge GENERATE src, dest, fo, fr ;
--- SymmEdges_1   = FILTER  SymmEdges_0 BY (fo >= 1.0) AND (fr >= 1.0) ;
--- SymmEdges     = FOREACH SymmEdges_1 GENERATE src, dest ;
--- -- rm twnew/graphs/symm_edges; STORE SymmEdges INTO 'twnew/graphs/symm_edges' ;
--- SymmEdges = LOAD 'twnew/graphs/symm_edges' AS (src:int , dest:int);
--- 
--- AnyoutEdges_0 = FOREACH MultiEdge GENERATE src, dest, fo, re_out, fv_out ;
--- AnyoutEdges_1 = FILTER AnyoutEdges_0 BY (fo >= 1.0) OR (re_out >= 1.0) OR (fv_out >= 1.0) ;
--- AnyoutEdges   = FOREACH AnyoutEdges_1 GENERATE src, dest ;
--- -- rm twnew/graphs/anyout_edges; STORE AnyoutEdges INTO 'twnew/graphs/anyout_edges' ;
--- AnyoutEdges = LOAD 'twnew/graphs/anyout_edges' AS (src:int , dest:int);
--- 
--- 
--- InitPagerankSymm_0 = GROUP SymmEdges BY src ;
--- InitPagerankSymm_1 = FOREACH InitPagerankSymm_0 GENERATE
---   group                 AS src,
---   1.0F                  AS pagerank:float,
---   SymmEdges.(dest)  AS dests
---   ;
--- rm                             twnew/pagerank-symm/pagerank_graph_000 ;
--- STORE InitPagerankSymm_1 INTO 'twnew/pagerank-symm/pagerank_graph_000';
--- 
--- 
--- InitPagerankAnyout_0 = GROUP   AnyoutEdges BY src ;
--- InitPagerankAnyout_1 = FOREACH InitPagerankAnyout_0 GENERATE
---   group                 AS src,
---   1.0F                  AS pagerank:float,
---   AnyoutEdges.(dest)  AS dests
---   ;
--- rm                               twnew/pagerank-anyout/pagerank_graph_000 ;
--- STORE InitPagerankAnyout_1 INTO 'twnew/pagerank-anyout/pagerank_graph_000';

data/old/examples/pagerank/pagerank.rb

@@ -1,72 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong/script'
-
-module PageRank
-  #
-  # Damping factor (prob. of a 'random' jump)
-  # 0.85 works well in practice. See http://en.wikipedia.org/wiki/Pagerank
-  #
-  DAMPING_FACTOR = 0.85
-
-  # Each user's line looks like
-  #   user_a    pagerank        id1,id2,...,idN
-  # we need to disperse this user's pagerank to each of id1..idN, and
-  # rendezvous the list of outbound links at user_a's reducer as well.
-  module Iterating
-    class PagerankMapper < Wukong::Streamer::Base
-      #
-      # Send pagerank to each page, and send the dests list back to self
-      #
-      def process src, pagerank, dests_str, &block
-        # This lets us use Pig to generate the input
-        dests_str = dests_str.gsub(/[\(\{\}\)]/, '')
-        dests     = dests_str.split(",")
-        yield_pagerank_shares src, pagerank, dests, &block
-        yield_own_dest_list   src, dests_str,       &block
-      end
-
-      # Take the source node's pagerank and distribute it among all the out-nodes
-      def yield_pagerank_shares src, pagerank, dests
-        pagerank_share = pagerank.to_f / dests.length
-        dests.each do |dest|
-          yield [dest, 'p', pagerank_share]
-        end
-      end
-
-      # Dispatch this user's out-node list to rendezvous with itself.
-      def yield_own_dest_list src, dests_str
-        yield [src, 'd', dests_str]
-      end
-    end
-
-    class PagerankReducer < Wukong::Streamer::AccumulatingReducer
-      attr_accessor :node_id, :pagerank, :dests_str
-      # Begin reduction with 0 accumulated pagerank and no dests as yet
-      def start! node_id, *args
-        self.node_id   = node_id
-        self.pagerank  = 0.0
-        self.dests_str = nil
-      end
-      # We'll receive fractional pagerank from all incoming edges,
-      # and the destination list from this node's map stage
-      def accumulate node_id, what, val
-        case what
-        when 'p' then self.pagerank += val.to_f
-        when 'd' then self.dests_str = val
-        else     raise "Don't know how to accumulate #{[node_id, what, val].inspect}"
-        end
-      end
-      # To finalize, dump the damped pagerank and dest list
-      # in a form that can be fed back into this script
-      def finalize
-        damped_pagerank = (self.pagerank * DAMPING_FACTOR) + (1 - DAMPING_FACTOR)
-        self.dests_str = 'dummy' if self.dests_str.blank?
-        yield [node_id, damped_pagerank, dests_str]
-      end
-    end
-
-    Wukong.run(PagerankMapper, PagerankReducer,
-      :extra_args => ' -jobconf io.sort.record.percent=0.25 ')
-  end
-end

data/old/examples/pagerank/pagerank_initialize.rb

@@ -1,42 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong/script'
-require 'wukong/streamer/list_reducer'
-
-module PageRank
-  class Script < Wukong::Script
-    #
-    # Input format is
-    #
-    #   rsrc    src_id  dest_id  [... junk ...]
-    #
-    # All we want from the line are its src and dest IDs.
-    #
-    def map_command
-      %Q{/usr/bin/cut -d"\t" -f2,3}
-    end
-  end
-
-  #
-  # Accumulate the dests list in memory, dump as a whole. Multiple edges between
-  # any two nodes are permitted, and will accumulate pagerank according to the
-  # edge's multiplicity.
-  #
-  class Reducer < Wukong::Streamer::ListReducer
-    def accumulate src, dest
-      @values << dest
-    end
-
-    # Emit src, initial pagerank, and flattened dests list
-    def finalize
-      @values = ['dummy'] if @values.blank?
-      yield [key, 1.0, @values.to_a.join(",")]
-    end
-  end
-
-  # Execute the script
-  Script.new(nil, PageRank::Reducer, :io_sort_record_percent => 0.25).run
-end
-
-
-

data/old/examples/pagerank/run_pagerank.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-# Directory to pagerank on.
-work_dir=$1     ; shift
-if [ "$work_dir" == '' ] ; then echo "Please specify the parent of the directory made by gen_initial_pagerank: $0 initial_dir [number_of_iterations] [start_iteration]" ; exit ; fi
-# How many rounds to run: default 10
-n_iters=${1-10} ; shift
-# the iteration to start with: default 0
-start_i=${1-0}  ; shift
-
-# this directory
-script_dir="`dirname $0`"
-
-for (( iter=0 ; "$iter" < "$n_iters" ; iter++ )) ; do
-  curr_str=`printf "%03d" $(( $start_i + $iter     ))`
-  next_str=`printf "%03d" $(( $start_i + $iter + 1 ))`
-  curr_dir=$work_dir/pagerank_graph_${curr_str}
-  next_dir=$work_dir/pagerank_graph_${next_str}
-  echo -e "Iteration $(( $iter + 1 )) / $n_iters:\t `basename $curr_dir` => `basename $next_dir`"
-  $script_dir/pagerank.rb --rm --run $curr_dir $next_dir
-done