mrflip-wukong 0.1.0

data/examples/graph/breadth_first_search.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+$: << ENV['WUKONG_PATH']
+require 'wukong'
+
+#
+# Use this script to do a Breadth-First Search (BFS) of a graph.
+#
+# Usage:
+#   ./make_paths --head=[path_in_key] --tail=[path_out_key] --out_rsrc=[combined_path_key]
+#
+# For example, given an edge list in the file '1path.tsv' that looks like
+#   1path	n1	n2
+#   1path       n1	n3
+#   ... and so forth ...
+# you can run
+#   for t in 1 2 3 4 5 6 7 8 9 ; do next=$((t+1)) ; time cat 1path.tsv "${t}path.tsv" | ./make_paths.rb --map --head="1path"  --tail="${t}path" | sort -u | ./make_paths.rb --reduce --out_rsrc="${next}path" | sort -u > "${next}path.tsv" ; done
+# to do a 9-deep breadth-first search.
+#
+module Gen1HoodEdges
+  class Mapper < Wukong::Streamer::Base
+    attr_accessor :head, :tail
+    def initialize options
+      self.head = options[:head]
+      self.tail = options[:tail]
+    end
+    def process rsrc, *nodes
+      yield [ nodes.last,  'i', nodes[0..-2] ] if (rsrc == self.head)
+      yield [ nodes.first, 'o', nodes[1..-1] ] if (rsrc == self.tail)
+    end
+  end
+
+  #
+  # Accumulate ( !!in memory!!) all inbound links onto middle node
+  #
+  # Then for each outbound link, loop over those inbound links and emit the
+  # triple (in, mid,out)
+  #
+  class Reducer < Wukong::Streamer::AccumulatingReducer
+    attr_accessor :paths_in, :out_rsrc
+    def initialize options
+      self.out_rsrc = options[:out_rsrc]
+    end
+    # clear the list of incoming paths
+    def start! *args
+      self.paths_in  = []
+    end
+    def accumulate mid, dir, *nodes
+      case dir
+      when 'i'
+        self.paths_in << nodes
+        if (self.paths_in.length % 1000 == 0) && (self.paths_in.length > 10000)
+          $stderr.puts ["Accumulating:", mid, self.paths_in.length].join("\t")
+        end
+      when 'o'
+        paths_in.each do |path_in|
+          yield [self.out_rsrc, path_in, mid, *nodes]
+        end
+      end
+    end
+    def finalize
+    end
+    def get_key mid, *_
+      mid
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :sort_fields => 2, :partition_fields => 1
+    end
+  end
+
+end
+
+# Execute the script
+Gen1HoodEdges::Script.new(
+  Gen1HoodEdges::Mapper,
+  Gen1HoodEdges::Reducer
+  ).run

data/examples/graph/gen_2paths.rb

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong'
+
+class Edge < Struct.new(:src, :dest)
+end
+
+class MultiEdge < Struct.new(
+    :src,           :dest,
+    :a_follows_b,   :b_follows_a,
+    :a_replies_b,   :b_replies_a,
+    :a_favorites_b, :b_favorites_a
+    )
+end
+
+module Gen1HoodEdges
+  class Mapper < Wukong::Streamer::Base
+    def process rsrc, src, dest
+      # next if (src.to_i == 0) || (dest.to_i == 0)
+      yield [ dest, 'i', src ]
+      yield [ src,  'o', dest]
+    end
+  end
+
+  #
+  # Accumulate ( !!in memory!!) all inbound links onto middle node
+  #
+  # Then for each outbound link, loop over those inbound links and emit the
+  # triple (in, mid,out)
+  #
+  class Reducer < Wukong::Streamer::AccumulatingReducer
+    attr_accessor :ins
+    def start! *args
+      self.ins  = []
+    end
+    def accumulate mid, dir, node
+      case dir
+      when 'i'
+        self.ins << node
+        if (self.ins.length % 1000 == 0) && (self.ins.length > 10000)
+          $stderr.puts ["Accumulating:", mid, self.ins.length].join("\t")
+        end
+      when 'o'
+        ins.each do |inn|
+          yield ['path_2', inn, mid, node]
+        end
+      end
+    end
+    def finalize
+    end
+    def get_key mid, *_
+      mid
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :sort_fields => 2, :partition_fields => 1
+    end
+  end
+
+end
+
+# Execute the script
+Gen1HoodEdges::Script.new(
+  Gen1HoodEdges::Mapper,
+  Gen1HoodEdges::Reducer
+  ).run

data/examples/graph/gen_multi_edge.rb

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong'
+require 'wukong/models/graph'; include Wukong::Models
+
+#
+# Takes any number of flavors of directed edge with the form
+#
+#   a_relatesto_b src_id dest_id [optional fields]
+#
+# and prepares a combined adjacency list.  You need to supply a model named
+# "MultiEdge" with members for each edge type.
+#
+# For instance, suppose you have a social network with edges like
+#
+#   a_follows_b   user_a_id  user_b_id
+#   a_messages_b  user_a_id  user_b_id  message_id date
+#   a_favorites_b user_a_id  user_b_id  message_id date
+#
+# Your MultiEdge class might look like
+#
+#   class MultiEdge < Struct(
+#     :src, :dest,
+#     :a_follows_b,   :b_follows_a,
+#     :a_messages_b,   :b_messages_a,
+#     :a_favorites_b, :b_favorites_a
+#     )
+#   end
+#
+# The row for a user pair who follows each other; with user_a #24601 messaging b
+# 57 times and favoriting 5 of user_b's messages; and user_b #8675309 messaging
+# 62 times and favoriting none, will emerge as (tab separated, with [blank]
+# indicating there is no text in that slot):
+#
+#   ...
+#   24601       8675309 1       1       57      62      5       [blank]
+#   ...
+#
+module GenMultiEdge
+  #
+  # Emit each relation as
+  #
+  #   src       dest    rel
+  #
+  # Canonicalizes the src and dest ids to 10-character, zero-padded strings.
+  # (Ten chars fits a 32-bit up-to-4-billion-and-change unsigned integer.)
+  # Discards all the ancillary crap except +src+, +dest+ and +rel+
+  #
+  class Mapper < Wukong::Streamer::Base
+    def process rsrc, src, dest, *_
+      # note that a_retweets_b_id matches here
+      m = /^a_([a-z]+)_b.*/.match(rsrc) or return
+      rel = m.captures.first
+      src = src.to_i ; dest = dest.to_i
+      return if ((src == 0) || (dest == 0))
+      yield ["%010d"%src,  "%010d"%dest, "a_#{rel}_b"]
+      yield ["%010d"%dest, "%010d"%src,  "b_#{rel}_a"]
+    end
+  end
+
+  #
+  # Aggregate all sightings of relations for each pair into
+  # a single combined
+  #
+  # Note that [a,b] and [b,a] /each/ have a listing, with the a->b and b<-a
+  # relations repeated for each.  That is, if there is an "a_messages_b"
+  # relation, you'll have edges
+  #
+  #    x        y       ...     a_messages_b(x,y)       b_messages_a(y,x)  ...
+  #    y        x       ...     a_messages_b(y,x)       b_messages_a(x,y)  ...
+  #
+  #
+  class Reducer < Wukong::Streamer::AccumulatingReducer
+    attr_accessor :multi_edge
+    def get_key src, dest, rel
+      [src, dest]
+    end
+    def start! *args
+      self.multi_edge = MultiEdge.new
+    end
+    def accumulate src, dest, rel
+      self.multi_edge[rel] ||= 0
+      self.multi_edge[rel]  += 1
+    end
+    def finalize
+      multi_edge.src, multi_edge.dest = key
+      yield self.multi_edge
+    end
+  end
+
+  #
+  # Sort on the first two keys: each @[src, dest]@ pair winds up at the same
+  # reducer.
+  #
+  class Script < Wukong::Script
+    def default_options
+      super.merge :sort_fields => 2
+    end
+  end
+
+  # Execute the script
+  Script.new(Mapper, Reducer).run
+end

data/examples/graph/gen_symmetric_links.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong'
+
+class Edge < Struct.new(:src, :dest)
+end
+
+class ASymmetricB < Edge
+end
+
+module Wukong::Streamer
+  class EdgeStreamer < Wukong::Streamer::Base
+    def recordize line
+      rsrc, src, dest, *_ = super(line)
+      [ASymmetricB.new(src.to_i, dest.to_i)]
+    end
+  end
+end
+
+module FindSymmetricLinks
+
+  class Mapper < Wukong::Streamer::EdgeStreamer
+    def process edge
+      yield edge.to_flat(false)
+      yield ASymmetricB.new(edge.dest, edge.src).to_flat(false)
+    end
+  end
+
+  #
+  #
+  class Reducer < Wukong::Streamer::Base
+    def stream
+      %x{/usr/bin/uniq -c}.split("\n").each do |line|
+        key_count, rsrc, src, dest, data = line.chomp.strip.split(/\s+/, 4)
+        next unless key_count.to_i == 2
+        next unless src.to_i < dest.to_i
+        emit [src, dest, data].compact
+      end
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :sort_fields => 3
+    end
+  end
+end
+
+# Execute the script
+Wukong::Script.new(
+  FindSymmetricLinks::Mapper,
+  FindSymmetricLinks::Reducer
+  ).run

data/examples/package-local.rb

@@ -0,0 +1,100 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+
+require 'wukong'
+
+#
+# This is so very very kludgey
+#
+# Input is an 'ls' file, listing files to .bz2 package.
+#
+# Mapper takes each in turn and creates, within a parallel directory tree under
+# ~/pkgd on the HDFS, a .bz2 compressed version of the file.
+#
+# So, the file
+#   /user/me/fixd/all-20090103
+# is packaged onto the DFS as
+#   /user/me/pkgd/user/me/fixd/all-20090103
+#
+#   listing=tmp/fixd-all-package-listing
+#   hdp-rm $listing
+#   hadoop dfs -lsr fixd | egrep '(part-|\.tsv$)' | hdp-put - $listing ;
+#
+#   ./package.rb --run --rm --map_tasks=1 $listing $pkgd_log
+#
+module ExportPackager
+  PKGD_DIR = '/workspace/flip/pkgd'
+
+  #
+  #
+  class Reducer < Wukong::Streamer::Base
+    def announce *args
+      $stdout.puts *args
+      $stderr.puts *args
+    end
+
+    def handle_existing_target output_filename
+      return true unless File.exist?(output_filename)
+      #   announce "Exists! #{output_filename}"
+      #   return false
+      announce "Removing target file #{output_filename}"
+      begin announce `rm #{output_filename}`
+      rescue Exception => e ; announce e ; end
+      true
+    end
+
+    def mkdir_target_safely output_filename
+      output_dir = File.dirname(output_filename)
+      announce "Ensuring directory #{output_dir} exists"
+      begin announce `mkdir -p #{output_dir}`
+      rescue Exception => e ; announce e ; end
+    end
+
+    def bzip_into_pkgd_file input_filename, output_filename
+      announce "bzip'ing into #{output_filename}"
+      announce `( hadoop dfs -cat #{input_filename}/[^_]\** ) | bzip2 -c > #{output_filename}`
+    end
+
+    def gen_output_filename input_filename
+      input_filename += '.tsv' unless input_filename =~ /.*\.\w{2,}/
+      "%s/%s.bz2" % [PKGD_DIR, input_filename.gsub(/^\//, '')]
+    end
+
+    def rsync host, local_path, remote_path=nil
+      remote_path ||= local_path
+      announce `/usr/bin/rsync -Cuvrtlp #{local_path} #{host}:#{remote_path}`
+      sleep 5
+    end
+
+    def process input_filename
+      output_filename = gen_output_filename(input_filename)
+      handle_existing_target(output_filename) or return
+      mkdir_target_safely    output_filename
+      bzip_into_pkgd_file    input_filename, output_filename
+      rsync :lab3, output_filename
+      #
+    end
+
+    def recordize line
+      # handle ls or straight file list, either
+      line.split(/\s/).last
+    end
+
+    def stream
+      super
+      rsync :lab3, PKGD_DIR+'/'
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :map_tasks => 1,
+        :max_node_reduce_tasks => 1, # only one reducer per local filesystem
+        :timeout => 40 * 60 * 1000   # timeout in ms
+    end
+  end
+  # Execute the script
+  Script.new(nil, Reducer).run
+end
+
+

data/examples/package.rb

@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+$: << ENV['WUKONG_PATH'] if ENV['WUKONG_PATH']
+require 'wukong'
+
+#
+# This is so very very kludgey
+#
+# Input is an 'ls' file, listing files to .bz2 package.
+#
+# Reducer takes each in turn and creates, within a parallel directory tree under
+# ~/pkgd on the HDFS, a .bz2 compressed version of the file.
+#
+# So, the file
+#   /user/me/fixd/all-20090103
+# is packaged onto the DFS as
+#   /user/me/pkgd/user/me/fixd/all-20090103
+#
+#   listing=tmp/fixd-all-package-listing
+#   hdp-rm $listing
+#   hadoop dfs -lsr fixd | egrep '(part-|\.tsv$)' | hdp-put - $listing ;
+#
+#   ./package.rb --run --rm --map_tasks=1 $listing $pkgd_log
+#
+module ExportPackager
+  PKGD_DIR = 'pkgd'
+
+  #
+  #
+  class Reducer < Wukong::Streamer::Base
+    def announce str
+      return if str.blank?
+      $stderr.puts str
+      $stdout.puts str
+    end
+
+    def remove_target_filename output_filename
+      begin announce "rm\t#{"%-70s"%output_filename}\t" +
+          `( hadoop dfs -rmr #{output_filename} ) 2>&1`
+      rescue ; nil ; end
+    end
+
+    def mkdir_target_safely output_filename
+      output_dir = File.dirname(output_filename)
+      begin announce "mkdir\t#{"%-70s"%output_dir}\t" +
+          `( hadoop dfs -mkdir #{output_dir} ) 2>&1`
+      rescue ; nil ; end
+    end
+
+    def bzip_into_pkgd_file input_filename, output_filename
+      announce "cat|bz\t#{"%-70s"%input_filename}\t" +
+        `( hadoop dfs -cat #{input_filename}/[^_]\\* | bzip2 -c | hadoop dfs -put - #{output_filename} ) 2>&1`
+    end
+
+    def verify input_filename, output_filename
+      announce "sha1sum\t#{"%-70s"%output_filename}\t" +
+        `( hadoop dfs -cat #{output_filename}        | bzcat - | sha1sum ) 2>&1`
+      announce "sha1sum\t#{"%-70s"%input_filename}\t" +
+        `( hadoop dfs -cat #{input_filename}/[^_]\\*           | sha1sum ) 2>&1`
+    end
+
+    def gen_output_filename input_filename
+      "%s/%s.bz2" % [PKGD_DIR, input_filename.gsub(%r{^/},"")]
+    end
+
+    def process input_filename, output_filename
+      # remove_target_filename output_filename
+      # mkdir_target_safely    output_filename
+      bzip_into_pkgd_file    input_filename, output_filename
+      verify                 input_filename, output_filename
+    end
+
+    def stream
+      announce `hostname`
+      $stdin.each do |input_filename|
+        # handle ls or straight file list, either
+        input_filename = input_filename.chomp.strip.split(/\s/).last
+        output_filename = gen_output_filename input_filename
+        announce "********************************************************"
+        announce "Packing\t#{"%-70s"%input_filename}\t#{output_filename}"
+        process input_filename, output_filename
+        announce "Done\t#{"%-70s"%input_filename}\t#{output_filename}\n\n"
+      end
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :timeout => (24 * 60 * 60 * 1000)  # milliseconds in one day
+    end
+  end
+end
+
+#
+# Execute the script
+#
+ExportPackager::Script.new(nil, ExportPackager::Reducer, :reduce_tasks => 1000).run