wukong 0.1.1

data/examples/count_keys.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+require 'wukong'
+require 'wukong/streamer/count_keys'
+require 'wukong/streamer/count_lines'
+
+#
+#
+class CountKeysReducer < Wukong::Streamer::CountLines
+  #
+  # Taken from the actionpack Rails component ('action_view/helpers/number_helper')
+  #
+  # Formats a +number+ with grouped thousands using +delimiter+. You
+  # can customize the format using optional <em>delimiter</em> and <em>separator</em> parameters.
+  # * <tt>delimiter</tt>  - Sets the thousands delimiter, defaults to ","
+  # * <tt>separator</tt>  - Sets the separator between the units, defaults to "."
+  #
+  #  number_with_delimiter(12345678)      => 12,345,678
+  #  number_with_delimiter(12345678.05)   => 12,345,678.05
+  #  number_with_delimiter(12345678, ".")   => 12.345.678
+  def number_with_delimiter(number, delimiter=",", separator=".")
+    begin
+      parts = number.to_s.split('.')
+      parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
+      parts.join separator
+    rescue
+      number
+    end
+  end
+
+  # Override to look nice
+  def formatted_count item, key_count
+    key_count_str = number_with_delimiter(key_count.to_i)
+    "%-25s\t%12s" % [item, key_count_str]
+  end
+end
+
+#
+class CountKeysScript < Wukong::Script
+  def map_command
+    # Use `cut` to extract the first field
+    %Q{ cut -d"\t" -f1 }
+  end
+
+  #
+  # There's just the one field
+  #
+  def default_options
+    super.merge :sort_fields => 1
+  end
+end
+
+# Executes the script when run from command line
+if __FILE__ == $0
+  CountKeysScript.new(nil, CountKeysReducer).run
+end

data/examples/count_keys_at_mapper.rb

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../lib'
+require 'wukong'
+
+#
+#
+module CountKeys
+  #
+  class Mapper < Wukong::Streamer::Base
+    attr_accessor :keys_count
+    def initialize *args
+      self.keys_count = {}
+    end
+    def process key, *args
+      key.gsub!(/-.*/, '')  # kill off the slug
+      self.keys_count[key] ||= 0
+      self.keys_count[key]  += 1
+    end
+    def stream *args
+      super *args
+      self.keys_count.each do |key, count|
+        emit [key, count].to_flat
+      end
+    end
+  end
+  # Identity Mapper
+  class Reducer < Wukong::Streamer::AccumulatingReducer
+    attr_accessor :key_count
+    require 'active_support'
+    require 'action_view/helpers/number_helper'; include ActionView::Helpers::NumberHelper
+
+    # Override to look nice
+    def formatted_count item, key_count
+      key_count_str = number_with_delimiter(key_count.to_i, :delimiter => ',')
+      "%-25s\t%12s" % [item, key_count_str]
+    end
+    def start!  *args
+      self.key_count = 0
+    end
+    def accumulate key, count
+      self.key_count += count.to_i
+    end
+    def finalize
+      yield formatted_count(key, key_count)
+    end
+  end
+
+  #
+  class Script < Wukong::Script
+    # There's just the one field
+    def default_options
+      super.merge :sort_fields => 1, :reduce_tasks => 1
+    end
+  end
+end
+
+CountKeys::Script.new(CountKeys::Mapper, CountKeys::Reducer).run

data/examples/graph/adjacency_list.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+$: << '/home/flip/ics/wukong/lib' # ENV['WUKONG_PATH']
+require 'wukong'
+
+#
+# Given an adjacency pairs (from \t to) representation of a directed graph:
+#
+#    1     2
+#    1     7
+#    2     7
+#    2     9
+#    7     2
+#
+# It produces an "adjacency list":http://en.wikipedia.org/wiki/Adjacency_list representation:
+#
+#    1  >   2   7
+#    2  >   7   9
+#    7  >   2
+#    9  >
+#
+# and
+#
+#    1  <
+#    2  <   1   7
+#    7  <   1   2
+#    9  <   2
+#
+# (each column is tab-separated in the actual output)
+#
+#
+#
+module Gen1HoodEdges
+  class Mapper < Wukong::Streamer::Base
+    def process rsrc, src, dest, *_
+      src = src.to_i ; dest = dest.to_i
+      yield [ src,  '>', dest ]
+      yield [ dest, '<', src  ]
+    end
+  end
+
+  #
+  # Accumulate links onto single line.
+  #
+  # The reduce key is the target node and direction; we just stream through all
+  # pairs for each target node and output its neighbor nodes on the same line.
+  #
+  # To control memory usage, we will print directly to the output (and not run
+  # through the Emitter)
+  #
+  class Reducer < Wukong::Streamer::AccumulatingReducer
+    # clear the list of incoming paths
+    def start! target, dir, *args
+      print target + "\t" + dir  # start line with target and list type
+    end
+    def accumulate target, dir, neighbor
+      print "\t" + neighbor      # append neighbor to output, same line
+    end
+    def finalize
+      puts ''                    # start new line
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :sort_fields => 1, :partition_fields => 1
+    end
+  end
+end
+
+# Execute the script
+Gen1HoodEdges::Script.new(
+  Gen1HoodEdges::Mapper,
+  Gen1HoodEdges::Reducer
+  ).run

data/examples/graph/breadth_first_search.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+$: << ENV['WUKONG_PATH']
+require 'wukong'
+
+#
+# Use this script to do a Breadth-First Search (BFS) of a graph.
+#
+# Usage:
+#   ./make_paths --head=[path_in_key] --tail=[path_out_key] --out_rsrc=[combined_path_key]
+#
+# For example, given an edge list in the file '1path.tsv' that looks like
+#   1path	n1	n2
+#   1path       n1	n3
+#   ... and so forth ...
+# you can run
+#   for t in 1 2 3 4 5 6 7 8 9 ; do next=$((t+1)) ; time cat 1path.tsv "${t}path.tsv" | ./make_paths.rb --map --head="1path"  --tail="${t}path" | sort -u | ./make_paths.rb --reduce --out_rsrc="${next}path" | sort -u > "${next}path.tsv" ; done
+# to do a 9-deep breadth-first search.
+#
+module Gen1HoodEdges
+  class Mapper < Wukong::Streamer::Base
+    attr_accessor :head, :tail
+    def initialize options
+      self.head = options[:head]
+      self.tail = options[:tail]
+    end
+    def process rsrc, *nodes
+      yield [ nodes.last,  'i', nodes[0..-2] ] if (rsrc == self.head)
+      yield [ nodes.first, 'o', nodes[1..-1] ] if (rsrc == self.tail)
+    end
+  end
+
+  #
+  # Accumulate ( !!in memory!!) all inbound links onto middle node
+  #
+  # Then for each outbound link, loop over those inbound links and emit the
+  # triple (in, mid,out)
+  #
+  class Reducer < Wukong::Streamer::AccumulatingReducer
+    attr_accessor :paths_in, :out_rsrc
+    def initialize options
+      self.out_rsrc = options[:out_rsrc]
+    end
+    # clear the list of incoming paths
+    def start! *args
+      self.paths_in  = []
+    end
+    def accumulate mid, dir, *nodes
+      case dir
+      when 'i'
+        self.paths_in << nodes
+        if (self.paths_in.length % 1000 == 0) && (self.paths_in.length > 10000)
+          $stderr.puts ["Accumulating:", mid, self.paths_in.length].join("\t")
+        end
+      when 'o'
+        paths_in.each do |path_in|
+          yield [self.out_rsrc, path_in, mid, *nodes]
+        end
+      end
+    end
+    def finalize
+    end
+    def get_key mid, *_
+      mid
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :sort_fields => 2, :partition_fields => 1
+    end
+  end
+
+end
+
+# Execute the script
+Gen1HoodEdges::Script.new(
+  Gen1HoodEdges::Mapper,
+  Gen1HoodEdges::Reducer
+  ).run

data/examples/graph/gen_2paths.rb

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong'
+
+class Edge < Struct.new(:src, :dest)
+end
+
+class MultiEdge < Struct.new(
+    :src,           :dest,
+    :a_follows_b,   :b_follows_a,
+    :a_replies_b,   :b_replies_a,
+    :a_favorites_b, :b_favorites_a
+    )
+end
+
+module Gen1HoodEdges
+  class Mapper < Wukong::Streamer::Base
+    def process rsrc, src, dest
+      # next if (src.to_i == 0) || (dest.to_i == 0)
+      yield [ dest, 'i', src ]
+      yield [ src,  'o', dest]
+    end
+  end
+
+  #
+  # Accumulate ( !!in memory!!) all inbound links onto middle node
+  #
+  # Then for each outbound link, loop over those inbound links and emit the
+  # triple (in, mid,out)
+  #
+  class Reducer < Wukong::Streamer::AccumulatingReducer
+    attr_accessor :ins
+    def start! *args
+      self.ins  = []
+    end
+    def accumulate mid, dir, node
+      case dir
+      when 'i'
+        self.ins << node
+        if (self.ins.length % 1000 == 0) && (self.ins.length > 10000)
+          $stderr.puts ["Accumulating:", mid, self.ins.length].join("\t")
+        end
+      when 'o'
+        ins.each do |inn|
+          yield ['path_2', inn, mid, node]
+        end
+      end
+    end
+    def finalize
+    end
+    def get_key mid, *_
+      mid
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :sort_fields => 2, :partition_fields => 1
+    end
+  end
+
+end
+
+# Execute the script
+Gen1HoodEdges::Script.new(
+  Gen1HoodEdges::Mapper,
+  Gen1HoodEdges::Reducer
+  ).run

data/examples/graph/gen_multi_edge.rb

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong'
+require 'wukong/models/graph'; include Wukong::Models
+
+#
+# Takes any number of flavors of directed edge with the form
+#
+#   a_relatesto_b src_id dest_id [optional fields]
+#
+# and prepares a combined adjacency list.  You need to supply a model named
+# "MultiEdge" with members for each edge type.
+#
+# For instance, suppose you have a social network with edges like
+#
+#   a_follows_b   user_a_id  user_b_id
+#   a_messages_b  user_a_id  user_b_id  message_id date
+#   a_favorites_b user_a_id  user_b_id  message_id date
+#
+# Your MultiEdge class might look like
+#
+#   class MultiEdge < Struct(
+#     :src, :dest,
+#     :a_follows_b,   :b_follows_a,
+#     :a_messages_b,   :b_messages_a,
+#     :a_favorites_b, :b_favorites_a
+#     )
+#   end
+#
+# The row for a user pair who follows each other; with user_a #24601 messaging b
+# 57 times and favoriting 5 of user_b's messages; and user_b #8675309 messaging
+# 62 times and favoriting none, will emerge as (tab separated, with [blank]
+# indicating there is no text in that slot):
+#
+#   ...
+#   24601       8675309 1       1       57      62      5       [blank]
+#   ...
+#
+module GenMultiEdge
+  #
+  # Emit each relation as
+  #
+  #   src       dest    rel
+  #
+  # Canonicalizes the src and dest ids to 10-character, zero-padded strings.
+  # (Ten chars fits a 32-bit up-to-4-billion-and-change unsigned integer.)
+  # Discards all the ancillary crap except +src+, +dest+ and +rel+
+  #
+  class Mapper < Wukong::Streamer::Base
+    def process rsrc, src, dest, *_
+      # note that a_retweets_b_id matches here
+      m = /^a_([a-z]+)_b.*/.match(rsrc) or return
+      rel = m.captures.first
+      src = src.to_i ; dest = dest.to_i
+      return if ((src == 0) || (dest == 0))
+      yield ["%010d"%src,  "%010d"%dest, "a_#{rel}_b"]
+      yield ["%010d"%dest, "%010d"%src,  "b_#{rel}_a"]
+    end
+  end
+
+  #
+  # Aggregate all sightings of relations for each pair into
+  # a single combined
+  #
+  # Note that [a,b] and [b,a] /each/ have a listing, with the a->b and b<-a
+  # relations repeated for each.  That is, if there is an "a_messages_b"
+  # relation, you'll have edges
+  #
+  #    x        y       ...     a_messages_b(x,y)       b_messages_a(y,x)  ...
+  #    y        x       ...     a_messages_b(y,x)       b_messages_a(x,y)  ...
+  #
+  #
+  class Reducer < Wukong::Streamer::AccumulatingReducer
+    attr_accessor :multi_edge
+    def get_key src, dest, rel
+      [src, dest]
+    end
+    def start! *args
+      self.multi_edge = MultiEdge.new
+    end
+    def accumulate src, dest, rel
+      self.multi_edge[rel] ||= 0
+      self.multi_edge[rel]  += 1
+    end
+    def finalize
+      multi_edge.src, multi_edge.dest = key
+      yield self.multi_edge
+    end
+  end
+
+  #
+  # Sort on the first two keys: each @[src, dest]@ pair winds up at the same
+  # reducer.
+  #
+  class Script < Wukong::Script
+    def default_options
+      super.merge :sort_fields => 2
+    end
+  end
+
+  # Execute the script
+  Script.new(Mapper, Reducer).run
+end

data/examples/graph/gen_symmetric_links.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong'
+
+class Edge < Struct.new(:src, :dest)
+end
+
+class ASymmetricB < Edge
+end
+
+module Wukong::Streamer
+  class EdgeStreamer < Wukong::Streamer::Base
+    def recordize line
+      rsrc, src, dest, *_ = super(line)
+      [ASymmetricB.new(src.to_i, dest.to_i)]
+    end
+  end
+end
+
+module FindSymmetricLinks
+
+  class Mapper < Wukong::Streamer::EdgeStreamer
+    def process edge
+      yield edge.to_flat(false)
+      yield ASymmetricB.new(edge.dest, edge.src).to_flat(false)
+    end
+  end
+
+  #
+  #
+  class Reducer < Wukong::Streamer::Base
+    def stream
+      %x{/usr/bin/uniq -c}.split("\n").each do |line|
+        key_count, rsrc, src, dest, data = line.chomp.strip.split(/\s+/, 4)
+        next unless key_count.to_i == 2
+        next unless src.to_i < dest.to_i
+        emit [src, dest, data].compact
+      end
+    end
+  end
+
+  class Script < Wukong::Script
+    def default_options
+      super.merge :sort_fields => 3
+    end
+  end
+end
+
+# Execute the script
+Wukong::Script.new(
+  FindSymmetricLinks::Mapper,
+  FindSymmetricLinks::Reducer
+  ).run