wukong 1.5.4 → 2.0.0

data/examples/ignore_me/counting.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'wukong'
+
+require 'bloomfilter-rb'
+
+SIZE = 2**24
+
+class BucketCounter
+  def initialize(opts = {})
+    @opts = {
+      :size    => 100,
+      :server => {}
+    }.merge opts
+    @db   = ::Redis.new(@opts[:server])
+    @size = opts[:size]
+  end
+
+  def key_for val
+    (val.hash % @size)
+  end
+
+  def insert(val)
+    @db.incr(key_for(val))
+  end
+  alias :<< :insert
+
+  def delete(val)
+    if @db.decr(key_for(val)).to_i <= 0
+      @db.del(key_for(val))
+    end
+  end
+
+  def [](val)
+    @db.get(key_for(val)).to_i
+  end
+
+  def clear
+    @db.flushdb
+  end
+end
+
+bf     = BucketCounter.new(:size => 1_000, :server => {:host => 'localhost'})
+bf.clear
+counts = Hash.new{|h,k| h[k] = 0 }
+
+doc = File.read(__FILE__)
+doc.split(/\W+/).each do |word|
+  counts[word] += 1
+  bf << word
+end
+
+counts.keys.sort.each do |word|
+  puts [ bf[word] - counts[word], bf[word], counts[word], word.hash % SIZE, word ].join("\t")
+end

data/examples/ignore_me/grouper.rb

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'backports'
+require 'backports/1.8.8'
+require 'extlib'
+
+class Source
+  # include Enumerable
+  attr_reader :streamer
+
+  def recordize line
+    # line.strip.split("\t")
+    [line[0..5]]
+  end
+
+  def each *args
+    $stdin.each(*args) do |raw_record|
+      record = recordize(raw_record)
+      next if record.blank?
+      yield *record
+      break if raw_record =~ /end/
+    end
+  end
+end
+
+# def process_group group
+# end
+#
+
+class Streamer
+
+  def recordize line
+    [line[0..5]]
+  end
+
+  def each_group
+    while not $stdin.eof? do
+      Enumerator.new do |yielder|
+        $stdin.each do |line|
+        yield yielder
+          p yielder
+          break if line =~ /end/
+        end
+      end
+    end
+  end
+end
+
+foo = Streamer.new
+
+foo.each_group do |group|
+  puts "hi"
+  p group.each do |line|
+    p line.reverse
+  end
+  #   .map do |record|
+  #   1
+  # end
+end
+
+
+# i = 0
+# # s = source.new(Streamer.new)
+# $stdin.each do
+#   process_group do |output|
+#     puts output
+#   end
+#   $stderr.puts [Time.now, i] if (i += 1) % 10 == 0
+# end
+
+

data/examples/network_graph/adjacency_list.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
-$: << '/home/flip/ics/wukong/lib' # ENV['WUKONG_PATH']
-require 'wukong'
+$: << File.dirname(__FILE__)+'/../lib'
+require 'wukong/script'
 
 #
 # Given an adjacency pairs (from \t to) representation of a directed graph:

data/examples/network_graph/breadth_first_search.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
-$: << ENV['WUKONG_PATH']
-require 'wukong'
+$: << File.dirname(__FILE__)+'/../lib'
+require 'wukong/script'
 
 #
 # Use this script to do a Breadth-First Search (BFS) of a graph.
@@ -9,19 +9,18 @@ require 'wukong'
 #   ./make_paths --head=[path_in_key] --tail=[path_out_key] --out_rsrc=[combined_path_key]
 #
 # For example, given an edge list in the file '1path.tsv' that looks like
-#   1path	n1	n2
-#   1path       n1	n3
+#   1path       n1      n2
+#   1path       n1      n3
 #   ... and so forth ...
 # you can run
 #   for t in 1 2 3 4 5 6 7 8 9 ; do next=$((t+1)) ; time cat 1path.tsv "${t}path.tsv" | ./make_paths.rb --map --head="1path"  --tail="${t}path" | sort -u | ./make_paths.rb --reduce --out_rsrc="${next}path" | sort -u > "${next}path.tsv" ; done
 # to do a 9-deep breadth-first search.
 #
 module Gen1HoodEdges
-  class Mapper < Wukong::Streamer::Base
-    attr_accessor :head, :tail
-    def initialize options
-      self.head = options[:head]
-      self.tail = options[:tail]
+  class Mapper < Wukong::Streamer::RecordStreamer
+    def initialize
+      @head = Settings[:head]
+      @tail = Settings[:tail]
     end
     def process rsrc, *nodes
       yield [ nodes.last,  'i', nodes[0..-2] ] if (rsrc == self.head)
@@ -37,8 +36,8 @@ module Gen1HoodEdges
   #
   class Reducer < Wukong::Streamer::AccumulatingReducer
     attr_accessor :paths_in, :out_rsrc
-    def initialize options
-      self.out_rsrc = options[:out_rsrc]
+    def initialize
+      self.out_rsrc = Settings[:out_rsrc]
     end
     # clear the list of incoming paths
     def start! *args
@@ -63,17 +62,11 @@ module Gen1HoodEdges
       mid
     end
   end
-
-  class Script < Wukong::Script
-    def default_options
-      super.merge :sort_fields => 2, :partition_fields => 1
-    end
-  end
-
 end
 
 # Execute the script
-Gen1HoodEdges::Script.new(
+Wukong.run(
   Gen1HoodEdges::Mapper,
-  Gen1HoodEdges::Reducer
-  ).run
+  Gen1HoodEdges::Reducer,
+   :sort_fields => 2, :partition_fields => 1
+  )

data/examples/network_graph/gen_multi_edge.rb

@@ -2,7 +2,6 @@
 require 'rubygems'
 $: << File.dirname(__FILE__)+'/../../lib'
 require 'wukong'
-require 'wukong/models/graph'; include Wukong::Models
 
 #
 # Takes any number of flavors of directed edge with the form
@@ -88,17 +87,27 @@ module GenMultiEdge
       yield self.multi_edge
     end
   end
+end
 
-  #
-  # Sort on the first two keys: each @[src, dest]@ pair winds up at the same
-  # reducer.
-  #
-  class Script < Wukong::Script
-    def default_options
-      super.merge :sort_fields => 2
-    end
-  end
+Edge = TypedStruct.new(
+  [:src,              Integer],
+  [:dest,             Integer]
+  )
 
-  # Execute the script
-  Script.new(Mapper, Reducer).run
-end
+MultiEdge = TypedStruct.new(
+  [:src,              Integer],
+  [:dest,             Integer],
+  [:a_follows_b,      Integer],
+  [:b_follows_a,      Integer],
+  [:a_replies_b,      Integer],
+  [:b_replies_a,      Integer],
+  [:a_atsigns_b,      Integer],
+  [:b_atsigns_a,      Integer],
+  [:a_retweets_b,     Integer],
+  [:b_retweets_a,     Integer],
+  [:a_favorites_b,    Integer],
+  [:b_favorites_a,    Integer]
+  )
+
+# Execute the script
+Script.new(Mapper, Reducer, :sort_fields => 2).run

data/examples/pagerank/pagerank.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 $: << File.dirname(__FILE__)+'/../../lib'
-require 'wukong'
+require 'wukong/script'
 
 #
 #

data/examples/pagerank/pagerank_initialize.rb

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 $: << File.dirname(__FILE__)+'/../../lib'
-require 'wukong'
-require 'wukong/streamer/set_reducer'
+require 'wukong/script'
+require 'wukong/streamer/list_reducer'
 
 module PageRank
   class Script < Wukong::Script
@@ -15,10 +15,6 @@ module PageRank
     def map_command
       %Q{/usr/bin/cut -d"\t" -f2,3}
     end
-
-    def default_options
-      super.merge :extra_args => ' -jobconf io.sort.record.percent=0.25 '
-    end
   end
 
   #
@@ -28,18 +24,18 @@ module PageRank
   #
   class Reducer < Wukong::Streamer::ListReducer
     def accumulate src, dest
-      self.values << dest
+      @values << dest
     end
 
     # Emit src, initial pagerank, and flattened dests list
     def finalize
-      self.values = ['dummy'] if self.values.blank?
-      yield [key, 1.0, self.values.to_a.join(",")]
+      @values = ['dummy'] if @values.blank?
+      yield [key, 1.0, @values.to_a.join(",")]
     end
   end
 
   # Execute the script
-  Script.new(nil, PageRank::Reducer).run
+  Script.new(nil, PageRank::Reducer, :io_sort_record_percent => 0.25).run
 end
 
 

data/examples/sample_records.rb

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib'
 require 'rubygems'
-require 'wukong'
+require 'wukong/script'
+
+Settings.define :sampling_fraction, :type => Float, :required => true, :description => "floating-point number between 0 and 1 giving the fraction of lines to emit: at sampling_fraction=1 all records are emitted, at 0 none are."
 
 #
 # Probabilistically emit some fraction of record/lines
@@ -14,30 +15,19 @@ require 'wukong'
 class Mapper < Wukong::Streamer::LineStreamer
   include Wukong::Streamer::Filter
 
-  #
-  # floating-point number between 0 and 1 giving the fraction of lines to emit:
-  # at sampling_fraction=1 all records are emitted, at 0 none are.
-  #
-  # Takes its value from a mandatory command-line option
-  #
-  def sampling_fraction
-    @sampling_fraction ||= ( options[:sampling_fraction] && options[:sampling_fraction].to_f ) or
-      raise "Please supply a --sampling_fraction= argument, a decimal number between 0 and 1"
-  end
-
   #
   # randomly decide to emit +sampling_fraction+ fraction of lines
   #
   def emit? line
-    rand < self.sampling_fraction
+    rand < Settings.sampling_fraction
   end
 end
 
 #
 # Executes the script
 #
-Wukong::Script.new( Mapper,
+Wukong.run( Mapper,
   nil,
   :reduce_tasks => 0,
   :reuse_jvms   => true
-  ).run
+  )

data/examples/server_logs/apache_log_parser.rb

@@ -1,22 +1,7 @@
 #!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib'
 require 'rubygems'
-require 'wukong'
+require 'wukong/script'
 
-MONTHS = {
-  'Jan' => '01',
-  'Feb' => '02',
-  'Mar' => '03',
-  'Apr' => '04',
-  'May' => '05',
-  'Jun' => '06',
-  'Jul' => '07',
-  'Aug' => '08',
-  'Sep' => '09',
-  'Oct' => '10',
-  'Nov' => '11',
-  'Dec' => '12',
-}
 module ApacheLogParser
   class Mapper < Wukong::Streamer::LineStreamer
 
@@ -40,6 +25,7 @@ module ApacheLogParser
        \s\"([^\"]*)\"                   # referer             "http://infochimps.org/search?query=CAC"
        \s\"([^\"]*)\"                   # ua                  "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.0.16) Gecko/2009120208 Firefox/3.0.16"
       \z}x)
+    MONTHS = { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06', 'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12', }
 
     # Use the regex to break line into fields
     # Emit each record as flat line
@@ -49,14 +35,13 @@ module ApacheLogParser
       if m
         (ip, j1, j2,
           ts_day, ts_mo, ts_year,
-          ts_hour, ts_min, ts_sec, req_tz,
+          ts_hour, ts_min, ts_sec, tz,
           http_method, path, protocol,
           response_code, duration,
           referer, ua, *cruft) = m.captures
-        # DateTime.parse("#{datepart} #{timepart}").to_flat # this takes way too long
-        req_date = [ts_year, MONTHS[ts_mo], ts_day].join("")
-        req_time = [ts_hour, ts_min, ts_sec].join("")
-        yield [:logline, ip, req_date, req_time, http_method, protocol, path, response_code, duration, referer, ua, req_tz]
+        date = [ts_year, MONTHS[ts_mo], ts_day].join("")
+        time = [ts_hour, ts_min, ts_sec].join("")
+        yield [:logline, ip, date, time, http_method, protocol, path, response_code, duration, referer, ua, tz]
       else
         yield [:unparseable, line]
       end
@@ -65,7 +50,7 @@ module ApacheLogParser
   end
 end
 
-Wukong::Script.new(ApacheLogParser::Mapper, nil, :sort_fields => 7).run
+Wukong.run(ApacheLogParser::Mapper, nil, :sort_fields => 7)
 
 # 55.55.155.55 - - [04/Feb/2008:11:37:52 +0000] 301 "GET /robots.txt HTTP/1.1" 185 "-" "WebAlta Crawler/2.0 (http://www.webalta.net/ru/about_webmaster.html) (Windows; U; Windows NT 5.1; ru-RU)" "-"
 

data/examples/server_logs/breadcrumbs.rb

@@ -1,3 +1,29 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'wukong/script'
+
+class Logline < Struct.new(
+  :ip, :date, :time, :http_method, :protocol, :path, :response_code, :duration, :referer, :ua, :tz)
+
+  def page_type
+    case
+    when path =~ /\.(css|js)$/                  then :asset
+    when path =~ /\.(png|gif|ico)$/             then :image
+    when path =~ /\.(pl|s?html?|asp|jsp|cgi)$/  then :page
+    else                                             :other
+    end
+  end
+
+  def is_page?
+    page_type == :page
+  end
+
+  def day_hr
+    visit.date + visit.time[0..1]
+  end
+end
+
+
 #
 # Group all visitors, and then troll through all the pages they've visited
 # breaking each into distinct visits (where more than an [hour|day|whatever]
@@ -12,6 +38,11 @@
 #
 # where the partition key is visitor_id, and we sort by visitor_id and datetime.
 #
+class VisitorDatePath < Wukong::Streamer::StructStreamer
+  def process visit, *args
+    yield [visit.ip, visit.day_hr, visit.path]
+  end
+end
 
 #
 # Reducer:
@@ -34,3 +65,11 @@
 #     page_trails  <pagen>      <n_pages_in_visit> <duration> <timestamp> < page1,page2,... >
 #
 # to discover all trails passing through a given page.
+class VisitorDatePath < Wukong::Streamer::Reducer
+  def get_key ip, day_hr, path, *args
+    [ip, day_hr]
+  end
+  def process_group visit, *args
+    yield [visit.ip, visit.day_hr, visit.path]
+  end
+end

data/examples/server_logs/logline.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'wukong/script'
+
+class Logline < Struct.new(
+  :ip, :date, :time, :http_method, :protocol, :path, :response_code, :duration, :referer, :ua, :tz)
+
+  def page_type
+    case
+    when path =~ /\.(css|js)$/                  then :asset
+    when path =~ /\.(png|gif|ico)$/             then :image
+    when path =~ /\.(pl|s?html?|asp|jsp|cgi)$/  then :page
+    else                                             :other
+    end
+  end
+
+  def is_page?
+    page_type == :page
+  end
+end
+
+class PageFilter < Wukong::Streamer::StructStreamer
+  def process visit, *args
+    yield visit.ua if visit.
+  end
+end
+Wukong.run(PageFilter)

data/examples/size.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 $: << File.dirname(__FILE__)+'/../lib'
-require 'wukong'
+require 'wukong/script'
 
 module Size
   #
@@ -56,5 +56,6 @@ end
 # Execute the script
 Size::Script.new(
   nil,
-  Size::Reducer
+  Size::Reducer,
+  :reduce_tasks => 1
   ).run

data/examples/{binning_percentile_estimator.rb → stats/binning_percentile_estimator.rb}

@@ -1,10 +1,8 @@
 #!/usr/bin/env ruby
-
-require 'rubygems'
-require 'wukong'
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong/script'
 require 'wukong/streamer/count_keys'
 
-
 #
 # Ch3ck out dis moist azz code bitches!!
 #
@@ -70,14 +68,14 @@ class Reducer < Wukong::Streamer::AccumulatingReducer
     table << "TRSTRANK_TABLE = " << count_bin.inspect
     table.close
   end
-  
+
   #
-  # Return percentile of a given trstrank for a given follower bracket 
+  # Return percentile of a given trstrank for a given follower bracket
   #
   def percentile bin, rank
-    ((count_less_than(bin,rank) + 0.5*frequency_of(bin,rank))/ total_num(bin) )*100.0 
+    ((count_less_than(bin,rank) + 0.5*frequency_of(bin,rank))/ total_num(bin) )*100.0
   end
-  
+
   #
   # Return the count of values less than rank
   #
@@ -119,7 +117,7 @@ class Reducer < Wukong::Streamer::AccumulatingReducer
     big_list.uniq.sort{|x,y| x.first <=> y.first}
   end
 
-  
+
   #
   # Nothing to see here, move along
   #
@@ -132,11 +130,11 @@ class Reducer < Wukong::Streamer::AccumulatingReducer
     num.times do |i|
       x = pair1.first + (i+1).to_f*dx
       y = m*x + b
-      points << [x,y] 
+      points << [x,y]
     end
     points                                                       # return an array of pairs
   end
-  
+
 end
 
 Wukong::Script.new(Mapper,Reducer).run

data/examples/{rank_and_bin.rb → stats/rank_and_bin.rb}

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib'
-require 'wukong'
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong/script'
 require 'wukong/streamer/rank_and_bin_reducer'
 
 #

data/examples/stupidly_simple_filter.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong'
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'wukong/script'
 
 # Run as (local mode)
 #
@@ -15,14 +15,14 @@ require 'wukong'
 #   cat input.tsv | ./examples/stupidly_simple_filter.rb --map input.tsv | more
 #
 
-#
-# A very simple mapper -- looks for a regex match in one field,
-# and emits the whole record if the field matches
-#
-class GrepMapper < Wukong::Streamer::RecordStreamer
-
+class Mapper < LineStreamer
+  include Filter
   MATCHER = %r{(ford|mercury|saab|mazda|isuzu)}
 
+  #
+  # A very simple mapper -- looks for a regex match in one field,
+  # and emits the whole record if the field matches
+  #
   #
   # Given a series of records like:
   #
@@ -31,13 +31,10 @@ class GrepMapper < Wukong::Streamer::RecordStreamer
   #
   # emits only the lines matching that regex
   #
-  def process rsrc, id, timestamp, text, *rest
-    yield [rsrc, id, timestamp, text, *rest] if line =~ MATCHER
+  def emit? line
+    MATCHER.match line
   end
 end
 
 # Execute the script
-Wukong::Script.new(
-  GrepMapper,
-  nil
-  ).run
+Wukong.run(Mapper)