ul-wukong 4.1.0

data/bin/wu-local

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'wukong'
+Wukong::Local::LocalRunner.run

data/bin/wu-plus

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+sum   = 0.0
+lines = 0
+$stdin.each do |n|
+  sum   += n.to_f
+  lines += 1
+end
+puts "%15d\t%15d"%[sum, lines]

data/bin/wu-source

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'wukong'
+require 'wukong/source'
+Wukong::Source::SourceRunner.run

data/bin/wu-sum

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'wukong'
+require 'wukong/streamer/summing_reducer'
+
+#
+#
+class Summer < Wukong::Streamer::SummingReducer
+  attr_accessor :sample_line
+
+  def initialize *args
+    self.summing_elements = [0]
+    super *args
+  end
+
+  def start! *args
+    self.sample_line = args
+    super *args
+  end
+
+  def get_key *fields
+    fields.values_at(2,3)
+  end
+
+  def finalize
+    summing_elements.each{|idx| sample_line[idx] = sums[idx]}
+    yield sample_line
+  end
+end
+
+Wukong::Script.new(Summer, nil).run

data/diagrams/wu_local.dot

@@ -0,0 +1,39 @@
+digraph WuLocalControlFlow {
+	size ="100,100";
+	stdin [label=<
+	       <TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="4">
+	       <TR><TD><FONT FACE="BOLD" POINT-SIZE="20">STDIN</FONT></TD></TR>
+	       <TR><TD><FONT POINT-SIZE="10">The line of input text</FONT></TD></TR>
+	       <TR><TD><FONT FACE="MONOSPACE">Shall I compare thee to a summers day?<BR/>Thou art more lovely and more temperate<BR/>...</FONT></TD></TR>
+	       
+	       </TABLE>>];
+	command [shape=diamond,label=<
+		 <TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="4">
+		 <TR><TD><FONT FACE="BOLD" POINT-SIZE="20">Command</FONT></TD></TR>
+		 <TR><TD><FONT POINT-SIZE="10">A UNIX process launched on the command line</FONT></TD></TR>
+		 <TR><TD><FONT FACE="MONOSPACE">wu-local word_counter</FONT></TD></TR>
+		 </TABLE>>];
+	runner [shape=box,label=<
+		<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="4">
+		<TR><TD><FONT FACE="BOLD" POINT-SIZE="20">Runner</FONT></TD></TR>
+		<TR><TD><FONT POINT-SIZE="10">Loads plugins and code, configures and resolves settings, boots plugins, validates command line, then runs.</FONT></TD></TR>
+		<TR><TD><FONT FACE="MONOSPACE">Wukong::Local::LocalRunner</FONT></TD></TR>
+		</TABLE>>];
+	driver [shape=box,label=<
+		<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="4">
+		<TR><TD><FONT FACE="BOLD" POINT-SIZE="20">Driver</FONT></TD></TR>
+		<TR><TD><FONT POINT-SIZE="10">Passes input to processor, handles output</FONT></TD></TR>
+		<TR><TD><FONT FACE="MONOSPACE">Wukong::Local::StdioDriver</FONT></TD></TR>
+		</TABLE>>];
+	stdout [label=<
+		<TABLE BORDER="0" CELLBORDER="0" CELLSPACING="0" CELLPADDING="4">
+		<TR><TD><FONT FACE="BOLD" POINT-SIZE="20">STDOUT</FONT></TD></TR>
+		<TR><TD><FONT POINT-SIZE="10">The resulting output lines</FONT></TD></TR>
+		<TR><TD><FONT FACE="MONOSPACE">8<BR/>7<BR/>...</FONT></TD></TR>
+		</TABLE>>];
+
+		command -> runner [label="Implemented By"];
+		runner -> driver [label="Instantiates"];
+		stdin -> driver [label="Reads 1 Line"];
+		driver -> stdout [label="Writes N Lines"];
+	}

data/examples/Gemfile

@@ -0,0 +1,38 @@
+source :rubygems
+
+gem "configliere",        '~> 0.4'
+gem "multi_json",         '>= 1.3.6'
+gem "gorillib",           '>= 0.4.2'
+gem "uuidtools"
+gem "eventmachine"
+gem "log4r"
+
+group :examples do
+  gem "forgery"
+  gem "nokogiri"
+  gem "sanitize"
+  gem "addressable"
+  gem "forgery"
+  gem "crack"
+  gem "oj"
+  gem "activesupport"
+end
+
+group :development do
+  gem "bundler",            '~> 1.1'
+  gem "rake",               '>= 0.9'
+  gem "rspec",              '>= 2.8'
+  gem "guard",              '>= 1.0'
+  gem "guard-rspec",        '>= 0.6'
+  gem "simplecov",          '>= 0.5'
+  gem "pry"
+  gem "ap"
+  gem "ruby-progressbar"
+end
+
+group :docs do
+  gem "yard"
+  gem "redcarpet"
+  gem "addressable"
+  gem "htmlentities"
+end

data/examples/README.md

@@ -0,0 +1,9 @@
+
+
+### Geographic data 
+
+
+
+### Munging
+
+

data/examples/basic/string_reverser.rb

@@ -0,0 +1,23 @@
+Wukong.processor(:string_reverser) do
+
+  def setup
+    log.info("Inside the setup method")
+    @count = 0
+    EM.add_periodic_timer(10){ notify('metrics', count: @count) }
+  end
+
+  def process(record) 
+    @count += 1
+    yield record.reverse
+    yield nil
+  end
+
+  def finalize
+    log.info("Finalizing flow")
+  end
+  
+  def stop
+    log.info("Inside the stop method")
+  end
+
+end

data/examples/basic/tiny_count.rb

@@ -0,0 +1,8 @@
+mapper{|i| i |
+  ->(l){ l.downcase.split(/\W+/) } |
+  select{|w| w.size>2 } }
+
+reducer{|i| i |
+  counter |
+  map(&:reverse) |
+  to_tsv }

data/examples/basic/word_count/accumulator.rb

@@ -0,0 +1,26 @@
+Wukong.processor(:accumulator) do
+  attr_accessor :count, :current
+
+  def reset!() @current = nil ; @count = 0 ; end
+
+  def report_then_reset!(&blk)
+    yield [current, count].join("\t") unless current.nil?
+    reset!
+  end
+
+  def accumulate(word, seen)
+    @current = word if @current.nil?
+    @count  += seen
+  end
+
+  def process(pair, &blk)
+    word, seen = pair.split("\t")
+    report_then_reset!(&blk) unless word == current
+    accumulate(word, seen.to_i)
+  end
+  
+  def finalize(&blk)
+    report_then_reset!(&blk)
+  end
+  
+end

data/examples/basic/word_count/tokenizer.rb

@@ -0,0 +1,13 @@
+Wukong.processor(:tokenizer) do
+
+  field :min_length, Integer, :default => 1
+
+  def process(record)
+    words   = record.downcase.strip.split(/\W/)
+    lengthy = words.select{ |word| word.length >= min_length }
+    lengthy.each do |word|
+      yield [ word, 1 ].join("\t")
+    end
+  end
+
+end

data/examples/basic/word_count/word_count.rb

@@ -0,0 +1,6 @@
+require_relative 'tokenizer'
+require_relative 'accumulator'
+
+Wukong.dataflow(:word_count) do
+  tokenizer(min_length: 3) > sort(label: 'first_sort') > accumulator > sort(on: 1, numeric: true, reverse: true)
+end

data/examples/dataflow/scraper_macro_flow.rb

@@ -0,0 +1,28 @@
+require 'wukong/widgets/sinks/hbase_record_sink.rb'
+
+Wukong.chain(:friend_graph) do
+  tail(:scrapables) do
+    directory   'scrapables/ids-%{t:ymd}.tsv'
+  end
+
+  requester = decorator('tw_requester.rb') do
+    input  :scrape_url,       Url
+    output :raw_json_request, JsonString
+    config do
+      define :request_types, :default => [:follower_ids, :friend_ids], :doc => 'which requests to make: follower_ids, user_timeline, etc'
+    end
+  end
+
+  retriable_requester = retriable do
+    with        :timeouts => [1,2,3]
+    on_failure  :sleep
+    guest       requester
+  end
+
+  tail(:scrapables)> retriable_requester > processor('tw_parse.rb') > hbase_record_sink
+end
+
+Wukong.processor(:tw_parse) do
+  def process
+  end
+end

data/examples/deploy_pack/Gemfile

@@ -0,0 +1,6 @@
+# This is a minimal Gemfile that lets Wukong boot up (but not work).
+gem 'configliere'
+gem 'gorillib'
+gem 'multi_json'
+gem 'log4r'
+gem 'eventmachine'

data/examples/deploy_pack/README.md

@@ -0,0 +1,6 @@
+This directory contains the skeleton of a Wukong deploy pack.  It's
+used to test the behavior of Wukong's command line tools within such a
+context.
+
+The Gemfile here contains the minimal subset of gems that let Wukong
+boot.

data/examples/deploy_pack/app/processors/string_reverser.rb

@@ -0,0 +1,5 @@
+Wukong.processor(:string_reverser) do
+  def process string
+    yield string.reverse
+  end
+end

data/examples/deploy_pack/config/environment.rb

@@ -0,0 +1 @@
+require_relative("../app/processors/string_reverser.rb")

data/examples/dsl/dataflow/fibonacci_series.rb

@@ -0,0 +1,101 @@
+require 'wukong/widget/many_to_many'
+require 'gorillib/enumerable/sum'
+
+#
+# An example dataflow --
+#
+
+Wukong.processor(:delay_buffer) do
+  attr_accessor :queue
+  field :delay, Integer, position: 0, doc: "number of records to hold in buffer"
+
+  def process(rec)
+    queue << rec
+    emit(next_item) if ready?
+  end
+
+  def next_item
+    queue.shift
+  end
+
+  # true if there are records at the end of the delay stage
+  def ready?
+    warn "Hmm, too many records in queue: #{queue}" if queue.size > delay+1
+    queue.size > delay
+  end
+
+  # resets to an empty state
+  def setup(*)
+    super
+    @queue = Array.new
+  end
+
+  # emits all remaining elements of the queue
+  def stop
+    queue.each{|rec| emit(rec) }
+    super
+  end
+end
+
+class Wukong::Batcher < Wukong::Processor
+  register_action
+  include Hanuman::Slottable
+  include Hanuman::OutputSlotted
+
+  attr_accessor :queues
+  consume :n_1,    Integer, doc: "n-1'th value: the one just emitted"
+  consume :tictoc, Integer, doc: "input to drive flow"
+  consume :n_2,    Integer, doc: "n-2'nd value: the one before the one just emitted"
+
+  # resets to an empty state, calls super
+  def initialize(*)
+    super
+    @queues = Hash.new{|h,k| h[k] = Array.new } # autovivifying
+  end
+
+  def process_input(channel, rec)
+    queues[channel] << rec
+    emit(next_item) if ready?
+  end
+
+  def next_item
+    queues.map{|_, queue| queue.shift }
+  end
+
+  # true if there is at least one record in each queue
+  def ready?
+    inslots.values.all?{|inslot| queues[inslot.name].length > 0 }
+  end
+end
+
+Wukong.chain(:fibonacci_series) do
+
+  delay_buffer(1, name: :my_delay)
+
+  # * I don't want to have to name everything
+  #   - are few/some/most things named?
+  # * I must be able to have the same stage type on a graph more than once
+  # * If naming things is a general case, I want it to
+  #   - be clean, and for it to
+  #   - not cause a ruckus when stage type has its own args
+  #
+
+  batcher(name: :feedback) >
+    map(name: :summer, &:sum) >
+    many_to_many(name: :fibonacci_n)
+
+  spew(6, item: 0, name: :ticker) > feedback.tictoc
+
+  fibonacci_n          > feedback.n_1
+  fibonacci_n          > output
+  fibonacci_n > :delay > feedback.n_2
+
+  # preload the feedback buffer
+  feedback.n_1.process(0)
+  feedback.n_2.process(0)
+  feedback.n_2.process(1)
+end
+
+# Wukong.dataflow(:dump) do
+#   stdout << Wukong.dataflow(:fibbonaci_series).out
+# end

data/examples/dsl/dataflow/scraper_macro_flow.rb

@@ -0,0 +1,28 @@
+require 'wukong/widgets/sinks/hbase_record_sink.rb'
+
+Wukong.chain(:friend_graph) do
+  tail(:scrapables) do
+    directory   'scrapables/ids-%{t:ymd}.tsv'
+  end
+
+  requester = decorator('tw_requester.rb') do
+    input  :scrape_url,       Url
+    output :raw_json_request, JsonString
+    config do
+      define :request_types, :default => [:follower_ids, :friend_ids], :doc => 'which requests to make: follower_ids, user_timeline, etc'
+    end
+  end
+
+  retriable_requester = retriable do
+    with        :timeouts => [1,2,3]
+    on_failure  :sleep
+    guest       requester
+  end
+
+  tail(:scrapables)> retriable_requester > processor('tw_parse.rb') > hbase_record_sink
+end
+
+Wukong.processor(:tw_parse) do
+  def process
+  end
+end

data/examples/dsl/dataflow/simple.rb

@@ -0,0 +1,12 @@
+require File.expand_path('../examples_helper', File.dirname(__FILE__))
+
+Wukong.dataflow(:simple) do
+  doc <<-DOC
+    A stupidly simple dataflow: reverses each input string
+  DOC
+
+  file_source(Pathname.path_to(:data, 'text/jabberwocky.txt')) >
+    map{|str| str.reverse } >
+    file_sink(Pathname.path_to(:tmp, 'dataflow/simple_output.rb'))
+
+end

data/examples/dsl/dataflow/telegram.rb

@@ -0,0 +1,45 @@
+require File.expand_path('../examples_helper', File.dirname(__FILE__))
+
+# "Telegram Problem"
+#
+# The "Telegram Problem", originally described by Peter Naur:
+# * accepts lines of text
+# * generates output lines that are shortern than a given length (or contain only one word)
+# * without splitting any of the words in the text (if a word is longer than the line width, emit it on its own line).
+
+Wukong.processor :recompose do
+  field :break_length, Integer
+  attr_accessor :line
+
+  def initialize(*) super; @line = "" ; end
+
+  def process(word)
+    if word == "" then flush! ; emit("") ; return ; end
+    flush! if "#{line} #{word}".lstrip.length > break_length
+    if word.length >= break_length
+      emit word
+    else
+      line << " " << word
+    end
+  end
+
+  def flush!
+    emit line[1..-1] unless line.blank?
+    self.line = ""
+  end
+
+  def stop
+    flush!
+  end
+end
+
+ExampleUniverse.dataflow(:telegram) do
+  input   :default, file_source(Pathname.path_to(:data, 'text/rectification_of_names.txt'))
+  output  :dump,    file_sink(  Pathname.path_to(:tmp,  'output/dataflow/telegram/names.txt'))
+
+  input(:default) >
+    map{|line| line.blank? ? [""] : line.strip.split(/\s+/m) } >
+    flatten >
+    recompose(:break_length => 80) >
+    output(:dump)
+end