ul-wukong 4.1.0

data/examples/serverlogs/visit_paths/serverlogs-04-page_page_edges-full.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+require_relative './common'
+
+class BreadcrumbsMapper < Wukong::Streamer::ModelStreamer
+  self.model_klass = Logline
+  def process visit, *args
+    # return unless Settings.page_types.include?(visit.page_type)
+    yield [visit.ip, visit.requested_at.to_i, visit.path]
+  end
+end
+
+class BreadcrumbEdgesReducer < Wukong::Streamer::Reducer
+  def get_key(ip, itime, path)
+    [ip]
+  end
+  def start!(*args)
+    @paths = Set.new
+    super
+  end
+  def accumulate ip, itime, path
+    @paths << path
+  end
+
+  # for each pair of paths, emit the edge in both directions
+  def finalize
+    @paths = @paths.to_a
+    while @paths.present?
+      from = @paths.shift
+      @paths.each do |into|
+        yield [key, from, into]
+        yield [key, into, from]
+      end
+    end
+  end
+end
+
+
+Wukong.run( BreadcrumbsMapper, BreadcrumbEdgesReducer, :sort_fields => 2 )

data/examples/splitter.rb

@@ -0,0 +1,94 @@
+require 'wukong'
+
+module Verbose
+
+  def verbose?
+  end
+  
+  def setup
+    # log.info("Setting up #{label}")
+  end
+
+  def finalize
+    # log.info("Finalizing #{label}")
+  end
+end
+  
+Wukong.processor(:upcaser) do
+  include Verbose
+  def process(string)
+    # log.info("#process #{string}")
+    yield string.upcase
+  end
+end
+
+Wukong.processor(:downcaser) do
+  include Verbose
+  def process(string)
+    # log.info("#process #{string}")
+    yield string.downcase
+  end
+end
+
+Wukong.processor(:tokenizer) do
+  include Verbose
+  def process string
+    # log.info("#process #{string}")
+    string.split.each { |token| yield token }
+  end
+end
+
+Wukong.processor(:stripper) do
+  include Verbose
+  def process(string)
+    # log.info("#process #{string}")
+    yield string.gsub(/[^\w\s]/,'')
+  end
+end
+
+Wukong.processor(:devoweler) do
+  include Verbose
+  def process(string)
+    # log.info("#process #{string}")
+    yield string.gsub(/[aeiou]/i,'')
+  end
+end
+
+# stripper  = Wukong.registry.retrieve(:stripper)
+# tokenizer = Wukong.registry.retrieve(:tokenizer)
+# upcaser   = Wukong.registry.retrieve(:upcaser)
+# downcaser = Wukong.registry.retrieve(:downcaser)
+# devoweler = Wukong.registry.retrieve(:devoweler)
+
+# Splitter = Class.new(Wukong::Dataflow)
+# builder = Wukong::DataflowBuilder.receive({label: :splitter,
+#                                             for_class: Splitter,
+#                                             stages: {
+#                                               stripper:  stripper,
+#                                               tokenizer: tokenizer,
+#                                               upcaser:   upcaser,
+#                                               downcaser: downcaser,
+#                                               devoweler: devoweler,
+#                                             },
+#                                             links: [
+#                                                     Hanuman::LinkFactory.connect(:simple, :stripper, :tokenizer),
+#                                                     Hanuman::LinkFactory.connect(:simple, :tokenizer, :upcaser),
+#                                                     Hanuman::LinkFactory.connect(:simple, :tokenizer, :downcaser),
+#                                                     Hanuman::LinkFactory.connect(:simple, :upcaser, :devoweler),
+#                                                    ]})
+
+# builder.extract_links!
+# Splitter.set_builder(builder)
+# Splitter.register
+
+Wukong.dataflow(:splitter) do
+  stripper | tokenizer |
+    [
+     upcaser | devoweler |
+     [
+      regexp | count,
+      identity
+     ],
+     downcaser | reject { |word| word == 'hell' }
+    ]
+end

data/examples/string_reverser.rb

@@ -0,0 +1,7 @@
+# A simple processor in its own little file.
+class StringReverser < Wukong::Processor
+  def process line
+    yield line.reverse
+  end
+  register(:string_reverser)
+end

data/examples/text/pig_latin/pig_latinizer.rb

@@ -0,0 +1,35 @@
+require File.expand_path('../examples_helper', File.dirname(__FILE__))
+
+Wukong.processor :pig_latinize do
+
+  CONSONANTS = "bcdfghjklmnpqrstvwxz"
+  UPPERCASE  = /^[A-Z]/
+
+  # Regular expression to identify the parts of a pig-latin-izable word
+  PIG_LATIN_WORD_RE = %r{
+    \b                  # word boundary
+    ([#{CONSONANTS}]*)  # all initial consonants
+    ([\w\']+)           # remaining word characters
+  }xi
+
+  def latinize(line)
+    line.gsub(PIG_LATIN_WORD_RE) do
+      init, rest = [$1, $2]
+      init = 'w'       if init.blank?
+      rest.capitalize! if init =~ UPPERCASE
+      "#{rest}#{init.downcase}ay"
+    end
+  end
+
+  def process(line)
+    emit latinize(line)
+  end
+
+end
+
+ExampleUniverse.dataflow(:pig_latin) do
+  set_input  :default, file_source(Pathname.path_to(:data, 'text/gift_of_the_magi.txt'))
+  set_output :default, file_sink(  Pathname.path_to(:tmp, 'text/pig_latin/gift_of_the_magi.txt'))
+
+  input(:default) > pig_latinize > output(:default)
+end

data/examples/text/pig_latin/pig_latinizer_widget.rb

@@ -0,0 +1,16 @@
+Wukong.dataflow(:pig_latinizer) do
+  doc <<-DOC
+    Accepts plaintext documents posted to its HTTP listener,
+    translates it into pig latin, and archives both the
+    translated and original texts into a mysql database
+  DOC
+
+  input  :raw_texts,       http_listener(:port => 8300)
+  output :original_texts,  mysql_sink
+  output :latinized_texts, mysql_sink
+
+  input(:raw_texts) > many_to_many([
+      :original_texts,
+      pig_latinizer > :latinized_texts
+    ])
+end

data/examples/text/regional_flavor/README.md

@@ -0,0 +1,14 @@
+# Find the Regional Flavor of topics using Geolocated Wikipedia Articles
+
+(Chapter 1 of "Big Data for Chimps")
+
+1. article -> wordbag
+2. join on page data to get geolocation
+3. use pagelinks to get larger pool of implied geolocations
+4. turn geolocations into quadtile keys
+5. aggregate topics by quadtile
+6. take summary statistics aggregated over term and quadkey
+7. combine those statistics to identify terms that occur more frequently than the base rate would predict
+8. explore and validate the results
+9. filter to find strongly-flavored words, and other reductions of the data for visualization
+

data/examples/text/regional_flavor/article_wordbags.pig

@@ -0,0 +1,39 @@
+%declare wukong_dir '/Users/flip/ics/core/wukong'
+%declare data_dir   '$wukong_dir/data'
+%declare dsfp_dir   '/Users/flip/ics/data_science_fun_pack'
+;
+
+register '$dsfp_dir/pig/varaha/target/varaha-1.0-SNAPSHOT.jar';
+register '$dsfp_dir/pig/varaha/lib/mallet-2.0.7-RC2.jar';
+register '$dsfp_dir/pig/varaha/lib/trove-2.0.4.jar';
+register '$dsfp_dir/pig/varaha/lib/lucene-core-3.1.0.jar';
+register '$dsfp_dir/pig/varaha/lib/pygmalion-1.1.0-SNAPSHOT.jar';
+register '$dsfp_dir/pig/pigsy/target/pigsy-2.1.0-SNAPSHOT.jar';
+register '$dsfp_dir/pig/datafu/dist/datafu-0.0.6-SNAPSHOT.jar';
+
+define JsonStrToString pigsy.pig.json.JsonStrToString();
+define TokenizeText    varaha.text.TokenizeText();
+define LDATopics       varaha.topic.LDATopics();
+define RangeConcat     org.pygmalion.udf.RangeBasedStringConcat('0', ' ');
+
+-- Load the markup-stripped wikipedia text
+torture_strings = LOAD '$data_dir/helpers/torture/string_handling_test.tsv' AS (
+  desc:chararray, len:int, bytesize:int, str:chararray, has_str:int, jsonized_str:chararray,
+  escaped_chars:chararray, escaped_bytes:chararray, chars_list:chararray, bytes_list:chararray
+  );
+
+-- Generate a random integer between 0 and n
+decoded = FOREACH torture_strings {
+  unjsonized_str = JsonStrToString(jsonized_str);
+  is_str_equal   = (((str == unjsonized_str) OR (has_str == 0)) ? 1 : 0);
+  is_len_equal   = (  len == SIZE(unjsonized_str) ? 1 : 0  );
+  GENERATE
+    is_str_equal,
+    is_len_equal,
+    str AS str,
+    unjsonized_str AS unjsonized_str
+    ;
+};
+
+-- take a dump on your terminal window
+DUMP decoded;

data/examples/text/regional_flavor/j01-article_wordbags.rb

@@ -0,0 +1,4 @@
+
+#
+# Summarize each article as a wordbag.
+#

data/examples/text/regional_flavor/simple_pig_script.pig

@@ -0,0 +1,27 @@
+%declare wukong_dir '/Users/flip/ics/core/wukong'
+%declare data_dir   '$wukong_dir/data'
+%declare dsfp_dir   '/Users/flip/ics/data_science_fun_pack'
+;
+
+register '$dsfp_dir/pig/datafu/dist/datafu-0.0.6-SNAPSHOT.jar';
+register '$dsfp_dir/pig/pigsy/target/pigsy-2.1.0-SNAPSHOT.jar';
+
+define RandInt         datafu.pig.numbers.RandInt();
+define ConcatBag       com.infochimps.hadoop.pig.ConcatBag();
+
+-- Load the data: the integers from 0 .. 1023
+ones = LOAD '$data_dir/helpers/numbers/integers-1ki.tsv' AS (val:int);
+
+-- Generate a random integer between 0 and n
+rands        = FOREACH ones  GENERATE val, RandInt(0,val) AS rand_val;
+rand_str_g   = GROUP rands ALL;
+rand_str_s   = FOREACH rand_str_g  {
+  joined_str = ConcatBag(rands.(rand_val));
+  len        = SIZE(joined_str);
+  GENERATE joined_str AS joined_str, len AS len;
+};
+
+-- take a dump on your terminal window
+STORE rand_str_s INTO '/tmp/dump';
+
+DESCRIBE rand_str_s;

data/examples/twitter.rb

@@ -0,0 +1,5 @@
+Wukong.dataflow(:twitter) do
+  from_json | reject { |obj| obj["delete"]  } |
+    [
+     
+end

data/lib/hanuman.rb

@@ -0,0 +1,36 @@
+require 'gorillib/some'
+require 'gorillib/model'
+
+require 'hanuman/registry'
+require 'hanuman/link'
+require 'hanuman/stage'              
+require 'hanuman/graph'
+require 'hanuman/tree'
+
+module Hanuman
+  module Shortcuts
+    
+    def builder_shortcut(builder_type, label, *args, &blk)
+      if GlobalRegistry.registered?(label)
+        builder = GlobalRegistry.retrieve(label)
+      else
+        builder = builder_type.receive(label: label)
+      end
+      GlobalRegistry.decorate_with_registry(builder) if builder.is_a?(GraphBuilder)
+      builder.define(*args, &blk)
+    end
+    
+    def add_shortcut_method_for(method_name, builder_type)
+      self.define_singleton_method(method_name){ |label, *args, &blk| builder_shortcut(builder_type, label, *args, &blk) }
+    end
+
+    def registry() Hanuman::GlobalRegistry ; end    
+    
+  end
+
+  extend Hanuman::Shortcuts
+  
+  add_shortcut_method_for(:stage, StageBuilder)
+  add_shortcut_method_for(:graph, GraphBuilder)
+
+end

data/lib/hanuman/graph.rb

@@ -0,0 +1,97 @@
+module Hanuman
+
+  module GraphInstanceMethods
+    def each_stage &block
+      stages.values.each(&block)
+    end
+
+    def descendents stage=nil
+      links.find_all do |link|
+        stage ? link.from == stage.label : true
+      end.map(&:into).uniq.map { |label| stages[label] }.compact
+    end
+
+    def ancestors stage=nil
+      links.find_all do |link|
+        stage ? link.into == stage.label : true
+      end.map(&:from).uniq.map { |label| stages[label] }.compact
+    end
+
+    def add_stage stage
+      stages[stage.label] = stage
+    end
+
+    def has_link? from, into
+      links.detect { |link| link.from == from.label && link.into == into.label } ? true : false
+    end
+
+    def add_link type, from, into
+      add_stage(from)
+      add_stage(into)
+      self.links << Hanuman::LinkFactory.connect(type, from.linkable_name(:in), into.linkable_name(:out))
+    end
+  end
+
+  class Graph < Stage
+    include GraphInstanceMethods
+    
+    field :stages, Hash,  :default => {}
+    field :links,  Array, :default => []
+  end
+
+  class GraphBuilder < StageBuilder
+
+    include GraphInstanceMethods
+    
+    field :stages, Hash,  :default => {}
+    field :links,  Array, :default => []
+
+    def define(&blk)     
+      graph = for_class || define_class(label)
+      self.instance_eval(&blk) if block_given?
+      extract_links!
+      graph.register
+    end
+
+    def build(options = {})
+      attrs  = serialize
+      stages = attrs.delete(:stages).inject({}){ |hsh, (name, builder)| hsh[name] = builder.build(stage_specific_options(name, options)) ; hsh }
+      for_class.receive attrs.merge(stages: stages)
+    end
+
+    def stage_specific_options(stage, options)
+      scope = options.delete(stage) || {}
+      options.merge(scope)
+    end
+
+    def namespace() Hanuman::Graph ; end
+
+    def handle_dsl_arguments_for(stage, *args, &blk)
+      options = args.extract_options!
+      stage.merge!(options)
+      stage
+    end
+
+    def extract_links!
+      self.links.replace([])
+      stages.each_pair{ |name, builder| links << builder.links }
+      links.flatten!
+    end
+    
+    def serialize
+      attrs = attributes
+      args  = attrs.delete(:args)
+      attrs.delete(:for_class)
+      attrs.merge(args)      
+    end
+
+    def clone
+      cloned_attrs  = Hash[ serialize.select{ |key, val| key != :stages }.map{ |key, val| dup_key = key.dup rescue key ; dup_val = val.dup rescue val ; [ dup_key, dup_val ] } ]
+      cloned_links  = links.map{ |link| link.dup }
+      cloned_stages = Hash[ stages.map{ |stage| stage.clone } ]
+      self.class.receive(cloned_attrs.merge(links: cloned_links).merge(stages: cloned_stages).merge(for_class: for_class))
+    end
+
+  end
+    
+end

data/lib/hanuman/graphvizzer.rb

@@ -0,0 +1,206 @@
+require 'graphviz'
+  
+module Hanuman
+  class Graphvizzer
+
+    attr_accessor :hanuman_graph, :visual
+    
+    def initialize(hanuman_graph)
+      @hanuman_graph = hanuman_graph
+      @visual = GraphViz.new(@hanuman_graph.label, :type => :digraph)
+    end
+
+    def create
+      hanuman_graph.directed_sort.each{ |stage| visual.add_nodes(stage.to_s) }
+      hanuman_graph.links.each{ |link| visual.add_edges(link.from.to_s, link.into.to_s) }
+      visual.output(:png => 'test.png')
+    end
+  
+  end
+end
+
+# module Hanuman
+#   module Graphvizzer
+#     include Gorillib::Builder
+
+#     class Item
+#       include Gorillib::Builder
+#       alias_method :configurate, :receive!
+
+#       field :name,  Symbol
+#       field :label, String, :default => ->{ name }
+#       field :owner, Item
+
+#       def initialize(attrs={}, &block)
+#         receive!(attrs, &block)
+#       end
+
+#       def depth
+#         owner.depth + 1
+#       end
+
+#       def indent(adj=0)
+#         "  " * (depth + adj)
+#       end
+
+#       def quote(str)
+#         return str if str.to_s.include?('"')
+#         %Q{"#{str}"}
+#       end
+
+#       def line(str, attrs={}, term=';')
+#         if attrs.empty?
+#           attr_strs = ''
+#         else
+#           width = 40 - indent.length
+#           str = "%-#{width}s" % str
+#           attr_strs = attrs.map{|attr, val| attrib(attr, val) }
+#           attr_strs = "[ #{attr_strs.join(",")} ]"
+#         end
+#         [indent, str, attr_strs, term].join
+#       end
+
+#       def attrib(attr, val)
+#         "#{attr} = #{val}"
+#       end
+
+#       def brace(str)
+#         "#{indent}#{str} {"
+#       end
+#       def close_brace
+#         "#{indent}}"
+#       end
+
+#       def pad_with_attributes(text, attrs=nil)
+#         width = 40 - (2 * graph.depth)
+#         if attrs then
+#           attr_strs = attrs.map{|attr, val| attribute_str(attr, val) }
+#           "%-#{width}s [ %s ]" % [text, attr_strs.join(',')]
+#         else
+#           text
+#         end
+#       end
+#     end
+
+#     class Node < Item
+#       field :inslots,  Array, :default => []
+#       field :outslots, Array, :default => []
+#       field :shape, Symbol, :default => :Mrecord
+
+#       def graph_attribs
+#         {
+#           :shape => shape,
+#           :label => quote(shape =~ /record/ ? structured_label : label),
+#           # :fixedsize => true, :width => "1.0",
+#         }
+#       end
+
+#       def inslots_str
+#         inslots.map{|slot| "<#{slot}>#{slot[0..0]}"}.join("|")
+#       end
+
+#       def outslots_str
+#         outslots.map{|slot| "<out_#{slot}>#{slot[0..0]}"}.join("|")
+#       end
+
+#       def label
+#         super.to_s.gsub(/_\d+$/, '').gsub(/[_\.]+/, "\\n")
+#       end
+
+#       def structured_label
+#         str = "{"
+#         str << "{" << inslots_str << "}|"  unless inslots.empty?
+#         str << label
+#         str << "|{" << outslots_str << "}" unless outslots.empty?
+#         str << "}"
+#       end
+
+#       def to_s
+#         str = []
+#         str << line(quote(name), graph_attribs)
+#         str.join("\n")
+#       end
+#     end
+
+#     class Edge < Item
+#       magic :from, String
+#       magic :into, String
+
+#       def to_s
+#         str = ""
+#         str << quote(from)
+#         str << " -> "
+#         str << quote(into)
+#         line(str)
+#       end
+#     end
+
+#     class Graph < Item
+#       field :items, Array, :default => []
+#       field :edges, Array, :default => []
+
+#       def graph(name, attrs={})
+#         obj = Graph.new(attrs.merge(:name => name, :owner => self))
+#         items << obj
+#         yield(obj) if block_given?
+#         obj
+#       end
+
+#       def node(name, attrs={})
+#         obj = Node.new(attrs.merge(:name => name, :owner => self))
+#         items << obj
+#         yield(obj) if block_given?
+#         obj
+#       end
+
+#       def edge(from, into, from_slot=nil, into_slot=nil)
+#         obj = Edge.new(
+#           :name => name, :owner => self,
+#           :from => from, :into => into,
+#           :from_slot => from_slot, :into_slot => into_slot)
+#         edges << obj
+#         yield(obj) if block_given?
+#         obj
+#       end
+
+#       def to_s
+#         str = []
+#         str << brace("subgraph #{quote("cluster_#{name}")}")
+#         str << line(attrib("  label", quote(label)))
+#         items.each do |item|
+#           str << item.to_s
+#         end
+#         edges.each do |edge|
+#           str << edge.to_s
+#         end
+#         str << close_brace
+#         str.join("\n")
+#       end
+#     end
+
+#     class Universe < Graph
+#       field :orient, Symbol, :doc => 'one of :TB, :BT, :LR, :RL', :default => :TB
+#       field :engine, Symbol, :default => :dot
+
+#       def to_s
+#         str = []
+#         str << brace("digraph #{name}")
+#         str << line("  rankdir = #{orient}")
+#         items.each do |item|
+#           str << item.to_s
+#         end
+#         str << close_brace
+#         str.join("\n")
+#       end
+
+#       def depth() 0; end
+
+#       def save(path, type=nil)
+#         File.open "#{path}.dot", "w" do |f|
+#           f.puts self.to_s
+#         end
+#         system "#{engine} -T#{type} #{path}.dot > #{path}.#{type}" if type
+#       end
+#     end
+#   end
+# end