wukong 3.0.1 → 4.0.0

data/bin/wu-source

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'wukong'
+require 'wukong/source'
+Wukong::Source::SourceRunner.run

data/examples/Gemfile

@@ -2,7 +2,6 @@ source :rubygems
 
 gem "configliere",        '~> 0.4'
 gem "multi_json",         '>= 1.3.6'
-gem "vayacondios-client", '>= 0.0.3'
 gem "gorillib",           '>= 0.4.2'
 gem "uuidtools"
 gem "eventmachine"

data/examples/deploy_pack/Gemfile

@@ -2,6 +2,5 @@
 gem 'configliere'
 gem 'gorillib'
 gem 'multi_json'
-gem 'vayacondios-client'
 gem 'log4r'
 gem 'eventmachine'

data/examples/improver/tweet_summary.rb

@@ -0,0 +1,73 @@
+
+#
+# Stupidly simple class for holding a twitter user's tweet summary
+#
+class UserTweetSummary  
+  attr_accessor :fields
+
+  def initialize fields
+    @fields = fields    
+  end
+    
+  def update tweets
+    deletes_this_batch = 0
+    tweets_this_batch  = 0
+    tweets.each do |tweet|
+      deletes_this_batch += (tweet.is_deleted ? 1 : 0)
+      tweets_this_batch  += 1
+    end
+    fields['deletes'] += deletes_this_batch
+    fields['tweets']  += (tweets_this_batch - deletes_this_batch)
+    self
+  end
+
+  def to_json *args
+    fields.to_json     
+  end  
+end
+
+#
+# Holds a tweet and whether
+# or not it's been deleted
+#
+class Tweet
+  attr_accessor :is_deleted
+  def initialize hash
+    @is_deleted = hash.keys.include?("delete")
+  end
+end
+
+
+#
+# Improver processor
+#
+class TweetSummarizer < Wukong::Processor::Improver
+  attr_accessor :user_id
+  
+  def zero
+    super
+    {
+      'tweets'  => 0,
+      'deletes' => 0
+    }
+  end
+
+  def accumulate record
+    @user_id = record[0]
+    json     = record[1]
+    self.group << Tweet.new(JSON.parse(json))
+  end
+
+  def improve summary, deltas    
+    UserTweetSummary.new(JSON.parse(summary)).update(deltas)
+  end  
+end
+
+#
+# Is this necessary?
+#
+TweetSummarizer.register(:tweet_summarizer)
+
+Wukong.dataflow(:summarize_tweets) do
+  tweet_summarizer | to_json
+end

data/examples/ruby_project/Gemfile

@@ -2,6 +2,5 @@
 gem 'configliere'
 gem 'gorillib'
 gem 'multi_json'
-gem 'vayacondios-client'
 gem 'log4r'
 gem 'eventmachine'

data/examples/splitter.rb

@@ -0,0 +1,94 @@
+require 'wukong'
+
+module Verbose
+
+  def verbose?
+  end
+  
+  def setup
+    # log.info("Setting up #{label}")
+  end
+
+  def finalize
+    # log.info("Finalizing #{label}")
+  end
+end
+  
+Wukong.processor(:upcaser) do
+  include Verbose
+  def process(string)
+    # log.info("#process #{string}")
+    yield string.upcase
+  end
+end
+
+Wukong.processor(:downcaser) do
+  include Verbose
+  def process(string)
+    # log.info("#process #{string}")
+    yield string.downcase
+  end
+end
+
+Wukong.processor(:tokenizer) do
+  include Verbose
+  def process string
+    # log.info("#process #{string}")
+    string.split.each { |token| yield token }
+  end
+end
+
+Wukong.processor(:stripper) do
+  include Verbose
+  def process(string)
+    # log.info("#process #{string}")
+    yield string.gsub(/[^\w\s]/,'')
+  end
+end
+
+Wukong.processor(:devoweler) do
+  include Verbose
+  def process(string)
+    # log.info("#process #{string}")
+    yield string.gsub(/[aeiou]/i,'')
+  end
+end
+
+# stripper  = Wukong.registry.retrieve(:stripper)
+# tokenizer = Wukong.registry.retrieve(:tokenizer)
+# upcaser   = Wukong.registry.retrieve(:upcaser)
+# downcaser = Wukong.registry.retrieve(:downcaser)
+# devoweler = Wukong.registry.retrieve(:devoweler)
+
+# Splitter = Class.new(Wukong::Dataflow)
+# builder = Wukong::DataflowBuilder.receive({label: :splitter,
+#                                             for_class: Splitter,
+#                                             stages: {
+#                                               stripper:  stripper,
+#                                               tokenizer: tokenizer,
+#                                               upcaser:   upcaser,
+#                                               downcaser: downcaser,
+#                                               devoweler: devoweler,
+#                                             },
+#                                             links: [
+#                                                     Hanuman::LinkFactory.connect(:simple, :stripper, :tokenizer),
+#                                                     Hanuman::LinkFactory.connect(:simple, :tokenizer, :upcaser),
+#                                                     Hanuman::LinkFactory.connect(:simple, :tokenizer, :downcaser),
+#                                                     Hanuman::LinkFactory.connect(:simple, :upcaser, :devoweler),
+#                                                    ]})
+
+# builder.extract_links!
+# Splitter.set_builder(builder)
+# Splitter.register
+
+Wukong.dataflow(:splitter) do
+  stripper | tokenizer |
+    [
+     upcaser | devoweler |
+     [
+      regexp | count,
+      identity
+     ],
+     downcaser | reject { |word| word == 'hell' }
+    ]
+end

data/examples/twitter.rb

@@ -0,0 +1,5 @@
+Wukong.dataflow(:twitter) do
+  from_json | reject { |obj| obj["delete"]  } |
+    [
+     
+end

data/lib/hanuman.rb

@@ -1,11 +1,11 @@
 require 'gorillib/some'
 require 'gorillib/model'
-require 'tsort'
 
 require 'hanuman/registry'
 require 'hanuman/link'
 require 'hanuman/stage'              
 require 'hanuman/graph'
+require 'hanuman/tree'
 
 module Hanuman
   module Shortcuts

data/lib/hanuman/graph.rb

@@ -1,29 +1,53 @@
 module Hanuman
-  class Graph < Stage
-    include TSort
 
-    field :stages, Hash,  :default => {}
-    field :links,  Array, :default => []
-    
-    def tsort_each_node(&blk)
-      stages.keys.each(&blk)
+  module GraphInstanceMethods
+    def each_stage &block
+      stages.values.each(&block)
     end
 
-    def tsort_each_child(node, &blk)
-      links.select{ |link| link.into == node }.map(&:from).each(&blk)
+    def descendents stage=nil
+      links.find_all do |link|
+        stage ? link.from == stage.label : true
+      end.map(&:into).uniq.map { |label| stages[label] }.compact
     end
 
-    def directed_sort() self.tsort ; end
+    def ancestors stage=nil
+      links.find_all do |link|
+        stage ? link.into == stage.label : true
+      end.map(&:from).uniq.map { |label| stages[label] }.compact
+    end
+
+    def add_stage stage
+      stages[stage.label] = stage
+    end
+
+    def has_link? from, into
+      links.detect { |link| link.from == from.label && link.into == into.label } ? true : false
+    end
+
+    def add_link type, from, into
+      add_stage(from)
+      add_stage(into)
+      self.links << Hanuman::LinkFactory.connect(type, from.linkable_name(:in), into.linkable_name(:out))
+    end
+  end
+
+  class Graph < Stage
+    include GraphInstanceMethods
+    
+    field :stages, Hash,  :default => {}
+    field :links,  Array, :default => []
   end
 
   class GraphBuilder < StageBuilder
-    include TSort
 
+    include GraphInstanceMethods
+    
     field :stages, Hash,  :default => {}
     field :links,  Array, :default => []
 
     def define(&blk)     
-      graph = for_class || define_class(label) 
+      graph = for_class || define_class(label)
       self.instance_eval(&blk) if block_given?
       extract_links!
       graph.register
@@ -49,6 +73,7 @@ module Hanuman
     end
 
     def extract_links!
+      self.links.replace([])
       stages.each_pair{ |name, builder| links << builder.links }
       links.flatten!
     end
@@ -60,21 +85,13 @@ module Hanuman
       attrs.merge(args)      
     end
 
-    def tsort_each_node(&blk)
-      stages.keys.each(&blk)
-    end
-
-    def tsort_each_child(node, &blk)
-      links.select{ |link| link.into == node }.map(&:from).each(&blk)
-    end
-
-    def directed_sort() self.tsort ; end
-
     def clone
       cloned_attrs  = Hash[ serialize.select{ |key, val| key != :stages }.map{ |key, val| dup_key = key.dup rescue key ; dup_val = val.dup rescue val ; [ dup_key, dup_val ] } ]
       cloned_links  = links.map{ |link| link.dup }
       cloned_stages = Hash[ stages.map{ |stage| stage.clone } ]
       self.class.receive(cloned_attrs.merge(links: cloned_links).merge(stages: cloned_stages).merge(for_class: for_class))
     end
+
   end
+    
 end

data/lib/hanuman/stage.rb

@@ -1,4 +1,22 @@
 module Hanuman
+
+  module StageInstanceMethods
+
+    attr_accessor :graph
+    
+    def linkable_name(direction) ; self.label ; end
+
+    def add_link(type, other_stage)
+      self.links << Hanuman::LinkFactory.connect(type, linkable_name(:in), other_stage.linkable_name(:out))
+      graph.add_link(type, self, other_stage) if graph
+    end
+
+    def root
+      graph ? graph.root(self) : self
+    end
+
+  end
+  
   module StageClassMethods
 
     def label() self.to_s.demodulize.underscore.to_sym ; end
@@ -23,13 +41,23 @@ module Hanuman
 
   class Stage
     include Gorillib::Model
+    include StageInstanceMethods
     extend  StageClassMethods
 
     field :label, Symbol, :doc => false
-  end  
+    field :links, Array, :default => [], doc: false
 
+    def clone
+      cloned_attrs = Hash[ attributes.map{ |key, val| dup_key = key.dup rescue key ; dup_val = val.dup rescue val ; [ dup_key, dup_val ] } ]
+      cloned_links = links.map{ |link| link.dup }
+      self.class.receive(cloned_attrs.merge(links: cloned_links))
+    end
+  end  
+  
   class StageBuilder
+
     include Gorillib::Model
+    include StageInstanceMethods
 
     field :args,      Hash,  :default => {}
     field :for_class, Class
@@ -65,18 +93,6 @@ module Hanuman
       klass
     end
     
-    def linkable_name(direction) self.label ; end
-
-    def add_link(level, from, into)    
-      links << Hanuman::LinkFactory.connect(level, from, into)
-    end
-
-    def into(other_stage)
-      self.add_link(:simple, self.linkable_name(:in), other_stage.linkable_name(:out))
-      other_stage
-    end
-    alias_method :|, :into
-
     def serialize()      
       attrs = attributes
       args  = attrs.delete(:args)
@@ -91,5 +107,22 @@ module Hanuman
       self.class.receive(cloned_attrs.merge(links: cloned_links).merge(for_class: for_class))
     end
 
+    def into(stage_or_stages)
+      return stage_or_stages if stage_or_stages.nil?
+      if stage_or_stages.is_a?(Array)
+        stage_or_stages.each do |other_stage_or_stages|
+          while other_stage_or_stages.is_a?(Array)
+            other_stage_or_stages = other_stage_or_stages.first
+          end
+          other_stage = other_stage_or_stages
+          self.into(other_stage)
+        end
+      else
+        self.add_link(:simple, stage_or_stages.root)
+      end
+      stage_or_stages
+    end
+    alias_method :|, :into
+    
   end
 end

data/lib/hanuman/tree.rb

@@ -0,0 +1,67 @@
+require 'tsort'
+module Hanuman
+
+  module TreeInstanceMethods
+    include TSort
+
+    MultipleRoots = Class.new(TSort::Cyclic)
+
+    def tsort_each_node(&blk)
+      stages.keys.each(&blk)
+    end
+
+    def tsort_each_child(label, &blk)
+      links.select { |link| link.into == label }.map(&:from).each(&blk)
+    end
+
+    def directed_sort() self.tsort ; end
+    
+    def each_stage &blk
+      directed_sort.map { |label| stages[label]}.compact.each(&blk)
+    end
+
+    def root stage=nil
+      return stages[directed_sort.first] unless stage
+      return stage unless ancestor(stage)
+      self.root(ancestor(stage))
+    end
+
+    def ancestor(stage)
+      ancestors(stage).first
+    end
+    
+    def leaves
+      the_leaves = (descendents - ancestors)
+      the_leaves.empty? ? [root] : the_leaves
+    end
+
+    def add_link type, from, into
+      return if has_link?(from, into)
+      raise TSort::Cyclic.new("Cannot link from a stage <#{from.label}> to itself") if into == from
+      raise MultipleRoots.new("Cannot link from <#{from.label}> to <#{into.label}> because <#{into.label}> aleady has an ancestor <#{ancestor(into).label}>") if ancestor(into)
+      raise TSort::Cyclic.new("Cannot link from leaf <#{from.label}> to the root <#{into.label}>") if into == root && leaves.include?(from)
+      super(type, from, into)
+    end
+
+    def prepend stage
+      add_link(:simple, stage, root)
+    end
+
+    def append stage
+      leaves.each do |leaf|
+        stage_for_leaf       = stage.clone
+        stage_for_leaf.label = "#{stage_for_leaf.label}_for_#{leaf.label}".to_sym
+        add_link(:simple, leaf, stage_for_leaf)
+      end
+    end
+    
+  end
+  
+  class Tree < Graph
+    include TreeInstanceMethods
+  end
+
+  class TreeBuilder < GraphBuilder
+    include TreeInstanceMethods
+  end
+end