wukong 3.0.1 → 4.0.0

data/lib/wukong/widget/reducers.rb

@@ -1,4 +1,5 @@
 require_relative("reducers/accumulator")
+require_relative("reducers/improver")
 require_relative("reducers/sort")
 require_relative("reducers/count")
 require_relative("reducers/group")
@@ -6,3 +7,4 @@ require_relative("reducers/group_concat")
 require_relative("reducers/moments")
 require_relative("reducers/bin")
 require_relative("reducers/uniq")
+require_relative("reducers/join_xml")

data/lib/wukong/widget/reducers/improver.rb

@@ -0,0 +1,71 @@
+module Wukong
+  class Processor
+
+    # A base widget for building more complex improver widgets.
+    class Improver < Processor
+
+      # The current group of records.
+      attr_accessor :group
+
+      # Sets up this improver by defining an initial key (with a
+      # value that is unlikely to be found in real data) and calling
+      # `#zero` with no record.
+      def setup
+        @key = :__first_group__
+        zero
+      end
+
+      def recordize record
+        record.split("\t")
+      end
+      
+      #
+      # All kinds of assumptions here,
+      # record is tab-delimited and the
+      # first field is a name of a function
+      # to call
+      #
+      def get_function record
+        record.first
+      end
+      
+      # Processes the `record`.
+      def process(record)
+        fields = recordize(record)
+        func   = get_function(fields)
+        case func
+        when 'zero' then
+          yield zero
+        when 'accumulate' then
+          accumulate(fields[1..-1])
+        when 'improve' then
+          yield improve(fields[1], self.group)
+          self.group = []
+        else
+          raise NoMethodError, "undefined method #{func} for Improver"
+        end
+        STDOUT.flush # WHY? Because.
+      end
+
+      # Starts accumulation for a new key. Return what you would
+      # with no improvements.
+      def zero
+        self.group = []
+      end
+
+      # Accumulates another +record+.
+      #
+      # @param [Object] record
+      def accumulate record
+        self.group << record
+      end
+
+      # Improve prev with group
+      #
+      #
+      def improve prev, group
+      end
+      
+    end
+  end
+end

data/lib/wukong/widget/reducers/join_xml.rb

@@ -0,0 +1,37 @@
+module Wukong
+  class Processor
+
+    # Joins XML input data based on a root tag.
+    class JoinXML < Processor
+
+      field :root, String, default: 'xml', doc: "Name of the root XML element"
+
+      def setup
+        @lines = []
+      end
+
+      def process line
+        if match = terminator.match(line)
+          if match.end(0) == line.size
+            @lines << line
+          else
+            @lines << line[0...match.end(0)]
+          end
+          yield @lines.join("\n")
+          @lines = []
+          @lines << line[match.end(0)..-1] unless match.end(0) == line.size
+        else
+          @lines << line
+        end
+      end
+    
+      def terminator
+        %r{<\s*/\s*#{root}\s*>}i
+      end
+      
+      register :join_xml
+    end
+  end
+end
+
+

data/lib/wukong/widget/serializers.rb

@@ -4,13 +4,11 @@ module Wukong
     SerializerError = Class.new(Error)
 
     class Serializer < Processor
-      field :on_error, String, default: 'log', :doc => "Action to take upon an error, either 'log' or 'notify'"
 
       def handle_error(record, err)
-        case on_error
-        when 'log'    then log.warn "#{err.class}: #{err.message}"
-        when 'notify' then notify('error', record: record, error: err)
-        end          
+        return if err.class == Errno::EPIPE
+        log.error "#{err.class}: #{err.message}"
+        err.backtrace.each { |line| log.debug(line) }
       end
 
     end
@@ -292,10 +290,27 @@ EOF
       def process(record)
         wire_format = record.try(:to_wire) || record
         raise SerializerError.new("Can only recordize a Hash-like record") unless wire_format.is_a?(Hash)
-        yield model.receive(wire_format)
+        klass = model_class_for(wire_format)
+        if klass
+          yield klass.receive(wire_format)
+        else
+          log.error("No default model class and no explicit model for: #{wire_format.inspect}")
+        end
       rescue => e
         handle_error(record, e)
       end
+
+      def model_class_for(record)
+        if explicit_type = (record[:_type] || record["_type"])
+          begin
+            return explicit_type.constantize
+          rescue NameError => e
+            log.warn("Could not find a class for <#{explicit_type}>")
+          end
+        end
+        return model if model
+      end
+      
       register
     end
   end

data/lib/wukong/widgets.rb

@@ -1,4 +1,7 @@
-require 'wukong/widget/processors'
-require 'wukong/widget/reducers'
-require 'wukong/widget/serializers'
 require 'wukong/widget/filters'
+require 'wukong/widget/serializers'
+require 'wukong/widget/operators'
+require 'wukong/widget/reducers'
+require 'wukong/widget/extract'
+require 'wukong/widget/logger'
+require 'wukong/widget/echo'

data/spec/hanuman/graph_spec.rb

@@ -2,17 +2,80 @@ require 'spec_helper'
 
 describe Hanuman::Graph, :hanuman => true do
 
-  # it 'makes a tree' do
-  #   example_graph.tree.should == {
-  #     :name => :pie,
-  #     :inputs => [:bake_pie],
-  #     :stages => [
-  #       {:name=>:make_pie, :inputs=>[:crust, :filling]},
-  #       {:name=>:bake_pie, :inputs=>[:make_pie]}
-  #     ],
-  #     }
-  # end
+  include_context "graphs"
+  
+  context "#ancestors" do
+    context "called without any arguments" do
+      it "returns all stages with an ancestor" do
+        graph.ancestors.size.should == 4
+      end
+    end
+
+    context "called with a stage as the first argument" do
+      it "returns the immediate ancestors of the stage" do
+        graph.ancestors(graph.stages[:second]).size.should == 2
+      end
+    end
+  end
 
+  context "#descendents" do
+    context "called without any arguments" do
+      it "returns all stages with a descendent" do
+        graph.descendents.size.should == 4
+      end
+    end
+
+    context "called with a stage as the first argument" do
+      it "returns the immediate descendents of the stage" do
+        graph.descendents(graph.stages[:second]).size.should == 2
+      end
+    end
+  end
+
+  context "#add_stage" do
+    let(:stage) { Hanuman::Stage.receive(label: :orphan) }
+    it "adds the stage to the graph" do
+      expect { graph.add_stage(stage) }.to change { graph.stages[:orphan] }.from(nil).to(stage)
+    end
+    it "doesn't create any links" do
+      expect { graph.add_stage(stage) }.to_not change { graph.links }
+    end
+  end
+
+  context "#add_link" do
+    context "when adding a new stage" do
+      let(:from) { graph.stages[:fourth]  }
+      let(:into) { Hanuman::Stage.receive(label: :fifth) }
+      it "adds the new stage to the graph" do
+        expect { graph.add_link(:simple, from, into) }.to change { graph.stages[:fifth] }.from(nil).to(into)
+      end
+      it "adds the new link to the graph" do
+        expect { graph.add_link(:simple, from, into) }.to change { graph.links.size }.by(1)
+      end
+    end
+    context "when adding an existing link" do
+      let(:from) { graph.stages[:third_a] }
+      let(:into) { graph.stages[:fourth]  }
+      it "duplicates the link in the graph" do
+        expect { graph.add_link(:simple, from, into) }.to change { graph.links.size }.by(1)
+      end
+    end
+    context "when making a cycle" do
+      let(:from) { graph.stages[:fourth] }
+      let(:into) { graph.stages[:first_a]  }
+      it "adds the link in the graph" do
+        expect { graph.add_link(:simple, from, into) }.to change { graph.links.size }.by(1)
+      end
+    end
+    context "when linking a stage to itself" do
+      let(:from) { tree.stages[:fourth] }
+      let(:into) { tree.stages[:fourth]  }
+      it "adds the link in the graph" do
+        expect { graph.add_link(:simple, from, into) }.to change { graph.links.size }.by(1)
+      end
+    end
+  end
+  
 end
 
 describe Hanuman::GraphBuilder, :hanuman => true do

data/spec/hanuman/stage_spec.rb

@@ -63,4 +63,19 @@ describe Hanuman::StageBuilder, :hanuman => true do
       subject.serialize.should_not include(:links)
     end
   end
+
+  context '#into' do
+    subject          { described_class.receive(label: :pyro)   }
+    let(:other_stage){ described_class.receive(label: :iceman) }
+    
+    it 'returns the linked into stage for chaining' do
+      subject.into(other_stage).should be(other_stage)
+    end
+    
+    it 'links two stages together with a simple link' do
+      subject.into(other_stage)
+      subject.links.should be_any{ |link| link.from == :pyro and link.into == :iceman }
+    end
+  end
+  
 end

data/spec/hanuman/tree_spec.rb

@@ -0,0 +1,119 @@
+require 'spec_helper'
+
+describe Hanuman::Tree do
+
+  include_context "graphs"
+
+  it "iterates over each stage in tree-order" do
+    tree.directed_sort.should == [:first, :second, :third_b, :third_a, :fourth]
+  end
+
+  context "#root" do
+    context "when called without arguments" do
+      it "returns the root of the whole tree" do
+        tree.root.should_not be_nil
+        tree.root.label.should == :first
+      end
+      it "returns the root of a sigle stage tree" do
+        single_stage_tree.root.should_not be_nil
+        single_stage_tree.root.label.should == :first
+      end
+      
+    end
+    context "when called with a stage" do
+      it "should return the root of the whole tree" do
+        tree.root(tree.stages[:fourth]).should_not be_nil
+        tree.root(tree.stages[:fourth]).label.should == :first
+      end
+    end
+  end
+
+  context "#ancestor" do
+    it "returns the ancestor of a stage" do
+      tree.ancestor(tree.stages[:fourth]).should_not be_nil
+      tree.ancestor(tree.stages[:fourth]).label.should == :third_a
+    end
+    it "returns nil for the root" do
+      tree.ancestor(tree.root).should be_nil
+    end
+  end
+
+  context "#leaves" do
+    it "returns the leaf stages of a tree" do
+      tree.leaves.map(&:label).should include(:fourth, :third_b)
+    end
+
+    it "returns the root of a tree that has no other leaves" do
+      single_stage_tree.leaves.map(&:label).should include(:first)
+    end
+    
+  end
+
+  context "#prepend" do
+    let(:zeroth) { Hanuman::Stage.receive(label: :zeroth) }
+    it "adds the given stage" do
+      expect { tree.prepend(zeroth) }.to change { tree.stages[:zeroth] }.from(nil).to(zeroth)
+    end
+    it "adds a link from the new stage to the old root" do
+      expect { tree.prepend(zeroth) }.to change { tree.has_link?(zeroth, tree.stages[:first]) }.from(false).to(true)
+    end
+    it "the root becomes the new stage" do
+      expect { tree.prepend(zeroth) }.to change { tree.root.label }.from(:first).to(:zeroth)
+    end
+  end
+
+  context "#append" do
+    let(:fifth) { Hanuman::Stage.receive(label: :fifth) }
+    it "adds a new stage for each leaf" do
+      expect { tree.append(fifth) }.to change { tree.stages.size }.by(2)
+    end
+    it "adds a link for each of the new stages" do
+      expect { tree.append(fifth) }.to change { tree.links.size }.by(2)
+    end
+    it "but doesn't change the number of leaves " do
+      expect { tree.append(fifth) }.to_not change { tree.leaves.size }
+    end
+  end
+  
+  context "#add_link" do
+    context "when adding a new stage" do
+      let(:from) { tree.stages[:fourth]  }
+      let(:into) { Hanuman::Stage.receive(label: :fifth) }
+      it "adds the new stage to the tree" do
+        expect { tree.add_link(:simple, from, into) }.to change { tree.stages[:fifth] }.from(nil).to(into)
+      end
+      it "adds the new link to the tree" do
+        expect { tree.add_link(:simple, from, into) }.to change { tree.links.size }.by(1)
+      end
+    end
+    context "when adding an existing link" do
+      let(:from) { tree.stages[:third_a] }
+      let(:into) { tree.stages[:fourth]  }
+      it "doesn't duplicate the link in the tree" do
+        expect { tree.add_link(:simple, from, into) }.to_not change { tree.links.size }
+      end
+    end
+    context "when adding a link to a stage with an existing parent" do
+      let(:from) { Hanuman::Stage.receive(label: :fifth) }
+      let(:into) { tree.stages[:fourth]                  }
+      it "raises an error" do
+        expect { tree.add_link(:simple, from, into) }.to raise_error(Hanuman::Tree::MultipleRoots)
+      end
+    end
+    context "when making a cycle" do
+      let(:from) { tree.stages[:fourth] }
+      let(:into) { tree.stages[:first]  }
+      it "raises an error" do
+        expect { tree.add_link(:simple, from, into) }.to raise_error(TSort::Cyclic)
+      end
+    end
+    context "when linking a stage to itself" do
+      let(:from) { tree.stages[:fourth] }
+      let(:into) { tree.stages[:fourth]  }
+      it "raises an error" do
+        expect { tree.add_link(:simple, from, into) }.to raise_error(TSort::Cyclic)
+      end
+    end
+  end
+  
+end

data/spec/spec_helper.rb

@@ -1,5 +1,7 @@
 require 'wukong'
 require 'wukong/spec_helpers'
+require 'wukong/source'
+require_relative './support/shared_context_for_graphs'
 require_relative './support/shared_examples_for_shortcuts'
 require_relative './support/shared_examples_for_builders'
 require_relative './support/shared_context_for_reducers'
@@ -18,13 +20,23 @@ RSpec.configure do |config|
   end
 
   def generic_runner *args
-    runner(Wukong::Runner, 'wu-generic', *args)
+    # the wu-generic program doesn't have to exist for this Runner to
+    # work if it's called from Ruby code
+    runner(Wukong::Runner, 'wu-generic', *args) 
   end
 
   def wu_local *args
     command('wu-local', *args)
   end
   
+  def wu_source *args
+    command('wu-source', *args)
+  end
+  
+  def wu *args
+    command('wu', *args)
+  end
+  
   # FIXME Why is this here?
   config.treat_symbols_as_metadata_keys_with_true_values = true
   

data/spec/support/example_test_helpers.rb

@@ -28,7 +28,6 @@ RSpec.configure do |config|
 end
 
 shared_context 'wukong', :helpers => true do
-
   RSpec::Matchers.define(:be_in){|expected| match{|actual| expected.include?(actual) } }
 
   def self.be_ish_matcher(handle, regexp)