wukong 3.0.0.pre → 3.0.0.pre2

data/lib/wukong/widget/sink.rb

@@ -1,58 +1,16 @@
 module Wukong
-  class Sink < Wukong::Processor
-
-    class NullSink < Wukong::Sink
+  class Sink < Processor
+    
+    class Stdout < Sink
       def process(record)
-        true # do nothing
-      end
-    end
-
-    # Write all lines to given file
-    class IO < Wukong::Sink
-      def process(record)
-        file.puts(record)
-      end
-    end
-
-    class FileSink < Wukong::Sink::IO
-      field :filename, Pathname, :doc => "Filename to write"
-      attr_reader :file
-
-      def self.make(workflow, filename, stage_name=nil, attrs={})
-        super(workflow, attrs.merge(:filename => filename, :name => stage_name))
-      end
-
-      def setup
-        super
-        filename.dirname.mkpath
-        @file = File.open(filename, "w")
-      end
-
-      def stop
-        @file.close if @file
-      end
-
-      register_processor
-    end
-
-    # Writes all lines to $stdout
-    class Stdout < Wukong::Sink::IO
-      def file() $stdout ; end
-      register_processor
-    end
-
-    # Writes all lines to $stderr
-    class Stderr < Wukong::Sink::IO
-      def file() $stderr ; end
-      register_processor
-    end
-
-    class ArraySink < Wukong::Sink
-      field :records, Array, :default => [], :writer => :protected
-
-      def process(record)
-        self.records << record
+        begin
+          $stdout.puts record
+        rescue Errno::EPIPE => e
+          exit(2)
+        end
       end
+      register
     end
+    
   end
 end

data/lib/wukong/widget/source.rb

@@ -1,120 +1,14 @@
 module Wukong
-  class Source < Hanuman::Action
-    include Hanuman::IsOwnOutputSlot
-    def self.register_source(name=nil, &block)
-      register_action(name, &block)
-    end
+  class Source < Processor
 
-    def drive
-      each do |record|
-        output.process(record)
-      end
-    end
-
-    def new_string_event string
-      metadata_hash = Hash.new
-      string.define_singleton_method(:_metadata) do
-        metadata_hash
-      end
-      string
-    end
-
-    class Iter < Source
-      # the enumerable object to delegate
-      attr_reader :obj
-
-      def initialize(obj)
-        @obj = obj
-      end
-      def each(&block)
-        obj.each(&block)
-      end
-    end
-
-    class IO < Source
-      attr_reader :file
-
-      def each(&block)
-        file.each do |line|
-          yield line.chomp
+    class Stdin < Source
+      def process
+        while line = $stdin.readline.chomp! rescue nil
+          yield line
         end
       end
-
-      def stop
-        file.close if file
-      end
-    end
-
-    # emits each line from $stdin
-    class Stdin < Wukong::Source::IO
-      def setup
-        super
-        @file = $stdin
-      end
-      register_source
-    end
-
-    class FileSource < Wukong::Source::IO
-      field :filename, Pathname, :doc => "Filename to read from"
-
-      def self.make(workflow, filename, stage_name=nil, attrs={})
-        super(workflow, attrs.merge(:filename => filename, :name => stage_name))
-      end
-
-      def setup
-        super
-        @file = File.open(filename)
-      end
-
-      register_source
-    end
-
-    module CappedGenerator
-      extend Gorillib::Concern
-      included do
-        attr_reader :num
-        field :size, Integer, :default => 2**63, :doc => "Number of items to generate", :writer => true
-      end
-
-      def setup
-        super
-        @num = 0
-      end
-
-      def max
-        size
-      end
-
-      def next_item
-      end
-
-      def each
-        loop do
-          break if @num > max
-          yield next_item
-          @num += 1
-        end
-      end
-    end
-
-    class Integers < Wukong::Source
-      register_source :integers
-      include CappedGenerator
-      field :init, Integer, :default => 0, :doc => "Initial offset", :writer => true
-
-      def max
-        init + size - 1
-      end
-
-      def next_item
-        @num
-      end
-
-      def self.make(dataflow, size=nil, attrs={})
-        attrs[:size] = size if not size.nil?
-        super(dataflow, attrs)
-      end
-    end
+      register
+    end    
 
   end
 end

data/lib/wukong/widget/utils.rb

@@ -0,0 +1,46 @@
+module Wukong
+
+  # :nodoc:
+  # 
+  # This code is gross and nasty.
+  module DynamicGet
+
+    # :nodoc:
+    def self.included klass
+      klass.send(:field, :separator, String,   :default => "\t")
+    end
+
+    # :nodoc:
+    def get field, obj
+      return obj unless field
+      case
+      when field.to_s.to_i > 0 && obj.is_a?(String)
+        obj.split(separator)[field.to_s.to_i - 1]
+      when field.to_s.to_i > 0
+        obj[field.to_s.to_i - 1]
+      when field.to_s.to_i == 0 && obj.is_a?(String) && obj =~ /^\s*\{/
+        begin
+          get_nested(field, MultiJson.load(obj))
+        rescue MultiJson::DecodeError => e
+        end
+      when field.to_s.to_i == 0 && (!field.to_s.include?('.')) && obj.respond_to?(field.to_s)
+        obj.send(field.to_s)
+      when field.to_s.to_i == 0 && obj.respond_to?(:[])
+        get_nested(field, obj)
+      else obj
+      end
+    end
+
+    # :nodoc:
+    def get_nested fields, obj
+      parts = fields.to_s.split('.')
+      field = parts.shift
+      return unless field
+      if slice = obj[field]
+        return slice if parts.empty?
+        get_nested(parts.join('.'), slice)
+      end
+    end
+
+  end
+end

data/lib/wukong/widgets.rb

@@ -0,0 +1,6 @@
+require 'wukong/widget/source'
+require 'wukong/widget/sink'
+require 'wukong/widget/processors'
+require 'wukong/widget/reducers'
+require 'wukong/widget/serializers'
+require 'wukong/widget/filters'

data/spec/examples/dataflow/fibonacci_series_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+# require 'wukong'
+
+# describe_example_script :fibonacci_series, 'dataflow/fibonacci_series.rb', examples_spec: true do
+#   subject{ Wukong.chain(:fibbonaci_series) }
+
+#   it 'generates a fibonacci sequence' do
+#     subject.ticker.qty(12)
+#     # subject.output > subject.array_sink(name: :numbers)
+#     # subject.setup
+#     # subject.ticker.drive
+#     #
+#     # subject.numbers.records.should == [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89]
+#   end
+
+#   it_generates_graphviz{|gv_filename| puts File.read(gv_filename) }
+
+# end

data/spec/examples/dataflow/parsing_spec.rb

@@ -1,13 +1,14 @@
 require 'spec_helper'
-require 'wukong'
+# require 'wukong'
 
-describe_example_script(:parse_apache_logs, 'dataflow/parse_apache_logs.rb') do
-  it 'runs' do
-    out, err = Gorillib::TestHelpers.capture_output do
-      Wukong::LocalRunner.receive(:flow => subject) do
-        run :default
-      end
-    end
-    out.string.split("\n").first.should =~ /\{\"ip_address\":\"[\d\.]+\",.*\"}/ 
-  end
-end
+# describe_example_script(:parse_apache_logs, 'dataflow/parse_apache_logs.rb') do
+#   it 'runs' do
+#     subject = Wukong.dataflow(:parse_apache_logs)
+#     out, err = Gorillib::TestHelpers.capture_output do
+#       Wukong::LocalRunner.receive(:flow => subject) do
+#         run :default
+#       end
+#     end
+#     out.string.split("\n").first.should == "127.0.0.1 - - [10/Apr/2007:10:39:11 +0300] \"GET / HTTP/1.1\" 500 606 \"-\" \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.3) Gecko/20061201 Firefox/2.0.0.3 (Ubuntu-feisty)\"\t"
+#   end
+# end

data/spec/examples/dataflow/simple_spec.rb

@@ -1,8 +1,34 @@
 require 'spec_helper'
-require 'wukong'
+# require 'wukong'
 
-describe_example_script(:simple, 'dataflow/simple.rb') do
-  it 'runs' do
-    Wukong::LocalRunner.run(subject, :default)
-  end
-end
+# Hanuman::Graph.class_eval do
+#   def foo_graph(label, &block)
+#     stage(label, :_type => Hanuman::FooGraph, &block)
+#   end
+# end
+# class Hanuman::FooGraph < Hanuman::Graph
+#   # field      :inputs,  Gorillib::Collection, :of => Hanuman::InputSlot,  :doc => 'inputs to this stage',  :default => ->{ Gorillib::Collection.new }
+#   # field      :outputs, Gorillib::Collection, :of => Hanuman::OutputSlot, :doc => 'outputs of this stage', :default => ->{ Gorillib::Collection.new }
+#
+#   collection   :inputs, Hanuman::InputSlot
+#
+# end
+
+# describe 'example', :examples_spec do
+#   # describe_example_script(:simple, 'dataflow/simple.rb', :only => true) do
+#   #   it 'runs' do
+#   #     p subject
+#   #   end
+#   # end
+
+#   it 'runs' do
+#     # load Pathname.path_to(:examples, 'dataflow/simple.rb')
+
+#     Wukong.dataflow(:bob) do
+#       ff = file_source(Pathname.path_to(:data, 'text/jabberwocky.txt')){ p self }
+
+
+#     end
+
+#   end
+# end

data/spec/examples/dataflow/telegram_spec.rb

@@ -1,43 +1,43 @@
 require 'spec_helper'
-require 'wukong'
+# require 'wukong'
 
-describe_example_script(:telegram, 'dataflow/telegram.rb') do
-  it 'runs' do
-    Wukong::LocalRunner.run(subject, :default)
-  end
+# describe_example_script(:telegram, 'dataflow/telegram.rb') do
+#   it 'runs' do
+#     Wukong::LocalRunner.run(subject, :default)
+#   end
 
-  context 'Recompose processor' do
-    subject{ Wukong::Widget::Recompose }
-    its(:field_names){ should include(:break_length) }
+#   context 'Recompose processor' do
+#     subject{ Wukong::Widget::Recompose }
+#     its(:field_names){ should include(:break_length) }
 
-    let(:words    ){
-      # 0    5    1    5    2    5    3    5    4    5    5    5    6    5    7    5    8
-      %w[
-        If names be not correct, language is not in accordance with
-        the truth of things.  If language be not in accordance with
-        the truth of things, affairs cannot be carried on to success. ] }
+#     let(:words    ){
+#       # 0    5    1    5    2    5    3    5    4    5    5    5    6    5    7    5    8
+#       %w[
+#         If names be not correct, language is not in accordance with
+#         the truth of things.  If language be not in accordance with
+#         the truth of things, affairs cannot be carried on to success. ] }
 
-    context '#process' do
-      it 'breaks lines correctly' do
-        (2..80).each do |len|
-          # run the data flow into an array sink
-          test_sink = Wukong::Sink::ArraySink.new
-          rc = subject.new(:break_length => len, :output => test_sink )
-          words.each{|word| rc.process(word) }
-          rc.stop
-          # start and end are correct
-          test_sink.records.first.should =~ /^If/
-          test_sink.records.last.should =~ /success\.$/
-          # lines should be as long as possible, but not longer
-          test_sink.records[0..-2].zip(test_sink.records[1..-1]) do |line, nextl|
-            nextw = nextl.split[0]
-            ((line.length <= len) || line !~ /\s/).should be_true
-            (line.length + nextw.length + 1 > len).should be_true
-          end
-        end
-      end
+#     context '#process' do
+#       it 'breaks lines correctly' do
+#         (2..80).each do |len|
+#           # run the data flow into an array sink
+#           test_sink = Wukong::Sink::ArraySink.new
+#           rc = subject.new(:break_length => len, :output => test_sink )
+#           words.each{|word| rc.process(word) }
+#           rc.stop
+#           # start and end are correct
+#           test_sink.records.first.should =~ /^If/
+#           test_sink.records.last.should =~ /success\.$/
+#           # lines should be as long as possible, but not longer
+#           test_sink.records[0..-2].zip(test_sink.records[1..-1]) do |line, nextl|
+#             nextw = nextl.split[0]
+#             ((line.length <= len) || line !~ /\s/).should be_true
+#             (line.length + nextw.length + 1 > len).should be_true
+#           end
+#         end
+#       end
 
-    end
-  end
+#     end
+#   end
 
-end
+# end

data/spec/examples/graph/minimum_spanning_tree_spec.rb

@@ -1,35 +1,34 @@
-require 'spec_helper'
-require 'wukong'
-require 'wukong/local_runner'
+# require 'spec_helper'
+# require 'wukong'
 
-load Pathname.path_to(:examples, 'graph/minimum_spanning_tree.rb')
+# load Pathname.path_to(:examples, 'graph/minimum_spanning_tree.rb')
 
-describe 'Minimum Spanning Tree', :examples_spec => true, :helpers => true do
+# describe 'Minimum Spanning Tree', :examples_spec, :helpers do
 
-  context Wukong::Widget::DisjointForest do
-    subject{ Wukong::Widget::DisjointForest.new }
+#   context Wukong::Widget::DisjointForest do
+#     subject{ Wukong::Widget::DisjointForest.new }
 
-    context 'operations' do
-      before do
-        %w[ AUS DFW ATL JFK SFO LGA LAX ].each{|el| subject.add el }
-        subject.union('DFW', 'AUS')
-        subject.union('ATL', 'JFK')
-        subject.union('ATL', 'DFW')
-      end
-      
-      context '#find' do
-        it 'collapses elements into a shallow tree during a find' do
-          subject.parent['ATL'].should == 'JFK'
-          subject.parent['JFK'].should == 'AUS'
-          subject.find('ATL').should == 'AUS'
-          subject.parent['ATL'].should == 'AUS'
-        end
-      end
-      context '#union' do
-        it 'joins shallow tree to deep tree' do
-        end
-      end
-    end
-    
-  end
-end
+#     context 'operations' do
+#       before do
+#         %w[ AUS DFW ATL JFK SFO LGA LAX ].each{|el| subject.add el }
+#         subject.union('DFW', 'AUS')
+#         subject.union('ATL', 'JFK')
+#         subject.union('ATL', 'DFW')
+#       end
+
+#       context '#find' do
+#         it 'collapses elements into a shallow tree during a find' do
+#           subject.parent['ATL'].should == 'JFK'
+#           subject.parent['JFK'].should == 'AUS'
+#           subject.find('ATL').should == 'AUS'
+#           subject.parent['ATL'].should == 'AUS'
+#         end
+#       end
+#       context '#union' do
+#         it 'joins shallow tree to deep tree' do
+#         end
+#       end
+#     end
+
+#   end
+# end