wukong 2.0.2 → 3.0.0.pre

data/old/wukong/streamer/instance_streamer.rb

@@ -0,0 +1,15 @@
+module Wukong
+  module Streamer
+    #
+    # Instantiate an instance of 'record_model' for each line
+    class InstanceStreamer < Wukong::Streamer::RecordStreamer
+      class_attribute :record_model
+
+      def recordize(raw_record)
+        fields = super(raw_record)
+        [ record_model.new(*fields) ] if fields
+      end
+
+    end
+  end
+end

data/old/wukong/streamer/json_streamer.rb

@@ -0,0 +1,21 @@
+module Wukong
+  module Streamer
+    #
+    #
+    # Note: it's on you to `require 'json'` somewhere (there's too many 
+    class JsonStreamer < Wukong::Streamer::Base
+
+      #
+      # Parses the incoming record as JSON, returns a single arg to #process
+      #
+      def recordize line
+        begin
+          [JSON.parse(line)]
+        rescue StandardError => boom
+          bad_record!(boom, line.to_s)
+        end
+      end
+
+    end
+  end
+end

data/spec/away/encoding_spec.rb

@@ -0,0 +1,32 @@
+# require 'spec_helper'
+# require 'wukong/encoding'
+#
+# describe "Wukong encoding" do
+#
+#   it 'en/decodes to xml by default' do
+#     Wukong.encode_str("&"           ).should == '&'
+#     Wukong.decode_str("&"       ).should == '&'
+#   end
+#   it 'en/decodes to xml with :xml' do
+#     Wukong.encode_str("&", :xml     ).should == '&'
+#     Wukong.decode_str("&", :xml ).should == '&'
+#   end
+#   it 'url en/decodes with :url' do
+#     Wukong.encode_str("&", :url     ).should == '%26'
+#     Wukong.decode_str("%26", :url   ).should == '&'
+#   end
+#   { "'" => "'", "\t" => "	", "\n" => "
", nil => '',}.each do |raw, enc|
+#     it 'encodes #{raw} to #{enc}' do
+#       Wukong.encode_str(raw, :xml   ).should == enc
+#     end
+#     it 'decodes #{enc} to #{raw}' do
+#       Wukong.decode_str(enc, :xml   ).should == raw.to_s
+#     end
+#   end
+#   ["normal_string with %punctuation should `not be molested", ""].each do |str|
+#     it 'doesn\'t change #{str}' do
+#       Wukong.encode_str(str, :xml   ).should == str
+#     end
+#   end
+#
+# end

data/spec/away/exe_spec.rb

@@ -0,0 +1,20 @@
+# require 'spec_helper'
+# require 'wukong'
+#
+# describe 'auto runner for scripts', :helpers => true do
+#
+#   context 'at exit hook' do
+#
+#     it 'defines a hook to run on exit'
+#
+#     it 'does not run a script twice'
+#
+#     it 'runs a script based on its invoked name (so that it works with symlinks)'
+#
+#   end
+#
+# end
+#
+# describe 'wukong shell at exit hook' do
+#   it 'runs pry at correct point'
+# end

data/spec/away/flow_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+require 'wukong'
+
+# describe 'wukong', :helpers => true do
+#   subject{ described_class.new(:example) }
+#
+#   describe Wukong::Flow do
+#     let(:test_sink){ test_array_sink }
+#     let(:example_flow) do
+#       test_sink = test_sink()
+#       Wukong.flow(:simple) do
+#         source(:iter, 1..100) | limit(7) | test_sink
+#       end
+#     end
+#
+#     context '#add_stage' do
+#       it 'creates right subclass of Wukong::Source' do
+#         subject.add_stage(:source, :iter, []).should be_a(Wukong::Source::Iter)
+#       end
+#     end
+#
+#     it 'works with a simple example' do
+#       example_flow.run
+#       test_array_sink.records.should == (1..7).to_a
+#     end
+#
+#     # context '#run' do
+#     #   let(:test_sink){ mock }
+#     #   it 'announces events and calls methods in right order' do
+#     #     test_sink = test_sink()
+#     #     test_sink.should_receive(:tell).with(:beg_stream).ordered
+#     #     test_sink.should_receive(:call).exactly(7).times.ordered
+#     #     test_sink.should_receive(:finally).once.ordered
+#     #     test_sink.should_receive(:tell).with(:end_stream).ordered
+#     #     example_flow.run
+#     #   end
+#     # end
+#
+#     context '#stdin' do
+#       its(:stdin){ should be_a(Wukong::Source::Iter) }
+#     end
+#     context '#stdout' do
+#       its(:stdout){ should be_a(Wukong::Sink::Stdout) }
+#     end
+#     context '#stderr' do
+#       its(:stderr){ should be_a(Wukong::Sink::Stderr) }
+#     end
+#
+#   end
+#
+#   describe Wukong do
+#     context '.streamer' do
+#       subject{ Wukong.streamer('from_meth'){ def call(rec) rec.reverse ; end ; def bob() 1 ; end } }
+#       it 'raises an error if the handle is not a valid identifier' do
+#         ->{ Wukong.streamer('1love')      }.should raise_error(ArgumentError, /no funny/)
+#         ->{ Wukong.streamer('this/that')  }.should raise_error(ArgumentError, /no funny/)
+#         ->{ Wukong.streamer('This::That') }.should raise_error(ArgumentError, /no funny/)
+#       end
+#
+#       it{ should < Wukong::Streamer }
+#       it{ should be_method_defined(:call) }
+#       it{ should be_method_defined(:bob) }
+#
+#       it 'defines a constant in Wukong::Streamer' do
+#         subject.to_s.should == 'Wukong::Streamer::FromMeth'
+#         Wukong::Streamer.should be_const_defined(:FromMeth)
+#       end
+#       it 'raises if already defined' do
+#         subject
+#         ->{ Wukong.streamer('from_meth') }.should raise_error(ArgumentError, /already defined/i)
+#       end
+#
+#       it 'works as expected' do
+#         subject.new.call("hi mom").should == "mom ih"
+#         subject.new.bob.should == 1
+#       end
+#
+#       after{ Wukong::Streamer.send(:remove_const, :FromMeth) if Wukong::Streamer.const_defined?(:FromMeth) }
+#     end
+#   end
+#
+# end

data/spec/away/graph_spec.rb

@@ -0,0 +1,6 @@
+# require 'spec_helper'
+# require 'wukong'
+#
+# describe Wukong::Graph, :helpers => true do
+#
+# end

data/spec/away/job_spec.rb

@@ -0,0 +1,15 @@
+# require 'spec_helper'
+# require 'wukong'
+#
+# describe Wukong::Job, :helpers => true do
+#
+#   context '#output_dir' do
+#     it 'has filename helpers'
+#   end
+#
+#   context '#dry_run' do
+#     it 'does nothing when dry run flag is set'
+#
+#     it 'announces each foregone action using Log.info'
+#   end
+# end

data/spec/away/rake_compat_spec.rb

@@ -0,0 +1,9 @@
+# require 'spec_helper'
+# require 'wukong'
+#
+# describe 'rake compatibility', :helpers => true do
+#   it 'loads rake'
+#   it 'warns if the rake DSL is included at global level'
+#
+#   it 'plays nice with Rails'
+# end

data/spec/away/script_spec.rb

@@ -0,0 +1,81 @@
+# require 'spec_helper'
+# require 'wukong/runner/hadoop'
+#
+# describe "Wukong::Runner::Hadoop" do
+#   before do
+#     ARGV.replace []
+#     @script = Wukong::Script.new 'mapper', 'reducer'
+#   end
+#
+#   describe 'initialize' do
+#     it 'sets :reduce_tasks to 0 if reducer_klass is nil and no reduce_command or explicit setting' do
+#       @script = Wukong::Script.new 'mapper', nil
+#       @script.options[:reduce_tasks].should == 0
+#     end
+#     it 'respects :reduce_tasks if set even if reducer_klass is nil' do
+#       @script = Wukong::Script.new 'mapper', nil, :reduce_tasks => 1
+#       @script.options[:reduce_tasks].should == 1
+#     end
+#     it "doesn't set :reduce_tasks reduce_command is given" do
+#       @script = Wukong::Script.new 'mapper', nil, :reduce_command => 1
+#       @script.options[:reduce_tasks].should be_nil
+#     end
+#     it 'sets mapper_klass in initializer' do
+#       @script.mapper_klass.should == 'mapper'
+#     end
+#     it 'sets reducer_klass in initializer' do
+#       @script.reducer_klass.should == 'reducer'
+#     end
+#   end
+#
+#   describe 'child processes' do
+#     it 'calls self if a mapper_klass is set' do
+#       @script.should_receive(:ruby_interpreter_path).and_return('/path/to/ruby')
+#       @script.should_receive(:this_script_filename).and_return('/path/to/this_script')
+#       @script.map_command.should == %Q{/path/to/ruby /path/to/this_script --map }
+#     end
+#     it 'calls default_mapper if nil mapper_klass and no map_command is set' do
+#       @script = Wukong::Script.new nil, 'reducer', :default_mapper => 'default_mapper'
+#       @script.map_command.should == 'default_mapper'
+#     end
+#     it 'calls map_command if nil mapper_klass and map_command is set' do
+#       @script = Wukong::Script.new nil, 'reducer', :map_command => 'map_command', :default_mapper => 'default_mapper'
+#       @script.map_command.should == 'map_command'
+#     end
+#
+#     it 'calls self if a reducer_klass is set' do
+#       @script.should_receive(:ruby_interpreter_path).and_return('/path/to/ruby')
+#       @script.should_receive(:this_script_filename).and_return('/path/to/this_script')
+#       @script.reduce_command.should == %Q{/path/to/ruby /path/to/this_script --reduce }
+#     end
+#     it 'calls default_reducer if nil reducer_klass and no reduce_command is set' do
+#       @script = Wukong::Script.new 'mapper', nil, :default_reducer => 'default_reducer'
+#       @script.reduce_command.should == 'default_reducer'
+#     end
+#     it 'calls reduce_command if nil reducer_klass and reduce_command is set' do
+#       @script = Wukong::Script.new 'mapper', nil, :reduce_command => 'reduce_command', :default_reducer => 'default_reducer'
+#       @script.reduce_command.should == 'reduce_command'
+#     end
+#
+#     it 'runs script | sort | script when in local mode' do
+#       @script.should_receive(:run_mode).and_return('local')
+#       @script.should_receive(:map_command).and_return('map_command')
+#       @script.should_receive(:reduce_command).and_return('reduce_command')
+#       @script.runner_command("/path/in", "/path/out").should == %Q{ cat '/path/in' | map_command | sort | reduce_command > '/path/out'}
+#     end
+#
+#     it 'calls out to hadoop when in non-local mode' do
+#       @script.should_receive(:run_mode).and_return('hadoop')
+#       @script.should_receive(:hadoop_command).and_return('hadoop_command whee!')
+#       @script.runner_command("/path/in", "/path/out").should == 'hadoop_command whee!'
+#     end
+#   end
+#
+#   describe 'runner phase'
+#   it 'preserves non-internal-to-wukong params in non_wukong_params' do
+#     @script.options[:foo] = 'bar'
+#     @script.non_wukong_params.should == "--foo=bar"
+#   end
+#
+#
+# end

data/spec/examples/dataflow/parse_apache_logs_spec.rb

@@ -0,0 +1,8 @@
+# require 'spec_helper'
+# require 'wukong'
+# 
+# describe_example_script(:parsing, 'dataflow/parsing.rb') do
+#   it 'runs' do
+#     Wukong::LocalRunner.run(subject, :default)
+#   end
+# end

data/spec/examples/dataflow/parsing_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+require 'wukong'
+
+describe_example_script(:parse_apache_logs, 'dataflow/parse_apache_logs.rb') do
+  it 'runs' do
+    out, err = Gorillib::TestHelpers.capture_output do
+      Wukong::LocalRunner.receive(:flow => subject) do
+        run :default
+      end
+    end
+    out.string.split("\n").first.should =~ /\{\"ip_address\":\"[\d\.]+\",.*\"}/ 
+  end
+end

data/spec/examples/dataflow/simple_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+require 'wukong'
+
+describe_example_script(:simple, 'dataflow/simple.rb') do
+  it 'runs' do
+    Wukong::LocalRunner.run(subject, :default)
+  end
+end

data/spec/examples/dataflow/telegram_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+require 'wukong'
+
+describe_example_script(:telegram, 'dataflow/telegram.rb') do
+  it 'runs' do
+    Wukong::LocalRunner.run(subject, :default)
+  end
+
+  context 'Recompose processor' do
+    subject{ Wukong::Widget::Recompose }
+    its(:field_names){ should include(:break_length) }
+
+    let(:words    ){
+      # 0    5    1    5    2    5    3    5    4    5    5    5    6    5    7    5    8
+      %w[
+        If names be not correct, language is not in accordance with
+        the truth of things.  If language be not in accordance with
+        the truth of things, affairs cannot be carried on to success. ] }
+
+    context '#process' do
+      it 'breaks lines correctly' do
+        (2..80).each do |len|
+          # run the data flow into an array sink
+          test_sink = Wukong::Sink::ArraySink.new
+          rc = subject.new(:break_length => len, :output => test_sink )
+          words.each{|word| rc.process(word) }
+          rc.stop
+          # start and end are correct
+          test_sink.records.first.should =~ /^If/
+          test_sink.records.last.should =~ /success\.$/
+          # lines should be as long as possible, but not longer
+          test_sink.records[0..-2].zip(test_sink.records[1..-1]) do |line, nextl|
+            nextw = nextl.split[0]
+            ((line.length <= len) || line !~ /\s/).should be_true
+            (line.length + nextw.length + 1 > len).should be_true
+          end
+        end
+      end
+
+    end
+  end
+
+end

data/spec/examples/graph/minimum_spanning_tree_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+require 'wukong'
+require 'wukong/local_runner'
+
+load Pathname.path_to(:examples, 'graph/minimum_spanning_tree.rb')
+
+describe 'Minimum Spanning Tree', :examples_spec => true, :helpers => true do
+
+  context Wukong::Widget::DisjointForest do
+    subject{ Wukong::Widget::DisjointForest.new }
+
+    context 'operations' do
+      before do
+        %w[ AUS DFW ATL JFK SFO LGA LAX ].each{|el| subject.add el }
+        subject.union('DFW', 'AUS')
+        subject.union('ATL', 'JFK')
+        subject.union('ATL', 'DFW')
+      end
+      
+      context '#find' do
+        it 'collapses elements into a shallow tree during a find' do
+          subject.parent['ATL'].should == 'JFK'
+          subject.parent['JFK'].should == 'AUS'
+          subject.find('ATL').should == 'AUS'
+          subject.parent['ATL'].should == 'AUS'
+        end
+      end
+      context '#union' do
+        it 'joins shallow tree to deep tree' do
+        end
+      end
+    end
+    
+  end
+end

data/spec/examples/text/pig_latin_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+require 'wukong'
+require 'wukong/local_runner'
+
+load Pathname.path_to(:examples, 'text/pig_latin.rb')
+
+describe 'Pig Latin', :examples_spec => true, :helpers => true do
+
+  context 'processor' do
+    subject{ Wukong::Widget::PigLatinize.new }
+    it 'breaks text into pig latin' do
+      subject.should_receive(:emit).with("Iway indfay ethay astramipay otay ebay ethay ostmay ensualsay ofway allway ethay altedsay uredcay eatsmay.")
+      subject.process("I find the pastrami to be the most sensual of all the salted cured meats.")
+    end
+  end
+
+  it 'runs' do
+    Wukong::LocalRunner.run(ExampleUniverse.dataflow(:pig_latin), :default)
+  end
+
+end