wukong 3.0.0.pre → 3.0.0.pre2

data/old/wukong/typed_struct.rb

@@ -1,12 +0,0 @@
-class TypedStruct < Struct
-  def self.new *members_types
-    members, mtypes = members_types.transpose
-    klass           = Struct.new *members.map(&:to_sym)
-    klass.class_eval do
-      cattr_accessor :mtypes, :members_types
-      self.mtypes        = mtypes
-      self.members_types = Hash.zip(members, mtypes)
-    end
-    klass
-  end
-end

data/spec/away/encoding_spec.rb

@@ -1,32 +0,0 @@
-# require 'spec_helper'
-# require 'wukong/encoding'
-#
-# describe "Wukong encoding" do
-#
-#   it 'en/decodes to xml by default' do
-#     Wukong.encode_str("&"           ).should == '&'
-#     Wukong.decode_str("&"       ).should == '&'
-#   end
-#   it 'en/decodes to xml with :xml' do
-#     Wukong.encode_str("&", :xml     ).should == '&'
-#     Wukong.decode_str("&", :xml ).should == '&'
-#   end
-#   it 'url en/decodes with :url' do
-#     Wukong.encode_str("&", :url     ).should == '%26'
-#     Wukong.decode_str("%26", :url   ).should == '&'
-#   end
-#   { "'" => "'", "\t" => "	", "\n" => "
", nil => '',}.each do |raw, enc|
-#     it 'encodes #{raw} to #{enc}' do
-#       Wukong.encode_str(raw, :xml   ).should == enc
-#     end
-#     it 'decodes #{enc} to #{raw}' do
-#       Wukong.decode_str(enc, :xml   ).should == raw.to_s
-#     end
-#   end
-#   ["normal_string with %punctuation should `not be molested", ""].each do |str|
-#     it 'doesn\'t change #{str}' do
-#       Wukong.encode_str(str, :xml   ).should == str
-#     end
-#   end
-#
-# end

data/spec/away/exe_spec.rb

@@ -1,20 +0,0 @@
-# require 'spec_helper'
-# require 'wukong'
-#
-# describe 'auto runner for scripts', :helpers => true do
-#
-#   context 'at exit hook' do
-#
-#     it 'defines a hook to run on exit'
-#
-#     it 'does not run a script twice'
-#
-#     it 'runs a script based on its invoked name (so that it works with symlinks)'
-#
-#   end
-#
-# end
-#
-# describe 'wukong shell at exit hook' do
-#   it 'runs pry at correct point'
-# end

data/spec/away/flow_spec.rb

@@ -1,82 +0,0 @@
-require 'spec_helper'
-require 'wukong'
-
-# describe 'wukong', :helpers => true do
-#   subject{ described_class.new(:example) }
-#
-#   describe Wukong::Flow do
-#     let(:test_sink){ test_array_sink }
-#     let(:example_flow) do
-#       test_sink = test_sink()
-#       Wukong.flow(:simple) do
-#         source(:iter, 1..100) | limit(7) | test_sink
-#       end
-#     end
-#
-#     context '#add_stage' do
-#       it 'creates right subclass of Wukong::Source' do
-#         subject.add_stage(:source, :iter, []).should be_a(Wukong::Source::Iter)
-#       end
-#     end
-#
-#     it 'works with a simple example' do
-#       example_flow.run
-#       test_array_sink.records.should == (1..7).to_a
-#     end
-#
-#     # context '#run' do
-#     #   let(:test_sink){ mock }
-#     #   it 'announces events and calls methods in right order' do
-#     #     test_sink = test_sink()
-#     #     test_sink.should_receive(:tell).with(:beg_stream).ordered
-#     #     test_sink.should_receive(:call).exactly(7).times.ordered
-#     #     test_sink.should_receive(:finally).once.ordered
-#     #     test_sink.should_receive(:tell).with(:end_stream).ordered
-#     #     example_flow.run
-#     #   end
-#     # end
-#
-#     context '#stdin' do
-#       its(:stdin){ should be_a(Wukong::Source::Iter) }
-#     end
-#     context '#stdout' do
-#       its(:stdout){ should be_a(Wukong::Sink::Stdout) }
-#     end
-#     context '#stderr' do
-#       its(:stderr){ should be_a(Wukong::Sink::Stderr) }
-#     end
-#
-#   end
-#
-#   describe Wukong do
-#     context '.streamer' do
-#       subject{ Wukong.streamer('from_meth'){ def call(rec) rec.reverse ; end ; def bob() 1 ; end } }
-#       it 'raises an error if the handle is not a valid identifier' do
-#         ->{ Wukong.streamer('1love')      }.should raise_error(ArgumentError, /no funny/)
-#         ->{ Wukong.streamer('this/that')  }.should raise_error(ArgumentError, /no funny/)
-#         ->{ Wukong.streamer('This::That') }.should raise_error(ArgumentError, /no funny/)
-#       end
-#
-#       it{ should < Wukong::Streamer }
-#       it{ should be_method_defined(:call) }
-#       it{ should be_method_defined(:bob) }
-#
-#       it 'defines a constant in Wukong::Streamer' do
-#         subject.to_s.should == 'Wukong::Streamer::FromMeth'
-#         Wukong::Streamer.should be_const_defined(:FromMeth)
-#       end
-#       it 'raises if already defined' do
-#         subject
-#         ->{ Wukong.streamer('from_meth') }.should raise_error(ArgumentError, /already defined/i)
-#       end
-#
-#       it 'works as expected' do
-#         subject.new.call("hi mom").should == "mom ih"
-#         subject.new.bob.should == 1
-#       end
-#
-#       after{ Wukong::Streamer.send(:remove_const, :FromMeth) if Wukong::Streamer.const_defined?(:FromMeth) }
-#     end
-#   end
-#
-# end

data/spec/away/graph_spec.rb

@@ -1,6 +0,0 @@
-# require 'spec_helper'
-# require 'wukong'
-#
-# describe Wukong::Graph, :helpers => true do
-#
-# end

data/spec/away/job_spec.rb

@@ -1,15 +0,0 @@
-# require 'spec_helper'
-# require 'wukong'
-#
-# describe Wukong::Job, :helpers => true do
-#
-#   context '#output_dir' do
-#     it 'has filename helpers'
-#   end
-#
-#   context '#dry_run' do
-#     it 'does nothing when dry run flag is set'
-#
-#     it 'announces each foregone action using Log.info'
-#   end
-# end

data/spec/away/rake_compat_spec.rb

@@ -1,9 +0,0 @@
-# require 'spec_helper'
-# require 'wukong'
-#
-# describe 'rake compatibility', :helpers => true do
-#   it 'loads rake'
-#   it 'warns if the rake DSL is included at global level'
-#
-#   it 'plays nice with Rails'
-# end

data/spec/away/script_spec.rb

@@ -1,81 +0,0 @@
-# require 'spec_helper'
-# require 'wukong/runner/hadoop'
-#
-# describe "Wukong::Runner::Hadoop" do
-#   before do
-#     ARGV.replace []
-#     @script = Wukong::Script.new 'mapper', 'reducer'
-#   end
-#
-#   describe 'initialize' do
-#     it 'sets :reduce_tasks to 0 if reducer_klass is nil and no reduce_command or explicit setting' do
-#       @script = Wukong::Script.new 'mapper', nil
-#       @script.options[:reduce_tasks].should == 0
-#     end
-#     it 'respects :reduce_tasks if set even if reducer_klass is nil' do
-#       @script = Wukong::Script.new 'mapper', nil, :reduce_tasks => 1
-#       @script.options[:reduce_tasks].should == 1
-#     end
-#     it "doesn't set :reduce_tasks reduce_command is given" do
-#       @script = Wukong::Script.new 'mapper', nil, :reduce_command => 1
-#       @script.options[:reduce_tasks].should be_nil
-#     end
-#     it 'sets mapper_klass in initializer' do
-#       @script.mapper_klass.should == 'mapper'
-#     end
-#     it 'sets reducer_klass in initializer' do
-#       @script.reducer_klass.should == 'reducer'
-#     end
-#   end
-#
-#   describe 'child processes' do
-#     it 'calls self if a mapper_klass is set' do
-#       @script.should_receive(:ruby_interpreter_path).and_return('/path/to/ruby')
-#       @script.should_receive(:this_script_filename).and_return('/path/to/this_script')
-#       @script.map_command.should == %Q{/path/to/ruby /path/to/this_script --map }
-#     end
-#     it 'calls default_mapper if nil mapper_klass and no map_command is set' do
-#       @script = Wukong::Script.new nil, 'reducer', :default_mapper => 'default_mapper'
-#       @script.map_command.should == 'default_mapper'
-#     end
-#     it 'calls map_command if nil mapper_klass and map_command is set' do
-#       @script = Wukong::Script.new nil, 'reducer', :map_command => 'map_command', :default_mapper => 'default_mapper'
-#       @script.map_command.should == 'map_command'
-#     end
-#
-#     it 'calls self if a reducer_klass is set' do
-#       @script.should_receive(:ruby_interpreter_path).and_return('/path/to/ruby')
-#       @script.should_receive(:this_script_filename).and_return('/path/to/this_script')
-#       @script.reduce_command.should == %Q{/path/to/ruby /path/to/this_script --reduce }
-#     end
-#     it 'calls default_reducer if nil reducer_klass and no reduce_command is set' do
-#       @script = Wukong::Script.new 'mapper', nil, :default_reducer => 'default_reducer'
-#       @script.reduce_command.should == 'default_reducer'
-#     end
-#     it 'calls reduce_command if nil reducer_klass and reduce_command is set' do
-#       @script = Wukong::Script.new 'mapper', nil, :reduce_command => 'reduce_command', :default_reducer => 'default_reducer'
-#       @script.reduce_command.should == 'reduce_command'
-#     end
-#
-#     it 'runs script | sort | script when in local mode' do
-#       @script.should_receive(:run_mode).and_return('local')
-#       @script.should_receive(:map_command).and_return('map_command')
-#       @script.should_receive(:reduce_command).and_return('reduce_command')
-#       @script.runner_command("/path/in", "/path/out").should == %Q{ cat '/path/in' | map_command | sort | reduce_command > '/path/out'}
-#     end
-#
-#     it 'calls out to hadoop when in non-local mode' do
-#       @script.should_receive(:run_mode).and_return('hadoop')
-#       @script.should_receive(:hadoop_command).and_return('hadoop_command whee!')
-#       @script.runner_command("/path/in", "/path/out").should == 'hadoop_command whee!'
-#     end
-#   end
-#
-#   describe 'runner phase'
-#   it 'preserves non-internal-to-wukong params in non_wukong_params' do
-#     @script.options[:foo] = 'bar'
-#     @script.non_wukong_params.should == "--foo=bar"
-#   end
-#
-#
-# end

data/spec/hanuman/graphviz_spec.rb

@@ -1,29 +0,0 @@
-require 'spec_helper'
-
-require 'wukong'
-require 'hanuman/graphvizzer'
-require 'hanuman/graphviz'
-
-describe 'Graphviz View' do
-  describe 'Cherry Pie Example', :if => GRAPHVIZ, :examples_spec => true, :helpers => true do
-    it 'makes a png' do
-      require Pathname.path_to(:examples, 'workflow/cherry_pie.rb')
-      gv = Warrant.to_graphviz
-
-      basename = Pathname.path_to(:tmp, 'cherry_pie')
-      gv.save(basename, 'png')
-      # puts File.read("#{basename}.dot")
-    end
-  end
-
-  describe 'Telegram Dataflow Example', :if => GRAPHVIZ, :examples_spec => true, :helpers => true do
-    it 'makes a png' do
-      require Pathname.path_to(:examples, 'dataflow/telegram.rb')
-      gv = ExampleUniverse.to_graphviz
-
-      basename = Pathname.path_to(:tmp, 'telegram')
-      gv.save(basename, 'png')
-      # puts File.read("#{basename}.dot")
-    end
-  end
-end

data/spec/hanuman/slot_spec.rb

@@ -1,2 +0,0 @@
-require 'spec_helper'
-require 'hanuman'

data/spec/support/examples_helper.rb

@@ -1,10 +0,0 @@
-RSpec::Core::DSL.module_eval do
-  def describe_example_script(example_name, source_file, attrs={}, &block)
-    load Pathname.path_to(:examples, source_file)
-
-    describe "Example: #{example_name}", attrs.merge(:examples_spec => true, :helpers => true) do
-      subject{ ExampleUniverse.dataflow(example_name) }
-      instance_eval(&block)
-    end
-  end
-end

data/spec/support/streamer_test_helpers.rb

@@ -1,6 +0,0 @@
-
-shared_context 'streamer', :streamers => true do
-  let(:identity_streamer){ Wukong::Streamer::Identity.new }
-
-  let(:mock_record){ mock }
-end

data/spec/support/wukong_widget_helpers.rb

@@ -1,66 +0,0 @@
-shared_context 'widgets', :helpers => true do
-  let(:sample_dataflow){ Wukong.dataflow(:sample) }
-  let(:next_stage){ mock('next stage') }
-
-  def mock_next_stage(obj=nil)
-    (obj ||= subject).set_output next_stage
-  end
-end
-
-shared_examples_for 'a processor' do
-
-  it{ should respond_to(:process) }
-  it{ should respond_to(:setup) }
-  it{ should respond_to(:stop) }
-  it{ should respond_to(:notify) }
-  it{ should respond_to(:report) }
-  its(:report){ should be_a(Hash) }
-end
-
-shared_examples_for "a filter processor" do |objects|
-  it_behaves_like 'a processor'
-
-  it 'accepts good objects' do
-    objects[:good].each do |obj|
-      subject.select?(obj).should be_true
-      subject.reject?(obj).should be_false
-    end
-  end unless objects[:good].empty?
-
-  it 'rejects bad objects' do
-    objects[:bad].each do |obj|
-      subject.select?(obj).should be_false
-      subject.reject?(obj).should be_true
-    end
-  end unless objects[:bad].empty?
-
-  context '#process' do
-    before{ mock_next_stage }
-
-    objects[:good].each do |obj|
-      it "passes along objects like #{obj.inspect}" do
-        next_stage.should_receive(:process).with(obj)
-        subject.process(obj)
-      end
-    end unless objects[:good].empty?
-    objects[:bad].each do |obj|
-      it "drops objects like #{obj.inspect}" do
-        next_stage.should_not_receive(:process)
-        subject.process(obj)
-      end
-    end unless objects[:bad].empty?
-
-    it "passes along good objects if select? is true" do
-      subject.stub(:select?).and_return(true)
-      subject.stub(:reject?).and_return(false)
-      next_stage.should_receive(:process).with(mock_record)
-      subject.process(mock_record)
-    end
-    it "drops objects if reject? is true" do
-      subject.stub(:select?).and_return(false)
-      subject.stub(:reject?).and_return(true)
-      next_stage.should_not_receive(:process)
-      subject.process(mock_record)
-    end
-  end
-end

data/spec/wukong/processor_spec.rb

@@ -1,109 +0,0 @@
-require 'spec_helper'
-require 'wukong'
-
-describe :processors, :helpers => true, :widgets => true do
-  subject{ described_class.new }
-  let(:mock_dataflow){ md = mock('dataflow') ; md }
-
-  describe Wukong::Processor do
-    it_behaves_like 'it can be linked from'
-    it_behaves_like 'it can be linked into'
-  end
-
-  describe Wukong::Map do
-    it_behaves_like 'a processor'
-    let(:sample_proc){ ->(rec){ rec.reverse } }
-    subject{ described_class.new(sample_proc) }
-
-    it 'emits whatever the proc does' do
-      subject.should_receive(:emit).with("won ytineres")
-      subject.process("serenity now")
-    end
-
-    it 'accepts a proc or block arg' do
-      subject = sample_dataflow.map(->(rec){ rec.reverse })
-      subject.should_receive(:emit).with("won ytineres")
-      subject.process("serenity now")
-    end
-
-    it 'swallows a nil result' do
-      sample_proc.should_receive(:call).with(mock_val).and_return(nil)
-      subject.should_not_receive(:emit)
-      subject.process(mock_val)
-    end
-
-    it 'registers a dataflow helper `map`' do
-      st = sample_dataflow.map{|rec| rec.reverse }
-      st.should be_a(described_class)
-    end
-  end
-
-  describe Wukong::Foreach do
-    it_behaves_like 'a processor'
-    let(:sample_proc){ ->(rec){ emit rec.reverse } }
-    subject{ described_class.new(sample_proc) }
-
-    it 'calls the proc' do
-      subject.should_receive(:emit).with("won ytineres")
-      subject.process("serenity now")
-    end
-
-    it 'does not call emit on your behalf' do
-      subject = sample_dataflow.foreach{|rec| rec.reverse }
-      mock_val.stub(:reverse)
-      subject.should_not_receive(:emit)
-      subject.process(mock_val)
-    end
-
-    it 'accepts a proc or block arg' do
-      subject = sample_dataflow.foreach{|rec| emit rec.reverse }
-      subject.should_receive(:emit).with("won ytineres")
-      subject.process("serenity now")
-    end
-
-    it 'registers a dataflow helper `foreach`' do
-      st = sample_dataflow.foreach{|rec| rec.reverse }
-      st.should be_a(described_class)
-    end
-  end
-
-  describe Wukong::Flatten do
-    it_behaves_like 'a processor'
-    it 'emits each item in each input' do
-      subject.set_output test_sink
-      [ [:this, :that], [], 1..5, { :a => :b} ].each{|rec| subject.process(rec) }
-      test_sink.records.should == [:this, :that, 1, 2, 3, 4, 5, [:a, :b]]
-    end
-    it 'registers a dataflow helper `flatten`' do
-      st = sample_dataflow.flatten
-      st.should be_a(described_class)
-    end
-  end
-
-  describe Wukong::AsIs do
-    it_behaves_like 'a processor'
-    it 'emits each record' do
-      subject.should_receive(:emit).with(:this)
-      subject.should_receive(:emit).with(:that)
-      [:this, :that].each{|rec| subject.process(rec) }
-    end
-    it 'registers a dataflow helper `as_is`' do
-      st = sample_dataflow.as_is
-      st.should be_a(described_class)
-    end
-  end
-
-  describe Wukong::Null do
-    it_behaves_like 'a processor'
-    it 'emits each record' do
-      subject.should_not_receive(:emit)
-      [:this, :that].each{|rec| subject.process(rec) }
-    end
-    it 'registers a dataflow helper `null`' do
-      st = sample_dataflow.null
-      st.should be_a(described_class)
-    end
-  end
-
-
-end