wukong 3.0.0.pre3 → 3.0.0

data/spec/wukong/processor_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe Wukong::Processor do
+
+  subject { Wukong::Processor.new }  
+
+  describe "has an interface" do
+    it{ should respond_to(:setup)    }
+    it{ should respond_to(:process)  }
+    it{ should respond_to(:finalize) }
+    it{ should respond_to(:stop)     }
+    it{ should respond_to(:notify)   }
+  end
+
+  describe "default process method" do
+    it "yields the original input record by default on process" do
+      expect { |b| subject.process(1, &b) }.to yield_with_args(1)
+    end
+  end
+  
+end
+  

data/spec/wukong/runner_spec.rb

@@ -1,12 +1,132 @@
 require 'spec_helper'
-# require 'wukong'
 
-# describe :runner, :helpers => true do
+describe Wukong::Runner do
 
-#   context 'tiny_count example script' do
-#     it 'is shorter than a tweet' do
-#       example_script_contents('tiny_count.rb').length.should < 140
-#     end
+  context "can boot itself by" do
 
-#   end
-# end
+    describe "loading" do
+      it "files passed on the command-line" do
+        loadable = examples_dir('loadable.rb')
+        generic_runner(loadable) do
+          should_receive(:load_ruby_file).with(loadable)
+        end
+      end
+      it "files when its in a deploy pack" do
+        generic_runner { should_receive(:load_deploy_pack) }
+      end
+    end
+
+    describe "configuring" do
+      subject do
+        Class.new(Wukong::Runner).tap do |configured|
+          configured.class_eval do
+            usage       'a nice usage message'
+            description 'a lovely description'
+          end
+        end
+      end
+      
+      it "a usage message" do
+        runner(subject, 'wu-generic').settings.usage.should =~ /^usage: .* a nice usage message$/
+      end
+
+      it "a description" do
+        runner(subject, 'wu-generic').settings.description.should == 'a lovely description'
+      end
+
+      it "plugins" do
+        generic_runner { Wukong.should_receive(:configure_plugins) }
+      end
+    end
+
+    describe "resolving" do
+      it "doesn't move on if resolve causes an error" do
+        generic_runner do
+          settings.should_receive(:resolve!).and_raise(RuntimeError)
+          should_not_receive(:setup)
+          should_not_receive(:validate)
+          should_not_receive(:run)
+        end
+      end
+    end
+
+    describe "setting up" do
+      it "asks plugins to boot" do
+        generic_runner do
+          Wukong.should_receive(:boot_plugins)
+        end
+      end
+    end
+
+    describe "validating" do
+      it "should run if validatation passes" do
+        generic_runner do
+          should_receive(:validate).and_return(true)
+          should_receive(:run)
+        end
+      end
+      it "dies if validate fails" do
+        generic_runner do
+          should_receive(:validate).and_return(false)
+          should_not_receive(:run)
+          should_receive(:die)
+        end
+      end
+    end
+  end
+
+  context "situating itself within a deploy pack" do
+    context "in an arbitrary directory" do
+      let(:dir) { examples_dir('empty') }
+      before    { FileUtils.cd(dir)     }
+      subject   { generic_runner            }
+      its(:deploy_pack_dir)  { should == '/'                      }
+      its(:environment_file) { should == '/config/environment.rb' }
+      its(:in_deploy_pack?)  { should be_false                    }
+    end
+
+    context "when BUNDLE_GEMFILE points at a regular Ruby project" do
+      let(:dir)             { examples_dir('empty')       }
+      let(:deploy_pack_dir) { examples_dir('ruby_project') }
+      before do
+        FileUtils.cd(dir)
+        ENV.stub!(:[]).with("BUNDLE_GEMFILE").and_return(File.join(deploy_pack_dir, 'Gemfile'))
+      end
+      subject   { generic_runner }
+      its(:deploy_pack_dir)  { should == '/'                      }
+      its(:environment_file) { should == '/config/environment.rb' }
+      its(:in_deploy_pack?)  { should be_false                    }
+    end
+    
+    context "when BUNDLE_GEMFILE points at a deploy pack" do
+      let(:dir)             { examples_dir('empty')       }
+      let(:deploy_pack_dir) { examples_dir('deploy_pack') }
+      before do
+        FileUtils.cd(dir)
+        ENV.stub!(:[]).with("BUNDLE_GEMFILE").and_return(File.join(deploy_pack_dir, 'Gemfile'))
+      end
+      subject   { generic_runner }
+      its(:deploy_pack_dir)  { should == deploy_pack_dir.to_s                               }
+      its(:environment_file) { should == File.join(deploy_pack_dir, 'config/environment.rb')}
+      its(:in_deploy_pack?)  { should be_true                                               }
+    end
+    
+    context "in an arbitrary Ruby project with a Gemfile" do
+      let(:dir) { examples_dir('ruby_project') }
+      before    { FileUtils.cd(dir)            }
+      subject   { generic_runner                   }
+      its(:deploy_pack_dir)  { should == '/'                      }
+      its(:environment_file) { should == '/config/environment.rb' }
+      its(:in_deploy_pack?)  { should be_false                    }
+    end
+
+    context "in a deploy pack with a Gemfile and a config/environment.rb" do
+      let(:dir) { examples_dir('deploy_pack')  }
+      before    { FileUtils.cd(dir)            }
+      subject   { generic_runner                       }
+      its(:deploy_pack_dir)  { should == dir                                    }
+      its(:environment_file) { should == File.join(dir, 'config/environment.rb')}
+      its(:in_deploy_pack?)  { should be_true                                   }
+    end
+  end
+end

data/spec/wukong/widget/filters_spec.rb

@@ -2,21 +2,21 @@ require 'spec_helper'
 
 describe "Filters" do
 
-  context :null do
+  describe :null do
     it_behaves_like 'a processor', :named => :null
     it "should not pass anything, ever" do
       processor.given('', 3, 'hi', nil).should emit(0).records
     end
   end
 
-  context :identity do
+  describe :identity do
     it_behaves_like 'a processor', :named => :identity
     it "should pass everything, always" do
       processor.given('', 3, 'hi', nil).should emit('', 3, 'hi', nil)
     end
   end
   
-  context :regexp do
+  describe :regexp do
     it_behaves_like 'a processor', :named => :regexp
     it "should pass everything given no 'match' argument" do
       processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
@@ -26,7 +26,7 @@ describe "Filters" do
     end
   end
 
-  context :not_regexp do
+  describe :not_regexp do
     it_behaves_like 'a processor', :named => :not_regexp
     it "should pass everything given no 'match' argument" do
       processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
@@ -36,7 +36,7 @@ describe "Filters" do
     end
   end
 
-  context :limit do
+  describe :limit do
     it_behaves_like 'a processor', :named => :limit
     it "should pass everything given no 'max' argument" do
       processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
@@ -46,16 +46,34 @@ describe "Filters" do
     end
   end
 
-  context :sample do
+  describe :sample do
     it_behaves_like 'a processor', :named => :sample
     it "should pass everything given no 'fraction' argument" do
       processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
     end
-    it "should pass everything given no 'fraction' argument" do
-      processor(:fraction => 0.5).tap do |proc|
-        proc.should_receive(:rand).and_return(0.7, 0.1, 0.6)
-      end.given('snap', 'crackle', 'pop').should emit('crackle')
+    it "should pass a fraction of records matching its 'fraction' argument" do
+      processor(:fraction => 0.5) { |proc| proc.should_receive(:rand).and_return(0.7, 0.1, 0.6) }.given('snap', 'crackle', 'pop').should emit('crackle')
+    end
+  end
+
+  describe :head do
+    it_behaves_like 'a processor', :named => :head
+    it "should pass the first 10 records given no argument" do
+      processor.given(*(1..100).to_a).should emit(10).records
+    end
+    it "should pass the first n records" do
+      processor(:n => 5).given(*(1..100).to_a).should emit(5).records
     end
   end
 
+  describe :tail do
+    it_behaves_like 'a processor', :named => :tail
+    it "should pass all records given no argument" do
+      processor.given(*(1..100).to_a).should emit(100).records
+    end
+    it "should skip the first n records" do
+      processor(:n => 5).given(*(1..100).to_a).should emit(95).records
+    end
+  end
+  
 end

data/spec/wukong/widget/processors_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Wukong::Processor do
+describe "Processors" do
 
   let(:hsh) { { "hi" => "there", "top" => { "lower" => { "lowest" => "value" } } } }
   let(:ary) { ['1', 2, 'three'] }
@@ -12,8 +12,8 @@ describe Wukong::Processor do
       log = mock("logger")
       log.should_receive(:info).with('hi there')
       log.should_receive(:info).with('buddy')
-      processor(:logger) do
-        stub!(:log).and_return(log)
+      processor(:logger) do |proc|
+        proc.stub!(:log).and_return(log)
       end.given('hi there', 'buddy').should emit(0).records
     end
 
@@ -21,8 +21,8 @@ describe Wukong::Processor do
       log = mock("logger")
       log.should_receive(:debug).with('hi there')
       log.should_receive(:debug).with('buddy')
-      processor(:logger, level: :debug) do
-        stub!(:log).and_return(log)
+      processor(:logger, level: :debug) do |proc|
+        proc.stub!(:log).and_return(log)
       end.given('hi there', 'buddy').should emit(0).records
     end
   end

data/spec/wukong/widget/reducers/bin_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Wukong::Processor do
+describe "Reducers" do
   describe :bin do
     include_context "reducers"
     it_behaves_like 'a processor', :named => :bin
@@ -32,34 +32,34 @@ describe Wukong::Processor do
     end
 
     it "counts correctly in each bin" do
-      processor(num_bins: 5).given(*nums).should emit_tsv(*bins)
+      processor(num_bins: 5).given(*nums).should emit(*bins)
     end
 
     it "can express counts logarithmically" do
-      row = processor(num_bins: 5, log_counts: true).given(*nums).tsv_output.first
+      row = processor(num_bins: 5, log_counts: true).given(*nums).output.first
       row.size.should == 3
       row[2].to_f.should be_within(0.1).of(2.197)
     end
 
     it "can add a normalized frequency" do
-      row = processor(num_bins: 5, normalize: true).given(*nums).tsv_output.first
+      row = processor(num_bins: 5, normalize: true).given(*nums).output.first
       row.size.should == 4
       row[3].to_f.should be_within(0.1).of(0.18)
     end
 
     it "can add a normalized frequency and express counts logarithmically" do
-      row = processor(num_bins: 5, normalize: true, log_counts: true).given(*nums).tsv_output.first
+      row = processor(num_bins: 5, normalize: true, log_counts: true).given(*nums).output.first
       row.size.should == 4
       row[2].to_f.should be_within(0.1).of(2.197)
       row[3].to_f.should be_within(0.1).of(-1.715)
     end
 
     it "can bin on the fly given min, max, and num_bins options" do
-      output = processor(min: -30, max: 30, num_bins: 3) do
+      output = processor(min: -30, max: 30, num_bins: 3) do |proc|
         # we can bin on the fly
-        values.should_not_receive(:<<)
-        should_not_receive(:bin!) 
-      end.given(*nums).tsv_output
+        proc.values.should_not_receive(:<<)
+        proc.should_not_receive(:bin!) 
+      end.given(*nums).output
 
       output.size.should == 3
       output.first[0].to_f.should be_within(0.1).of(-30)
@@ -67,11 +67,11 @@ describe Wukong::Processor do
     end
 
     it "can bin on the fly given fixed bin edges" do
-      output = processor(edges: [0,1,5,10]) do
+      output = processor(edges: [0,1,5,10]) do |proc|
         # we can bin on the fly
-        values.should_not_receive(:<<)
-        should_not_receive(:bin!)
-      end.given(*nums).tsv_output
+        proc.values.should_not_receive(:<<)
+        proc.should_not_receive(:bin!)
+      end.given(*nums).output
       output.size.should == 3
       output[0][0].to_f.should be_within(0.1).of(0.0)
       output[0][1].to_f.should be_within(0.1).of(1.0)
@@ -82,7 +82,7 @@ describe Wukong::Processor do
     end
 
     it "can extract the value to bin by from an object" do
-      output = processor(by: 'data.n', min: 0).given(*json).tsv_output
+      output = processor(by: 'data.n', min: 0).given(*json).output
       output.size.should == 2
       output.first[0].to_f.should be_within(0.1).of(0.0)
       output.last[1].to_f.should be_within(0.1).of(100.0)

data/spec/wukong/widget/reducers/count_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Wukong::Processor do
+describe "Reducers" do
   describe :count do
     include_context "reducers"
     it_behaves_like 'a processor', :named => :count

data/spec/wukong/widget/reducers/group_spec.rb

@@ -1,20 +1,21 @@
 require 'spec_helper'
 
-describe Wukong::Processor do
+describe "Reducers" do
   describe :group do
     include_context "reducers"
     it_behaves_like 'a processor', :named => :group
 
-    let(:grouped_strings) { [['apple', '2'], ['banana', '1'], ['cookie', '1']]             }
-    let(:grouped_nums)    { [['', '2'], ['1', '1'], ['5', '1'], ['10', '1'], ['100', '1']] }
+    let(:grouped_strings)        { [['apple', 2], ['banana', 1], ['cookie', 1]]  }
+    let(:grouped_nums_from_json) { [[nil, 2], [1, 1], [5, 1], [10, 1], [100, 1]] }
+    let(:grouped_nums_from_tsv)  { [[nil, 2], ['1', 1], ['5', 1], ['10', 1], ['100', 1]] }
     it "will group single values" do
-      processor(:group).given(*strings.sort).should emit_tsv(*grouped_strings)
+      processor(:group).given(*strings.sort).should emit(*grouped_strings)
     end
     it "can group from within a JSON hash" do
-      proc = processor(:group, by: 'data.n').given(*json_sorted_n).should emit_tsv(*grouped_nums)
+      proc = processor(:group, by: 'data.n').given(*json_sorted_n).should emit(*grouped_nums_from_json)
     end
     it "can group from within a TSV row" do
-      proc = processor(:group, by: '3').given(*tsv_sorted).should emit_tsv(*grouped_nums)
+      proc = processor(:group, by: '3').given(*tsv_sorted).should emit(*grouped_nums_from_tsv)
     end
   end
 end

data/spec/wukong/widget/reducers/moments_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Wukong::Processor do
+describe "Reducers" do
   describe :moments do
     include_context "reducers"
     it_behaves_like 'a processor', :named => :moments
@@ -23,7 +23,7 @@ describe Wukong::Processor do
     end
 
     it "will leave off the standard deviation if desired" do
-      processor(:moments, group_by: 'outer', of: 'data.n', std_dev: false).given(*json_sorted_outer).should emit(
+      processor(:moments, group_by: 'outer', of: 'data.n', no_std_dev: true).given(*json_sorted_outer).should emit(
         {group: nil,      count: 2, results: {"data.n" => {}}},
         {group: 'apple',  count: 2, results: {"data.n"=>{:count=>2, :mean=>3.0   }}},
         {group: 'banana', count: 1, results: {"data.n"=>{:count=>1, :mean=>100.0 }}},

data/spec/wukong/widget/reducers/sort_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Wukong::Processor do
+describe "Reducers" do
   describe :sort do
     include_context "reducers"
     it_behaves_like 'a processor', :named => :sort

data/spec/wukong/widget/serializers_spec.rb

@@ -1,118 +1,114 @@
 require 'spec_helper'
 
-shared_context 'serializers', serializer: true do
-  let(:bad_record){ nil }
-  let(:serializer){ create_processor(self.class.top_level_description, on_error: :skip) }
-end
+describe "Serializers" do
 
-shared_examples_for 'a serializer' do
-  it_behaves_like 'a processor'
-end
+  describe :to_json do
+    it_behaves_like 'a processor', :named => :to_json
+    
+    let(:valid_record)    { { hi: 'there' }                          }
+    let(:record_as_json)  { '{"hi":"there"}'                         }
+    let(:model_as_json)   { '{"model":"json"}'                       }
+    let(:valid_model)     { double('model', to_json: model_as_json)  }
+    
+    it 'serializes records to JSON' do
+      processor.given(valid_record).should emit(record_as_json)
+    end
+    
+    it 'serializes records as pretty JSON when asked' do
+      processor(:pretty => true).given(valid_record).output.first.should include("\n")
+    end
 
-shared_examples_for 'a serializer that complains on bad recors', :handles_errors => true do
-  it 'handles errors on bad records' do
-    serializer.should_receive(:handle_error).with(bad_record, a_kind_of(Exception)).and_return(nil)
-    serializer.given(bad_record).should emit(0).records
+    it 'defers to models to let them serialize themselves as JSON' do
+      processor.given(valid_model).should emit(model_as_json)
+    end
   end
-end
 
-describe :to_json, serializer: true do
-  let(:valid_record) { { hi: 'there' }        }
-  it 'serializes valid records' do
-    serializer.given(valid_record).should emit('{"hi":"there"}')
-  end
 
-  it 'handles errors on bad records' do
-    MultiJson.should_receive(:dump).with(:unserializable_record, kind_of(Hash)).and_raise(StandardError)
-    serializer.should_receive(:handle_error).with(:unserializable_record, a_kind_of(Exception)).and_return(nil)
-    serializer.given(:unserializable_record).should emit(0).records
+  describe :from_json do
+    it_behaves_like 'a processor', :named => :from_json
+    
+    let(:valid_json)   { '{"hi": "there"}'                        }
+    let(:json_parsed)  { {"hi" => "there"}                        }
+    let(:invalid_json) { '{"832323:'                              }
+    
+    it 'deserializes valid JSON' do
+      processor.given(valid_json).should emit(json_parsed)
+    end
+    
+    it 'handles errors on invalid JSON' do
+      processor { |proc| proc.should_receive(:handle_error).with(invalid_json, kind_of(Exception)) }.given(invalid_json).should emit(0).records
+    end
   end
   
-  context 'pretty' do
-    let(:serializer){ create_processor(:to_json, pretty: true) }
+  describe :to_tsv do
+    it_behaves_like 'a processor', :named => :to_tsv
     
-    it 'prettifies valid records' do
-      serializer.given(valid_record).output.first.should include("\n")
-    end  
-  end
-
-  context 'given a model' do
-    let(:json_record) {  '{"foo":"bar"}'                      }
-    let(:valid_model) { double('model', to_json: json_record) }
+    let(:valid_record)   { ["foo", 2, :a]  }
+    let(:invalid_record) { nil             }
+    let(:record_as_tsv)  { "foo\t2\ta"     }
+    let(:model_as_tsv)   { "foo\tbar\tbaz" }
+    let(:valid_model)    { double('model', to_tsv: model_as_tsv) }
     
-    it 'defers to the model to serialize' do
-      valid_model.should_receive(:to_json).and_return(json_record)
-      serializer.given(valid_model).should emit(json_record)
+    it 'serializes records to JSON' do
+      processor.given(valid_record).should emit(record_as_tsv)
     end
-  end
-end
 
-describe :to_tsv, serializer: true, handles_errors: true do
-  let(:valid_record) { ["foo", 2, :a] }
-  
-  it 'serializes valid records' do
-    serializer.given(valid_record).should emit("foo\t2\ta")
-  end
+    it 'defers to models to let them serialize themselves as JSON' do
+      processor.given(valid_model).should emit(model_as_tsv)
+    end
 
-  context 'given a model' do
-    let(:tsv_record)  { "foo\tbar\tbaz"                     }
-    let(:valid_model) { double('model', to_tsv: tsv_record) }
-    
-    it 'defers to the model to serialize' do
-      valid_model.should_receive(:to_tsv).and_return(tsv_record)
-      serializer.given(valid_model).should emit(tsv_record)
+    it 'handles errors on bad records' do
+      processor { |proc| proc.should_receive(:handle_error) }.given(invalid_record).should emit(0).records
     end
   end
-end
-
-describe :from_json, serializer: true, handles_errors: true do
-  let(:valid_record) { '{"hi": "there"}' }
-  let(:bad_record)   { '{"832323:'       }
-  
-  it 'deserializes valid records' do
-    serializer.given(valid_record).should emit({'hi' => 'there'})
-  end  
 
-  context 'given a model' do
-    let(:wire_format) { { foo: 'bar' }                          }
-    let(:valid_model) { double('model', from_json: wire_format) }
+  describe :from_tsv, serializer: true, handles_errors: true do
+    it_behaves_like 'a processor', :named => :from_tsv
+    
+    let(:valid_tsv)   { "foo\t2\ta"       }
+    let(:tsv_parsed)  { ["foo", "2", "a"] }
+    let(:invalid_tsv) { nil               }
     
-    it 'defers to the model to serialize' do
-      valid_model.should_receive(:from_json).and_return(wire_format)
-      serializer.given(valid_model).should emit(wire_format)
+    it 'deserializes valid TSV' do
+      processor.given(valid_tsv).should emit(tsv_parsed)
     end
-  end
-end
 
-describe :from_tsv, serializer: true, handles_errors: true do
-  let(:valid_record) { "foo\t2\ta" }
-  
-  it 'deserializes valid records' do
-    serializer.given(valid_record).should emit(['foo', '2', 'a' ])
+    it "handles errors on invalid TSV" do
+      processor { |proc| proc.should_receive(:handle_error).with(invalid_tsv, kind_of(Exception)) }.given(invalid_tsv).should emit(0).records
+    end
   end
 
-  context 'given a model' do
-    let(:wire_format) { { foo: 'bar' }                         }
-    let(:valid_model) { double('model', from_tsv: wire_format) }
+  describe :to_inspect do
+    it_behaves_like 'a processor', :named => :to_inspect
+    
+    let(:valid_record)      { {"a" => 1 }                                   }
+    let(:record_as_inspect) { valid_record.inspect                          }
+    let(:model_as_inspect)  { '<Model #13e233>'                             }
+    let(:valid_model)       { double('model', inspect: model_as_inspect)    }
     
-    it 'defers to the model to serialize' do
-      valid_model.should_receive(:from_tsv).and_return(wire_format)
-      serializer.given(valid_model).should emit(wire_format)
+    it 'serializes records via inspect' do
+      processor.given(valid_record).should emit(record_as_inspect)
+    end
+
+    it 'defers to models to let them inspect themselves' do
+      processor.given(valid_model).should emit(model_as_inspect)
     end
   end
-end
 
-describe :to_inspect do
-  it_behaves_like 'a processor'
-end
+  describe :recordize do
+    let(:model_instance) { double('model')                                                   }
+    let(:model_klass)    { double('model_def', receive: model_instance)                      }
+    let(:serializer)     { processor(:recordize, model: model_klass, on_error: :skip)        }
+    let(:valid_record)   { { foo: 'bar' }                                                    }
+    let(:invalid_record) { [1,2,3]                                                           }
 
-describe :recordize, serializer: true, handles_errors: true do
-  let(:model_instance) { double('model')                                                   }
-  let(:model_klass)    { double('model_def', receive: model_instance)                      }
-  let(:serializer)     { create_processor(:recordize, model: model_klass, on_error: :skip) }
-  let(:valid_record)   { { foo: 'bar' }                                                    }
+    it 'recordizes valid records' do
+      processor(model: model_klass).given(valid_record).should emit(model_instance)
+    end
 
-  it 'recordizes valid records' do
-    serializer.given(valid_record).should emit(model_instance)
+    it 'handles errors on invalid models' do
+      processor(model: model_klass) { |proc| proc.should_receive(:handle_error).with(invalid_record, kind_of(Exception)) }.given(invalid_record).should emit(0).records
+    end
+    
   end
 end