wukong 3.0.0.pre → 3.0.0.pre2

data/spec/wukong/runner_spec.rb

@@ -1,12 +1,12 @@
 require 'spec_helper'
-require 'wukong'
+# require 'wukong'
 
-describe :runner, :helpers => true do
+# describe :runner, :helpers => true do
 
-  context 'tiny_count example script' do
-    it 'is shorter than a tweet' do
-      example_script_contents('tiny_count.rb').length.should < 140
-    end
+#   context 'tiny_count example script' do
+#     it 'is shorter than a tweet' do
+#       example_script_contents('tiny_count.rb').length.should < 140
+#     end
 
-  end
-end
+#   end
+# end

data/spec/wukong/widget/filters_spec.rb

@@ -0,0 +1,61 @@
+require 'spec_helper'
+
+describe "Filters" do
+
+  context :null do
+    it_behaves_like 'a processor', :named => :null
+    it "should not pass anything, ever" do
+      processor.given('', 3, 'hi', nil).should emit(0).records
+    end
+  end
+
+  context :identity do
+    it_behaves_like 'a processor', :named => :identity
+    it "should pass everything, always" do
+      processor.given('', 3, 'hi', nil).should emit('', 3, 'hi', nil)
+    end
+  end
+  
+  context :regexp do
+    it_behaves_like 'a processor', :named => :regexp
+    it "should pass everything given no 'match' argument" do
+      processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
+    end
+    it "should pass everything its 'match' argument matches" do
+      processor(match: /a/).given('snap', 'crackle', 'pop').should emit('snap', 'crackle')
+    end
+  end
+
+  context :not_regexp do
+    it_behaves_like 'a processor', :named => :not_regexp
+    it "should pass everything given no 'match' argument" do
+      processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
+    end
+    it "should pass everything its 'match' argument matches" do
+      processor(match: /a/).given('snap', 'crackle', 'pop').should emit('pop')
+    end
+  end
+
+  context :limit do
+    it_behaves_like 'a processor', :named => :limit
+    it "should pass everything given no 'max' argument" do
+      processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
+    end
+    it "should pass only as many records as its 'max' argument" do
+      processor(max: 2).given('snap', 'crackle', 'pop', 'whoa').should emit('snap', 'crackle')
+    end
+  end
+
+  context :sample do
+    it_behaves_like 'a processor', :named => :sample
+    it "should pass everything given no 'fraction' argument" do
+      processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
+    end
+    it "should pass everything given no 'fraction' argument" do
+      processor(:fraction => 0.5).tap do |proc|
+        proc.should_receive(:rand).and_return(0.7, 0.1, 0.6)
+      end.given('snap', 'crackle', 'pop').should emit('crackle')
+    end
+  end
+
+end

data/spec/wukong/widget/processors_spec.rb

@@ -0,0 +1,126 @@
+require 'spec_helper'
+
+describe Wukong::Processor do
+
+  let(:hsh) { { "hi" => "there", "top" => { "lower" => { "lowest" => "value" } } } }
+  let(:ary) { ['1', 2, 'three'] }
+
+  context :logger do
+    it_behaves_like "a processor", :named => :logger
+
+    it "logs each event at the 'info' level by default" do
+      log = mock("logger")
+      log.should_receive(:info).with('hi there')
+      log.should_receive(:info).with('buddy')
+      processor(:logger) do
+        stub!(:log).and_return(log)
+      end.given('hi there', 'buddy').should emit(0).records
+    end
+
+    it "logs each event at the a desired level set with an argument" do
+      log = mock("logger")
+      log.should_receive(:debug).with('hi there')
+      log.should_receive(:debug).with('buddy')
+      processor(:logger, level: :debug) do
+        stub!(:log).and_return(log)
+      end.given('hi there', 'buddy').should emit(0).records
+    end
+  end
+  
+  
+  context :extract do
+    subject { processor(:extract) }
+    
+    it_behaves_like 'a processor', :named => :extract
+
+    context "on a string" do
+      it "emits the string with no arguments" do
+        processor(:extract).given('hi there', 'buddy').should emit('hi there', 'buddy')
+      end
+    end
+    context "on a Fixnum" do
+      it "emits the number with no arguments" do
+        processor(:extract).given(3, 3.0).should emit(3, 3.0)
+      end
+    end
+    context "on a Hash" do
+      it "emits the hash with no arguments" do
+        processor(:extract).given(hsh).should emit(hsh)
+      end
+      it "can extract a key" do
+        processor(:extract, part: 'hi').given(hsh).should emit('there')
+      end
+      it "emits nil when the value of the key is nil" do
+        processor(:extract, part: 'bye').given(hsh).should emit(nil)
+      end
+      it "can extract a nested key" do
+        processor(:extract, part: 'top.lower.lowest').given(hsh).should emit('value')
+      end
+      it "emits nil when the value of this nested key is nil" do
+        processor(:extract, part: 'foo.bar.baz').given(hsh).should emit(nil)
+      end
+    end
+    context "on an Array" do
+      it "emits the array with no arguments" do
+        processor(:extract).given(ary).should emit(ary)
+      end
+      it "can extract the nth value with an integer argument" do
+        processor(:extract, part: 2).given(ary).should emit(2)
+      end
+      it "can extract the nth value with a string argument" do
+        processor(:extract, part: '2').given(ary).should emit(2)
+      end
+    end
+    context "on JSON" do
+      let(:garbage) { '{"239823:' }
+      it "emits the JSON with no arguments" do
+        processor(:extract).given_json(hsh).should emit_json(hsh)
+      end
+      it "will skip badly formed records" do
+        processor(:extract).given(garbage).should emit(garbage)
+      end
+      it "can extract a key" do
+        processor(:extract, part: 'hi').given_json(hsh).should emit('there')
+      end
+      it "can extract a nested key" do
+        processor(:extract, part: 'top.lower.lowest').given_json(hsh).should emit('value')
+      end
+      it "emits nil when the record is missing the key" do
+        processor(:extract, part: 'foo.bar.baz').given_json(hsh).should emit(nil)
+      end
+    end
+    context "on delimited data" do
+      it "emits the row with no arguments" do
+        processor(:extract).given_delimited('|', ary).should emit(ary.map(&:to_s).join('|'))
+      end
+      it "can extract the nth value with an integer argument" do
+        processor(:extract, part: 2, separator: '|').given_delimited('|', ary).should emit('2')
+      end
+      it "can extract nth value with a string argument" do
+        processor(:extract, part: '2', separator: '|').given_delimited('|', ary).should emit('2')
+      end
+    end
+    context "on TSV" do
+      it "emits the TSV with no arguments" do
+        processor(:extract).given_tsv(ary).should emit(ary.map(&:to_s).join("\t"))
+      end
+      it "can extract the nth value with an integer argument" do
+        processor(:extract, part: 2).given_tsv(ary).should emit('2')
+      end
+      it "can extract the nth value with a string argument" do
+        processor(:extract, part: '2').given_tsv(ary).should emit('2')
+      end
+    end
+    context "on CSV" do
+      it "emits the CSV with no arguments" do
+        processor(:extract).given_csv(ary).should emit(ary.map(&:to_s).join(","))
+      end
+      it "can extract the nth value with an integer argument" do
+        processor(:extract, part: 2, separator: ',').given_csv(ary).should emit('2')
+      end
+      it "can extract the nth value with a string argument" do
+        processor(:extract, part: '2', separator: ',').given_csv(ary).should emit('2')
+      end
+    end
+  end
+end

data/spec/wukong/widget/reducers/bin_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+
+describe Wukong::Processor do
+  describe :bin do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :bin
+
+    let(:bins) {
+      [
+       ['0.0',    '2.000',  '9.000'],
+       ['2.000',  '4.000',  '9.000'],
+       ['4.000',  '6.000',  '8.000'],
+       ['6.000',  '8.000', '11.000'],
+       ['8.000', '10.000', '13.000']
+      ]
+    }
+
+    it "raises an error when called with a non-positive-definite number of bins" do
+      lambda { processor(num_bins: -1) }.should raise_error(Wukong::Error)
+    end
+
+    it "raises an error when called with a a minimum that's less than or equal to the maximum" do
+      lambda { processor(min: 10, max: 0) }.should raise_error(Wukong::Error)
+    end
+
+    it "will bin 50 numbers into 7 bins (uses the square root)" do
+      processor.given(*nums).should emit(7).records
+    end
+
+    it "will bin 50 numbers into 5 bins if asked" do
+      processor(num_bins: 10).given(*nums).should emit(10).records
+    end
+
+    it "counts correctly in each bin" do
+      processor(num_bins: 5).given(*nums).should emit_tsv(*bins)
+    end
+
+    it "can express counts logarithmically" do
+      row = processor(num_bins: 5, log_counts: true).given(*nums).tsv_output.first
+      row.size.should == 3
+      row[2].to_f.should be_within(0.1).of(2.197)
+    end
+
+    it "can add a normalized frequency" do
+      row = processor(num_bins: 5, normalize: true).given(*nums).tsv_output.first
+      row.size.should == 4
+      row[3].to_f.should be_within(0.1).of(0.18)
+    end
+
+    it "can add a normalized frequency and express counts logarithmically" do
+      row = processor(num_bins: 5, normalize: true, log_counts: true).given(*nums).tsv_output.first
+      row.size.should == 4
+      row[2].to_f.should be_within(0.1).of(2.197)
+      row[3].to_f.should be_within(0.1).of(-1.715)
+    end
+
+    it "can bin on the fly given min, max, and num_bins options" do
+      output = processor(min: -30, max: 30, num_bins: 3) do
+        # we can bin on the fly
+        values.should_not_receive(:<<)
+        should_not_receive(:bin!) 
+      end.given(*nums).tsv_output
+
+      output.size.should == 3
+      output.first[0].to_f.should be_within(0.1).of(-30)
+      output.last[1].to_f.should be_within(0.1).of(30)
+    end
+
+    it "can bin on the fly given fixed bin edges" do
+      output = processor(edges: [0,1,5,10]) do
+        # we can bin on the fly
+        values.should_not_receive(:<<)
+        should_not_receive(:bin!)
+      end.given(*nums).tsv_output
+      output.size.should == 3
+      output[0][0].to_f.should be_within(0.1).of(0.0)
+      output[0][1].to_f.should be_within(0.1).of(1.0)
+      output[1][0].to_f.should be_within(0.1).of(1.0)
+      output[1][1].to_f.should be_within(0.1).of(5.0)
+      output[2][0].to_f.should be_within(0.1).of(5.0)
+      output[2][1].to_f.should be_within(0.1).of(10.0)
+    end
+
+    it "can extract the value to bin by from an object" do
+      output = processor(by: 'data.n', min: 0).given(*json).tsv_output
+      output.size.should == 2
+      output.first[0].to_f.should be_within(0.1).of(0.0)
+      output.last[1].to_f.should be_within(0.1).of(100.0)
+    end
+
+  end
+end

data/spec/wukong/widget/reducers/count_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe Wukong::Processor do
+  describe :count do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :count
+    it "should emit the total count of records" do
+      processor.given(*strings).should emit(4)
+    end
+  end
+end

data/spec/wukong/widget/reducers/group_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Wukong::Processor do
+  describe :group do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :group
+
+    let(:grouped_strings) { [['apple', '2'], ['banana', '1'], ['cookie', '1']]             }
+    let(:grouped_nums)    { [['', '2'], ['1', '1'], ['5', '1'], ['10', '1'], ['100', '1']] }
+    it "will group single values" do
+      processor(:group).given(*strings.sort).should emit_tsv(*grouped_strings)
+    end
+    it "can group from within a JSON hash" do
+      proc = processor(:group, by: 'data.n').given(*json_sorted_n).should emit_tsv(*grouped_nums)
+    end
+    it "can group from within a TSV row" do
+      proc = processor(:group, by: '3').given(*tsv_sorted).should emit_tsv(*grouped_nums)
+    end
+  end
+end

data/spec/wukong/widget/reducers/moments_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Wukong::Processor do
+  describe :moments do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :moments
+
+    it "behaves like group when not called with any arguments" do
+      processor(:moments).given(*strings.sort).should emit(
+        {group: 'apple',  count: 2, results: {}},
+        {group: 'banana', count: 1, results: {}},
+        {group: 'cookie', count: 1, results: {}}
+      )
+    end
+
+    it "behaves calculates the moments of numeric fields" do
+      processor(:moments, group_by: 'outer', of: 'data.n').given(*json_sorted_outer).should emit(
+        {group: nil,      count: 2, results: {"data.n" => {}}},
+        {group: 'apple',  count: 2, results: {"data.n"=>{:count=>2, :mean=>3.0,   :std_dev=>2.0}}},
+        {group: 'banana', count: 1, results: {"data.n"=>{:count=>1, :mean=>100.0, :std_dev=>0.0}}},
+        {group: 'cookie', count: 1, results: {"data.n"=>{:count=>1, :mean=>10.0,  :std_dev=>0.0}}}
+      )
+    end
+
+    it "will leave off the standard deviation if desired" do
+      processor(:moments, group_by: 'outer', of: 'data.n', std_dev: false).given(*json_sorted_outer).should emit(
+        {group: nil,      count: 2, results: {"data.n" => {}}},
+        {group: 'apple',  count: 2, results: {"data.n"=>{:count=>2, :mean=>3.0   }}},
+        {group: 'banana', count: 1, results: {"data.n"=>{:count=>1, :mean=>100.0 }}},
+        {group: 'cookie', count: 1, results: {"data.n"=>{:count=>1, :mean=>10.0  }}}
+      )
+    end
+
+  end
+end
+

data/spec/wukong/widget/reducers/sort_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Wukong::Processor do
+  describe :sort do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :sort
+    it "will use ascending order by default" do
+      processor(:sort).given(*strings).should emit(*strings.sort)
+    end
+    it "can sort in reversed (descending) order" do
+      processor(:sort, reverse: true).given(*strings).should emit(*strings.sort.reverse)
+    end
+    it "will use lexical order by default" do
+      processor(:sort).given(*nums).should emit(*nums.sort)
+    end
+    it "can sort in numerical order" do
+      processor(:sort, numeric: true).given(*nums).should emit(*nums.map(&:to_i).sort.map(&:to_s))
+    end
+    it "can sort from within a JSON hash" do
+      proc = processor(:sort, numeric: true, on: 'data.n').given(*json).should emit(*json_sorted_n)
+    end
+    it "can sort from within a TSV row" do
+      proc = processor(:sort, numeric: true, on: '3').given(*tsv).should emit(*tsv_sorted)
+    end
+  end
+end

data/spec/wukong/widget/serializers_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+
+describe "Serializing" do
+
+  context :to_json do
+    
+    let(:emittable)     { {"hi" => "there"} }
+    let(:not_emittable) { {"n" => Float::INFINITY} }
+
+    it_behaves_like 'a processor', :named => :to_json
+    
+    it "should handle valid records" do
+      processor.given(emittable).should emit_json(emittable)
+    end
+
+    it "should skip bad records" do
+      processor.given(not_emittable).should emit(0).records
+    end
+    
+  end
+
+  context :to_tsv do
+    let(:emittable)     { ["foo", 2, :a] }
+    let(:not_emittable) { nil            }
+
+    it_behaves_like 'a processor', :named => :to_tsv
+    
+    it "should handle valid records" do
+      processor.given(emittable).should emit_tsv(emittable.map(&:to_s))
+    end
+    
+    it "should skip bad records" do
+      processor.given(not_emittable).should emit(0).records
+    end
+  end
+end
+
+describe "Deserializing" do
+
+  context :from_json do
+    let(:parseable)     { '{"hi": "there"}' }
+    let(:not_parseable) { '{"832323:'       }
+
+    it_behaves_like 'a processor', :named => :from_json
+
+    it "should handle valid records" do
+      processor.given(parseable).should emit({'hi' => 'there'})
+    end
+    
+    it "should skip bad records" do
+      processor.given(not_parseable).should emit(0).records
+    end
+  end
+  
+  context :from_tsv do
+
+    let(:parseable)     { "foo\t2\ta"    }
+    let(:not_parseable) { nil            }
+
+    it_behaves_like 'a processor', :named => :from_tsv
+    
+    it "should handle valid records" do
+      processor.given(parseable).should emit(parseable.split("\t"))
+    end
+    
+    it "should skip bad records" do
+      processor.given(not_parseable).should emit(0).records
+    end
+  end
+end
+
+describe "Pretty printing" do
+
+  context "JSON" do
+    let(:parseable)     { '{"hi": "there"}' }
+    let(:not_parseable) { '{"832323:'       }
+
+    it_behaves_like 'a processor', :named => :pretty
+
+    it "should prettify parseable records" do
+      processor(:pretty).given(parseable).should emit_json({'hi' => 'there'})
+    end
+
+    it "should pass on non parseable records" do
+      processor(:pretty).given(not_parseable).should emit(not_parseable)
+    end
+  end
+
+  it "should pass on everything else" do
+    processor(:pretty).given('foobar').should emit('foobar')
+  end
+end