ul-wukong 4.1.0

data/spec/wukong/source_spec.rb

@@ -0,0 +1,6 @@
+require 'spec_helper'
+
+describe Wukong::Source do
+  it_behaves_like 'a plugin'
+end
+

data/spec/wukong/widget/extract_spec.rb

@@ -0,0 +1,101 @@
+require 'spec_helper'
+
+describe :extract do
+
+  let(:hsh) { { "hi" => "there", "top" => { "lower" => { "lowest" => "value" } } } }
+  let(:ary) { ['1', 2, 'three'] }
+
+  subject { processor(:extract) }
+  
+  it_behaves_like 'a processor', :named => :extract
+
+  context "on a string" do
+    it "emits the string with no arguments" do
+      processor(:extract).given('hi there', 'buddy').should emit('hi there', 'buddy')
+    end
+  end
+  context "on a Fixnum" do
+    it "emits the number with no arguments" do
+      processor(:extract).given(3, 3.0).should emit(3, 3.0)
+    end
+  end
+  context "on a Hash" do
+    it "emits the hash with no arguments" do
+      processor(:extract).given(hsh).should emit(hsh)
+    end
+    it "can extract a key" do
+      processor(:extract, part: 'hi').given(hsh).should emit('there')
+    end
+    it "emits nil when the value of the key is nil" do
+      processor(:extract, part: 'bye').given(hsh).should emit(nil)
+    end
+    it "can extract a nested key" do
+      processor(:extract, part: 'top.lower.lowest').given(hsh).should emit('value')
+    end
+    it "emits nil when the value of this nested key is nil" do
+      processor(:extract, part: 'foo.bar.baz').given(hsh).should emit(nil)
+    end
+  end
+  context "on an Array" do
+    it "emits the array with no arguments" do
+      processor(:extract).given(ary).should emit(ary)
+    end
+    it "can extract the nth value with an integer argument" do
+      processor(:extract, part: 2).given(ary).should emit(2)
+    end
+    it "can extract the nth value with a string argument" do
+      processor(:extract, part: '2').given(ary).should emit(2)
+    end
+  end
+  context "on JSON" do
+    let(:garbage) { '{"239823:' }
+    it "emits the JSON with no arguments" do
+      processor(:extract).given_json(hsh).should emit_json(hsh)
+    end
+    it "will skip badly formed records" do
+      processor(:extract).given(garbage).should emit(garbage)
+    end
+    it "can extract a key" do
+      processor(:extract, part: 'hi').given_json(hsh).should emit('there')
+    end
+    it "can extract a nested key" do
+      processor(:extract, part: 'top.lower.lowest').given_json(hsh).should emit('value')
+    end
+    it "emits nil when the record is missing the key" do
+      processor(:extract, part: 'foo.bar.baz').given_json(hsh).should emit(nil)
+    end
+  end
+  context "on delimited data" do
+    it "emits the row with no arguments" do
+      processor(:extract).given_delimited('|', ary).should emit(ary.map(&:to_s).join('|'))
+    end
+    it "can extract the nth value with an integer argument" do
+      processor(:extract, part: 2, separator: '|').given_delimited('|', ary).should emit('2')
+    end
+    it "can extract nth value with a string argument" do
+      processor(:extract, part: '2', separator: '|').given_delimited('|', ary).should emit('2')
+    end
+  end
+  context "on TSV" do
+    it "emits the TSV with no arguments" do
+      processor(:extract).given_tsv(ary).should emit(ary.map(&:to_s).join("\t"))
+    end
+    it "can extract the nth value with an integer argument" do
+      processor(:extract, part: 2).given_tsv(ary).should emit('2')
+    end
+    it "can extract the nth value with a string argument" do
+      processor(:extract, part: '2').given_tsv(ary).should emit('2')
+    end
+  end
+  context "on CSV" do
+    it "emits the CSV with no arguments" do
+      processor(:extract).given_csv(ary).should emit(ary.map(&:to_s).join(","))
+    end
+    it "can extract the nth value with an integer argument" do
+      processor(:extract, part: 2, separator: ',').given_csv(ary).should emit('2')
+    end
+    it "can extract the nth value with a string argument" do
+      processor(:extract, part: '2', separator: ',').given_csv(ary).should emit('2')
+    end
+  end
+end

data/spec/wukong/widget/filters_spec.rb

@@ -0,0 +1,79 @@
+require 'spec_helper'
+
+describe "Filters" do
+
+  describe :null do
+    it_behaves_like 'a processor', :named => :null
+    it "should not pass anything, ever" do
+      processor.given('', 3, 'hi', nil).should emit(0).records
+    end
+  end
+
+  describe :identity do
+    it_behaves_like 'a processor', :named => :identity
+    it "should pass everything, always" do
+      processor.given('', 3, 'hi', nil).should emit('', 3, 'hi', nil)
+    end
+  end
+  
+  describe :regexp do
+    it_behaves_like 'a processor', :named => :regexp
+    it "should pass everything given no 'match' argument" do
+      processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
+    end
+    it "should pass everything its 'match' argument matches" do
+      processor(match: /a/).given('snap', 'crackle', 'pop').should emit('snap', 'crackle')
+    end
+  end
+
+  describe :not_regexp do
+    it_behaves_like 'a processor', :named => :not_regexp
+    it "should pass everything given no 'match' argument" do
+      processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
+    end
+    it "should pass everything its 'match' argument matches" do
+      processor(match: /a/).given('snap', 'crackle', 'pop').should emit('pop')
+    end
+  end
+
+  describe :limit do
+    it_behaves_like 'a processor', :named => :limit
+    it "should pass everything given no 'max' argument" do
+      processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
+    end
+    it "should pass only as many records as its 'max' argument" do
+      processor(max: 2).given('snap', 'crackle', 'pop', 'whoa').should emit('snap', 'crackle')
+    end
+  end
+
+  describe :sample do
+    it_behaves_like 'a processor', :named => :sample
+    it "should pass everything given no 'fraction' argument" do
+      processor.given('snap', 'crackle', 'pop').should emit('snap', 'crackle', 'pop')
+    end
+    it "should pass a fraction of records matching its 'fraction' argument" do
+      processor(:fraction => 0.5) { |proc| proc.should_receive(:rand).and_return(0.7, 0.1, 0.6) }.given('snap', 'crackle', 'pop').should emit('crackle')
+    end
+  end
+
+  describe :head do
+    it_behaves_like 'a processor', :named => :head
+    it "should pass the first 10 records given no argument" do
+      processor.given(*(1..100).to_a).should emit(10).records
+    end
+    it "should pass the first n records" do
+      processor(:n => 5).given(*(1..100).to_a).should emit(5).records
+    end
+  end
+
+  describe :tail do
+    it_behaves_like 'a processor', :named => :tail
+    it "should pass all records given no argument" do
+      processor.given(*(1..100).to_a).should emit(100).records
+    end
+    it "should skip the first n records" do
+      processor(:n => 5).given(*(1..100).to_a).should emit(95).records
+    end
+  end
+  
+end

data/spec/wukong/widget/logger_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe :logger do
+  it_behaves_like "a processor", :named => :logger
+
+  it "logs each event at the 'info' level by default" do
+    log = double("logger")
+    log.should_receive(:info).with('hi there')
+    log.should_receive(:info).with('buddy')
+    processor(:logger) do |proc|
+      proc.stub(:log).and_return(log)
+    end.given('hi there', 'buddy').should emit(0).records
+  end
+
+  it "logs each event at the a desired level set with an argument" do
+    log = double("logger")
+    log.should_receive(:debug).with('hi there')
+    log.should_receive(:debug).with('buddy')
+    processor(:logger, level: :debug) do |proc|
+      proc.stub(:log).and_return(log)
+    end.given('hi there', 'buddy').should emit(0).records
+  end
+end

data/spec/wukong/widget/operators_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe "Operators" do
+
+  describe :map do
+    it_behaves_like 'a processor', :named => :map
+    it "performs an action on each input record" do
+      processor(:map, action: ->(input_record) { input_record.upcase }).given('snap', 'crackle', 'pop').should emit('SNAP', 'CRACKLE', 'POP')
+    end
+
+    it "can simultaneously filter out records" do
+      processor(:map, compact: true, action: ->(input_record) { input_record + 1 if input_record > 0 }).given(2, -4, 6).should emit(3, 7)
+    end
+  end
+
+  describe :flatten do
+    it_behaves_like 'a processor', :named => :flatten
+
+    it "yields each input record or its contents" do
+      processor(:flatten).given('foo', ['bar', 'baz'], 'bing').should emit('foo', 'bar', 'baz', 'bing')
+    end
+    
+  end
+  
+end

data/spec/wukong/widget/reducers/bin_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+
+describe "Reducers" do
+  describe :bin do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :bin
+
+    let(:bins) {
+      [
+       ['0.0',    '2.000',  '9.000'],
+       ['2.000',  '4.000',  '9.000'],
+       ['4.000',  '6.000',  '8.000'],
+       ['6.000',  '8.000', '11.000'],
+       ['8.000', '10.000', '13.000']
+      ]
+    }
+
+    it "raises an error when called with a non-positive-definite number of bins" do
+      lambda { processor(num_bins: -1) }.should raise_error(Wukong::Error)
+    end
+
+    it "raises an error when called with a a minimum that's less than or equal to the maximum" do
+      lambda { processor(min: 10, max: 0) }.should raise_error(Wukong::Error)
+    end
+
+    it "will bin 50 numbers into 7 bins (uses the square root)" do
+      processor.given(*nums).should emit(7).records
+    end
+
+    it "will bin 50 numbers into 5 bins if asked" do
+      processor(num_bins: 10).given(*nums).should emit(10).records
+    end
+
+    it "counts correctly in each bin" do
+      processor(num_bins: 5).given(*nums).should emit(*bins)
+    end
+
+    it "can express counts logarithmically" do
+      row = processor(num_bins: 5, log_counts: true).given(*nums).output.first
+      row.size.should == 3
+      row[2].to_f.should be_within(0.1).of(2.197)
+    end
+
+    it "can add a normalized frequency" do
+      row = processor(num_bins: 5, normalize: true).given(*nums).output.first
+      row.size.should == 4
+      row[3].to_f.should be_within(0.1).of(0.18)
+    end
+
+    it "can add a normalized frequency and express counts logarithmically" do
+      row = processor(num_bins: 5, normalize: true, log_counts: true).given(*nums).output.first
+      row.size.should == 4
+      row[2].to_f.should be_within(0.1).of(2.197)
+      row[3].to_f.should be_within(0.1).of(-1.715)
+    end
+
+    it "can bin on the fly given min, max, and num_bins options" do
+      output = processor(min: -30, max: 30, num_bins: 3) do |proc|
+        # we can bin on the fly
+        proc.values.should_not_receive(:<<)
+        proc.should_not_receive(:bin!) 
+      end.given(*nums).output
+
+      output.size.should == 3
+      output.first[0].to_f.should be_within(0.1).of(-30)
+      output.last[1].to_f.should be_within(0.1).of(30)
+    end
+
+    it "can bin on the fly given fixed bin edges" do
+      output = processor(edges: [0,1,5,10]) do |proc|
+        # we can bin on the fly
+        proc.values.should_not_receive(:<<)
+        proc.should_not_receive(:bin!)
+      end.given(*nums).output
+      output.size.should == 3
+      output[0][0].to_f.should be_within(0.1).of(0.0)
+      output[0][1].to_f.should be_within(0.1).of(1.0)
+      output[1][0].to_f.should be_within(0.1).of(1.0)
+      output[1][1].to_f.should be_within(0.1).of(5.0)
+      output[2][0].to_f.should be_within(0.1).of(5.0)
+      output[2][1].to_f.should be_within(0.1).of(10.0)
+    end
+
+    it "can extract the value to bin by from an object" do
+      output = processor(by: 'data.n', min: 0).given(*json).output
+      output.size.should == 2
+      output.first[0].to_f.should be_within(0.1).of(0.0)
+      output.last[1].to_f.should be_within(0.1).of(100.0)
+    end
+
+  end
+end

data/spec/wukong/widget/reducers/count_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe "Reducers" do
+  describe :count do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :count
+    it "should emit the total count of records" do
+      processor.given(*strings).should emit(4)
+    end
+  end
+end

data/spec/wukong/widget/reducers/group_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe "Reducers" do
+  describe :group do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :group
+
+    let(:grouped_strings)        { [['apple', 2], ['banana', 1], ['cookie', 1]]  }
+    let(:grouped_nums_from_json) { [[nil, 2], [1, 1], [5, 1], [10, 1], [100, 1]] }
+    let(:grouped_nums_from_tsv)  { [[nil, 2], ['1', 1], ['5', 1], ['10', 1], ['100', 1]] }
+    it "will group single values" do
+      processor(:group).given(*strings.sort).should emit(*grouped_strings)
+    end
+    it "can group from within a JSON hash" do
+      proc = processor(:group, by: 'data.n').given(*json_sorted_n).should emit(*grouped_nums_from_json)
+    end
+    it "can group from within a TSV row" do
+      proc = processor(:group, by: '3').given(*tsv_sorted).should emit(*grouped_nums_from_tsv)
+    end
+  end
+end

data/spec/wukong/widget/reducers/join_xml_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe "Reducers" do
+  describe :join_xml do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :join_xml
+
+    it "joins XML spread out over multiple lines" do
+      processor(:join_xml).given('<xml>first line', 'second line', 'third line</xml>').should emit("<xml>first line\nsecond line\nthird line</xml>")
+    end
+    
+    it "joins XML one-per-line" do
+      processor(:join_xml).given('<xml>first line</xml>', '<xml>second line</xml>', '<xml>third line</xml>').should emit('<xml>first line</xml>', '<xml>second line</xml>', '<xml>third line</xml>')
+    end
+
+    it "joins XML split in the middle of a line" do
+      processor(:join_xml).given('<xml>first line', 'second</xml><xml> line', 'third line</xml>').should emit("<xml>first line\nsecond</xml>", "<xml> line\nthird line</xml>")
+    end
+
+    it "joins XML with a custom tag" do
+      processor(:join_xml, root: 'foobar').given('<foobar>first line', 'second line', 'third line</foobar>').should emit("<foobar>first line\nsecond line\nthird line</foobar>")
+    end
+    
+  end
+end

data/spec/wukong/widget/reducers/moments_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe "Reducers" do
+  describe :moments do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :moments
+
+    it "behaves like group when not called with any arguments" do
+      processor(:moments).given(*strings.sort).should emit(
+        {group: 'apple',  count: 2, results: {}},
+        {group: 'banana', count: 1, results: {}},
+        {group: 'cookie', count: 1, results: {}}
+      )
+    end
+
+    it "behaves calculates the moments of numeric fields" do
+      processor(:moments, group_by: 'outer', of: 'data.n').given(*json_sorted_outer).should emit(
+        {group: nil,      count: 2, results: {"data.n" => {}}},
+        {group: 'apple',  count: 2, results: {"data.n"=>{:count=>2, :mean=>3.0,   :std_dev=>2.0}}},
+        {group: 'banana', count: 1, results: {"data.n"=>{:count=>1, :mean=>100.0, :std_dev=>0.0}}},
+        {group: 'cookie', count: 1, results: {"data.n"=>{:count=>1, :mean=>10.0,  :std_dev=>0.0}}}
+      )
+    end
+
+    it "will leave off the standard deviation if desired" do
+      processor(:moments, group_by: 'outer', of: 'data.n', no_std_dev: true).given(*json_sorted_outer).should emit(
+        {group: nil,      count: 2, results: {"data.n" => {}}},
+        {group: 'apple',  count: 2, results: {"data.n"=>{:count=>2, :mean=>3.0   }}},
+        {group: 'banana', count: 1, results: {"data.n"=>{:count=>1, :mean=>100.0 }}},
+        {group: 'cookie', count: 1, results: {"data.n"=>{:count=>1, :mean=>10.0  }}}
+      )
+    end
+
+  end
+end
+

data/spec/wukong/widget/reducers/sort_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe "Reducers" do
+  describe :sort do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :sort
+    it "will use ascending order by default" do
+      processor(:sort).given(*strings).should emit(*strings.sort)
+    end
+    it "can sort in reversed (descending) order" do
+      processor(:sort, reverse: true).given(*strings).should emit(*strings.sort.reverse)
+    end
+    it "will use lexical order by default" do
+      processor(:sort).given(*nums).should emit(*nums.sort)
+    end
+    it "can sort in numerical order" do
+      processor(:sort, numeric: true).given(*nums).should emit(*nums.map(&:to_i).sort.map(&:to_s))
+    end
+    it "can sort from within a JSON hash" do
+      proc = processor(:sort, numeric: true, on: 'data.n').given(*json).should emit(*json_sorted_n)
+    end
+    it "can sort from within a TSV row" do
+      proc = processor(:sort, numeric: true, on: '3').given(*tsv).should emit(*tsv_sorted)
+    end
+  end
+end