ul-wukong 4.1.0

data/spec/wukong/widget/reducers/uniq_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe "Reducers" do
+  describe :uniq do
+    include_context "reducers"
+    it_behaves_like 'a processor', :named => :uniq
+    it "should remove duplicate records" do
+      processor.given(*strings.sort).should emit(*strings.sort.uniq)
+    end
+    it "should output nothing if given no records" do
+      processor.given().should emit()
+    end
+  end
+end

data/spec/wukong/widget/serializers_spec.rb

@@ -0,0 +1,114 @@
+require 'spec_helper'
+
+describe "Serializers" do
+
+  describe :to_json do
+    it_behaves_like 'a processor', :named => :to_json
+    
+    let(:valid_record)    { { hi: 'there' }                          }
+    let(:record_as_json)  { '{"hi":"there"}'                         }
+    let(:model_as_json)   { '{"model":"json"}'                       }
+    let(:valid_model)     { double('model', to_json: model_as_json)  }
+    
+    it 'serializes records to JSON' do
+      processor.given(valid_record).should emit(record_as_json)
+    end
+    
+    it 'serializes records as pretty JSON when asked' do
+      processor(:pretty => true).given(valid_record).output.first.should include("\n")
+    end
+
+    it 'defers to models to let them serialize themselves as JSON' do
+      processor.given(valid_model).should emit(model_as_json)
+    end
+  end
+
+
+  describe :from_json do
+    it_behaves_like 'a processor', :named => :from_json
+    
+    let(:valid_json)   { '{"hi": "there"}'                        }
+    let(:json_parsed)  { {"hi" => "there"}                        }
+    let(:invalid_json) { '{"832323:'                              }
+    
+    it 'deserializes valid JSON' do
+      processor.given(valid_json).should emit(json_parsed)
+    end
+    
+    it 'handles errors on invalid JSON' do
+      processor { |proc| proc.should_receive(:handle_error).with(invalid_json, kind_of(Exception)) }.given(invalid_json).should emit(0).records
+    end
+  end
+  
+  describe :to_tsv do
+    it_behaves_like 'a processor', :named => :to_tsv
+    
+    let(:valid_record)   { ["foo", 2, :a]  }
+    let(:invalid_record) { nil             }
+    let(:record_as_tsv)  { "foo\t2\ta"     }
+    let(:model_as_tsv)   { "foo\tbar\tbaz" }
+    let(:valid_model)    { double('model', to_tsv: model_as_tsv) }
+    
+    it 'serializes records to JSON' do
+      processor.given(valid_record).should emit(record_as_tsv)
+    end
+
+    it 'defers to models to let them serialize themselves as JSON' do
+      processor.given(valid_model).should emit(model_as_tsv)
+    end
+
+    it 'handles errors on bad records' do
+      processor { |proc| proc.should_receive(:handle_error) }.given(invalid_record).should emit(0).records
+    end
+  end
+
+  describe :from_tsv, serializer: true, handles_errors: true do
+    it_behaves_like 'a processor', :named => :from_tsv
+    
+    let(:valid_tsv)   { "foo\t2\ta"       }
+    let(:tsv_parsed)  { ["foo", "2", "a"] }
+    let(:invalid_tsv) { nil               }
+    
+    it 'deserializes valid TSV' do
+      processor.given(valid_tsv).should emit(tsv_parsed)
+    end
+
+    it "handles errors on invalid TSV" do
+      processor { |proc| proc.should_receive(:handle_error).with(invalid_tsv, kind_of(Exception)) }.given(invalid_tsv).should emit(0).records
+    end
+  end
+
+  describe :to_inspect do
+    it_behaves_like 'a processor', :named => :to_inspect
+    
+    let(:valid_record)      { {"a" => 1 }                                   }
+    let(:record_as_inspect) { valid_record.inspect                          }
+    let(:model_as_inspect)  { '<Model #13e233>'                             }
+    let(:valid_model)       { double('model', inspect: model_as_inspect)    }
+    
+    it 'serializes records via inspect' do
+      processor.given(valid_record).should emit(record_as_inspect)
+    end
+
+    it 'defers to models to let them inspect themselves' do
+      processor.given(valid_model).should emit(model_as_inspect)
+    end
+  end
+
+  describe :recordize do
+    let(:model_instance) { double('model')                                                   }
+    let(:model_klass)    { double('model_def', receive: model_instance)                      }
+    let(:serializer)     { processor(:recordize, model: model_klass, on_error: :skip)        }
+    let(:valid_record)   { { foo: 'bar' }                                                    }
+    let(:invalid_record) { [1,2,3]                                                           }
+
+    it 'recordizes valid records' do
+      processor(model: model_klass).given(valid_record).should emit(model_instance)
+    end
+
+    it 'handles errors on invalid models' do
+      processor(model: model_klass) { |proc| proc.should_receive(:handle_error).with(invalid_record, kind_of(Exception)) }.given(invalid_record).should emit(0).records
+    end
+    
+  end
+end

data/spec/wukong/widget/sink_spec.rb

@@ -0,0 +1,19 @@
+# require 'spec_helper'
+
+# describe :sinks, :helpers => true do
+
+#   describe Wukong::Sink::Stdout do
+#     it 'dumps records to $stdout' do
+#       $stdout.should_receive(:puts).with(mock_record)
+#       subject.process(mock_record)
+#     end
+#   end
+
+#   describe Wukong::Sink::Stderr do
+#     it 'dumps records to $stderr' do
+#       $stderr.should_receive(:puts).with(mock_record)
+#       subject.process(mock_record)
+#     end
+
+#   end
+# end

data/spec/wukong/widget/source_spec.rb

@@ -0,0 +1,65 @@
+# require 'spec_helper'
+# require 'wukong/widget/gibberish'
+
+# describe :sources, :helpers => true do
+
+#   describe Wukong::Source::Iter do
+#     subject{ described_class.new(obj: (9 .. 14), owner: test_dataflow) }
+#     it 'iterates over a given collection' do
+#       subject.to_enum.to_a.should == [9, 10, 11, 12, 13, 14]
+#     end
+#     context 'dataflow method' do
+#       it 'simplified args' do
+#         test_dataflow.iter(9 .. 14).should == subject
+#       end
+#     end
+#   end
+
+#   # describe Wukong::Source::FileSource do
+#   #   let(:example_filename){ Pathname.path_to(:data, 'text/jabberwocky.txt') }
+#   #   subject{ described_class.receive(filename: example_filename, owner: test_dataflow) }
+#   #   before{ subject.setup }
+#   #   it 'iterates over a given collection' do
+#   #     subject.to_enum.to_a[6, 4].should == ["`Twas brillig, and the slithy toves", "  Did gyre and gimble in the wabe:", "All mimsy were the borogoves,", "  And the mome raths outgrabe.",]
+#   #   end
+#   #   context 'dataflow method' do
+#   #     it 'simplified args' do
+#   #       test_dataflow.file_source(example_filename).should == subject
+#   #     end
+#   #   end
+#   # end
+
+#   describe Wukong::Source::Integers do
+#     subject{ described_class.receive(qty: 10, owner: test_dataflow) }
+#     before{ subject.setup }
+
+#     it 'generates integers up to the given limit' do
+#       subject.to_enum.to_a.should == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+#     end
+#     it 'generates nothing if the initial range is void' do
+#       subject.qty = 0
+#       subject.to_enum.to_a.should == []
+#     end
+#     it 'generates one thing if the min and max are equal' do
+#       subject.qty = 1
+#       subject.to_enum.to_a.should == [0]
+#     end
+
+#     context 'dataflow method' do
+#       it 'takes simplified args' do
+#         test_dataflow.integers(10).should == subject
+#       end
+#     end
+#   end
+
+#   describe Wukong::Widget::Gibberish do
+#     subject{ described_class.receive(:qty => 4) }
+#     before{ subject.setup }
+
+#     it 'generates integers up to the given limit' do
+#       subject.rng = Random.new(8675309)
+#       subject.to_enum.to_a.should == ["loaiaeiaeo neidgfo heeume sptfmeec naet sttptlm waaaioh detov elrrltv nii ulcsnn", "set ensr poeleaa seqi tmnreoee boooral oczncgp deaia rykcoao leo rim mmibpbfii", "artrru sto quuu doo peoehrile nto esl tia gaili tuiooey lkedotp sts kaiy weeeia", "crhi tyiiae mieubmbooa teeae roi ednz taieh zaloy syhe ret kuoa deeeo xittipl mo"]
+#     end
+
+#   end
+# end

data/spec/wukong/wu-local_spec.rb

@@ -0,0 +1,109 @@
+require 'spec_helper'
+
+describe 'wu-local' do
+
+  let(:input) { %w[1 2 3] }
+  
+  context "without any arguments" do
+    subject { wu_local() }
+    it {should exit_with(:non_zero) }
+    it "displays help on STDERR" do
+      should have_stderr(/provide a processor.*to run/)
+    end
+  end
+
+  context "running outside any Ruby project" do
+    subject { wu_local('count').in(examples_dir('empty')) < input }
+    it { should exit_with(0) }
+    it "runs the processor" do
+      should have_stdout("3")
+    end
+    context "when passed a BUNDLE_GEMFILE" do
+      context "that doesn't belong to a deploy pack" do
+        subject { wu_local('count').in(examples_dir('empty')).using(integration_env.merge("BUNDLE_GEMFILE" => examples_dir('ruby_project', 'Gemfile').to_s)) < input }
+        it { should exit_with(0) }
+        it "runs the processor" do
+          should have_stdout("3")
+        end
+      end
+      context "that belongs to a deploy pack" do
+        subject { wu_local('count').in(examples_dir('empty')).using(integration_env.merge("BUNDLE_GEMFILE" => examples_dir('deploy_pack', 'Gemfile').to_s)) < input }
+        it { should exit_with(0) }
+        it "runs the processor" do
+          should have_stdout("3")
+        end
+        context "loading the deploy pack" do
+          subject { wu_local('string_reverser').in(examples_dir('empty')).using(integration_env.merge("BUNDLE_GEMFILE" => examples_dir('deploy_pack', 'Gemfile').to_s)) < 'hi' }
+          it { should exit_with(0) }
+          it "runs the processor" do
+            should have_stdout("ih")
+          end
+        end
+      end
+    end
+  end
+
+  context "running within a Ruby project" do
+    context "at its root" do
+      subject { wu_local('count').in(examples_dir('ruby_project')) < input }
+      it { should exit_with(0) }
+      it "runs the processor" do
+        should have_stdout("3")
+      end
+    end
+    context "deep within it" do
+      subject { wu_local('count').in(examples_dir('ruby_project')) < input }
+      it { should exit_with(0) }
+      it "runs the processor" do
+        should have_stdout("3")
+      end
+    end
+  end
+
+  context "running within a deploy pack" do
+    context "at its root" do
+      subject { wu_local('count').in(examples_dir('deploy_pack')) < input }
+      it { should exit_with(0) }
+      it "runs the processor" do
+        should have_stdout("3")
+      end
+      context "loading the deploy pack" do
+        subject { wu_local('string_reverser').in(examples_dir('deploy_pack')) < 'hi' }
+        it { should exit_with(0) }
+        it "runs the processor" do
+          should have_stdout("ih")
+        end
+      end
+    end
+    context "deep within it" do
+      subject { wu_local('count').in(examples_dir('deploy_pack')) < input }
+      it { should exit_with(0) }
+      it "runs the processor" do
+        should have_stdout("3")
+      end
+      context "loading the deploy pack" do
+        subject { wu_local('string_reverser').in(examples_dir('deploy_pack')) < 'hi' }
+        it { should exit_with(0) }
+        it "runs the processor" do
+          should have_stdout("ih")
+        end
+      end
+    end
+  end
+
+  context "parsing records with the --from argument" do
+    subject { wu_local('identity --from=tsv') < "1\t2\t3" }
+    it { should exit_with(0) }
+    it "should emit a row for each input column" do
+      should have_stdout("1\n2\n3")
+    end
+  end
+
+  context "formatting records with the --to argument" do
+    subject { wu_local('identity --from=tsv --to=json') < "1\t2\t3" }
+    it { should exit_with(0) }
+    it "should emit a single JSON array for the input record" do
+      should have_stdout('["1","2","3"]')
+    end
+  end
+end

data/spec/wukong/wu-source_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe 'wu-source' do
+
+  let(:input) { %w[1 2 3] }
+  
+  context "without any arguments" do
+    subject { wu_source() }
+    it {should exit_with(:non_zero) }
+    it "displays help on STDERR" do
+      should have_stderr(/provide a processor.*to run/)
+    end
+  end
+
+  # FIXME -- it's hard to write an integration test for wu-source
+  # because it doesn't self-terminate under any conditions when run
+  # successfully.
+  #
+  # Options:
+  #
+  #   1) Add a --max (or similar) flag to wu-source allowing it to
+  #      exit after some number of records which could then be checked
+  #      by an integration test.
+  #
+  #   2) Launch it in a subprocess and wait a little while (how long?)
+  #   and ensure that it's produced a bunch of output in the meantime.
+  #   If the `per_sec` option is high, we shouldn't have to wait very
+  #   long to see a whole bunch of output records.  This is tricky b/c
+  #   what if the system is under load and we don't wait long enough
+  #   for the wu-source subprocess to boot up and start emitting?
+  
+end

data/spec/wukong/wu_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe 'wu' do
+
+  let(:input) { %w[1 2 3] }
+  
+  context "without any arguments" do
+    subject { wu() }
+    it {should exit_with(:non_zero) }
+    it "displays help on STDERR" do
+      should have_stderr(/provide a Wukong command to run/)
+    end
+  end
+end

data/spec/wukong/wukong_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe Wukong do
+
+  it_behaves_like Hanuman::Shortcuts
+
+  it{ should respond_to(:processor) }
+  it{ should respond_to(:dataflow)  }
+
+end

data/wukong.gemspec

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/wukong/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = 'ul-wukong'
+  gem.homepage    = 'https://github.com/undeadlabs/wukong'
+  gem.licenses    = ["Apache 2.0"]
+  gem.email       = 'coders@infochimps.com'
+  gem.authors     = ['Infochimps', 'Philip (flip) Kromer', 'Travis Dempsey']
+  gem.version     = Wukong::VERSION
+
+  gem.summary     = 'Fork of Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use it, yet handles terabyte-scale computation with ease.'
+  gem.description = <<-EOF
+  Treat your dataset like a:
+
+      * stream of lines when it's efficient to process by lines
+      * stream of field arrays when it's efficient to deal directly with fields
+      * stream of lightweight objects when it's efficient to deal with objects
+
+  Wukong is friends with Hadoop the elephant, Pig the query language, and the cat on your command line.
+EOF
+
+  gem.files         = `git ls-files`.split("\n").reject { |path| path =~ /^(data|docpages|notes|old)/ }
+  gem.executables   = ['wu-local', 'wu-source', 'wu']
+  gem.test_files    = gem.files.grep(/^spec/)
+  gem.require_paths = ['lib']
+
+  gem.add_dependency('configliere',             '>= 0.4.18')
+  gem.add_dependency('multi_json',              '>= 1.3.6')
+  gem.add_dependency('gorillib',                '>= 0.4.2')
+  gem.add_dependency('forgery')
+  gem.add_dependency('uuidtools')
+  gem.add_dependency('eventmachine')
+  gem.add_dependency('log4r')
+end

metadata

@@ -0,0 +1,465 @@
+--- !ruby/object:Gem::Specification
+name: ul-wukong
+version: !ruby/object:Gem::Version
+  version: 4.1.0
+platform: ruby
+authors:
+- Infochimps
+- Philip (flip) Kromer
+- Travis Dempsey
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: configliere
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.18
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.18
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.6
+- !ruby/object:Gem::Dependency
+  name: gorillib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+- !ruby/object:Gem::Dependency
+  name: forgery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: log4r
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! "  Treat your dataset like a:\n\n      * stream of lines when it's
+  efficient to process by lines\n      * stream of field arrays when it's efficient
+  to deal directly with fields\n      * stream of lightweight objects when it's efficient
+  to deal with objects\n\n  Wukong is friends with Hadoop the elephant, Pig the query
+  language, and the cat on your command line.\n"
+email: coders@infochimps.com
+executables:
+- wu-local
+- wu-source
+- wu
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .gitmodules
+- .rspec
+- .travis.yml
+- .yardopts
+- CHANGELOG.md
+- Gemfile
+- Guardfile
+- LICENSE.md
+- NOTES-travis.md
+- README-old.md
+- README.md
+- Rakefile
+- TODO.md
+- bin/cutc
+- bin/cuttab
+- bin/greptrue
+- bin/md5sort
+- bin/setcat
+- bin/tabchar
+- bin/uniq-ord
+- bin/uniqc
+- bin/wu
+- bin/wu-clean-encoding
+- bin/wu-date
+- bin/wu-datetime
+- bin/wu-hist
+- bin/wu-lign
+- bin/wu-local
+- bin/wu-plus
+- bin/wu-source
+- bin/wu-sum
+- diagrams/wu_local.dot
+- diagrams/wu_local.dot.png
+- examples/Gemfile
+- examples/README.md
+- examples/basic/string_reverser.rb
+- examples/basic/tiny_count.rb
+- examples/basic/word_count/accumulator.rb
+- examples/basic/word_count/tokenizer.rb
+- examples/basic/word_count/word_count.rb
+- examples/dataflow/scraper_macro_flow.rb
+- examples/deploy_pack/Gemfile
+- examples/deploy_pack/README.md
+- examples/deploy_pack/a/b/c/.gitkeep
+- examples/deploy_pack/app/processors/string_reverser.rb
+- examples/deploy_pack/config/environment.rb
+- examples/dsl/dataflow/fibonacci_series.rb
+- examples/dsl/dataflow/scraper_macro_flow.rb
+- examples/dsl/dataflow/simple.rb
+- examples/dsl/dataflow/telegram.rb
+- examples/dsl/workflow/cherry_pie.dot
+- examples/dsl/workflow/cherry_pie.md
+- examples/dsl/workflow/cherry_pie.png
+- examples/dsl/workflow/cherry_pie.rb
+- examples/empty/.gitkeep
+- examples/examples_helper.rb
+- examples/geo.rb
+- examples/geo/geo_grids.numbers
+- examples/geo/geolocated.rb
+- examples/geo/quadtile.rb
+- examples/geo/spec/geolocated_spec.rb
+- examples/geo/tile_fetcher.rb
+- examples/graph/implied_geolocation/README.md
+- examples/graph/minimum_spanning_tree/airfares_graphviz.rb
+- examples/improver/tweet_summary.rb
+- examples/loadable.rb
+- examples/munging/airline_flights/airline_flights.rake
+- examples/munging/airline_flights/airplane.rb
+- examples/munging/airline_flights/airport_id_unification.rb
+- examples/munging/airline_flights/airport_ok_chars.rb
+- examples/munging/airline_flights/indexable.rb
+- examples/munging/airline_flights/indexable_spec.rb
+- examples/munging/airline_flights/reconcile_airports.rb
+- examples/munging/airline_flights/tasks.rake
+- examples/munging/airline_flights/topcities.rb
+- examples/munging/geo/geo_json.rb
+- examples/munging/geo/geo_models.rb
+- examples/munging/geo/geonames_models.rb
+- examples/munging/geo/iso_codes.rb
+- examples/munging/geo/reconcile_countries.rb
+- examples/munging/geo/tasks.rake
+- examples/munging/wikipedia/articles/extract_articles-parsed.rb
+- examples/munging/wikipedia/articles/extract_articles-templated.rb
+- examples/munging/wikipedia/articles/textualize_articles.rb
+- examples/munging/wikipedia/articles/verify_structure.rb
+- examples/munging/wikipedia/articles/wp2txt-LICENSE.txt
+- examples/munging/wikipedia/articles/wp2txt_article.rb
+- examples/munging/wikipedia/articles/wp2txt_utils.rb
+- examples/munging/wikipedia/dbpedia/dbpedia_common.rb
+- examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb
+- examples/munging/wikipedia/dbpedia/extract_links-cruft.rb
+- examples/munging/wikipedia/dbpedia/extract_links.rb
+- examples/munging/wikipedia/dbpedia/sameas_extractor.rb
+- examples/rake_helper.rb
+- examples/ruby_project/Gemfile
+- examples/ruby_project/README.md
+- examples/ruby_project/a/b/c/.gitkeep
+- examples/server_logs/geo_ip_mapping/munge_geolite.rb
+- examples/server_logs/logline.rb
+- examples/server_logs/models.rb
+- examples/server_logs/page_counts.pig
+- examples/server_logs/server_logs-01-parse-script.rb
+- examples/server_logs/server_logs-02-histograms-full.rb
+- examples/server_logs/server_logs-02-histograms-mapper.rb
+- examples/server_logs/server_logs-03-breadcrumbs-full.rb
+- examples/server_logs/server_logs-04-page_page_edges-full.rb
+- examples/serverlogs/geo_ip_mapping/munge_geolite.rb
+- examples/serverlogs/models/logline.rb
+- examples/serverlogs/parser/apache_parser_widget.rb
+- examples/serverlogs/visit_paths/common.rb
+- examples/serverlogs/visit_paths/page_counts.pig
+- examples/serverlogs/visit_paths/serverlogs-01-parse-script.rb
+- examples/serverlogs/visit_paths/serverlogs-02-histograms-full.rb
+- examples/serverlogs/visit_paths/serverlogs-02-histograms-mapper.rb
+- examples/serverlogs/visit_paths/serverlogs-03-breadcrumbs-full.rb
+- examples/serverlogs/visit_paths/serverlogs-04-page_page_edges-full.rb
+- examples/splitter.rb
+- examples/string_reverser.rb
+- examples/text/pig_latin/pig_latinizer.rb
+- examples/text/pig_latin/pig_latinizer_widget.rb
+- examples/text/regional_flavor/README.md
+- examples/text/regional_flavor/article_wordbags.pig
+- examples/text/regional_flavor/j01-article_wordbags.rb
+- examples/text/regional_flavor/simple_pig_script.pig
+- examples/twitter.rb
+- lib/hanuman.rb
+- lib/hanuman/graph.rb
+- lib/hanuman/graphvizzer.rb
+- lib/hanuman/graphvizzer/gv_models.rb
+- lib/hanuman/graphvizzer/gv_presenter.rb
+- lib/hanuman/link.rb
+- lib/hanuman/registry.rb
+- lib/hanuman/stage.rb
+- lib/hanuman/tree.rb
+- lib/wu/geo.rb
+- lib/wu/geo/geo_grids.numbers
+- lib/wu/geo/geolocated.rb
+- lib/wu/geo/quadtile.rb
+- lib/wu/graph/union_find.rb
+- lib/wu/model/reconcilable.rb
+- lib/wu/munging.rb
+- lib/wu/social/models/twitter.rb
+- lib/wu/wikipedia/models.rb
+- lib/wukong.rb
+- lib/wukong/dataflow.rb
+- lib/wukong/doc_helpers.rb
+- lib/wukong/doc_helpers/dataflow_handler.rb
+- lib/wukong/doc_helpers/field_handler.rb
+- lib/wukong/doc_helpers/processor_handler.rb
+- lib/wukong/driver.rb
+- lib/wukong/driver/event_machine_driver.rb
+- lib/wukong/driver/wiring.rb
+- lib/wukong/local.rb
+- lib/wukong/local/runner.rb
+- lib/wukong/local/stdio_driver.rb
+- lib/wukong/logger.rb
+- lib/wukong/model/faker.rb
+- lib/wukong/model/flatpack_parser/flat.rb
+- lib/wukong/model/flatpack_parser/flatpack.rb
+- lib/wukong/model/flatpack_parser/lang.rb
+- lib/wukong/model/flatpack_parser/parser.rb
+- lib/wukong/model/flatpack_parser/tokens.rb
+- lib/wukong/plugin.rb
+- lib/wukong/processor.rb
+- lib/wukong/rake_helper.rb
+- lib/wukong/runner.rb
+- lib/wukong/runner/boot_sequence.rb
+- lib/wukong/runner/code_loader.rb
+- lib/wukong/runner/command_runner.rb
+- lib/wukong/runner/deploy_pack_loader.rb
+- lib/wukong/runner/help_message.rb
+- lib/wukong/source.rb
+- lib/wukong/source/source_driver.rb
+- lib/wukong/source/source_runner.rb
+- lib/wukong/spec_helpers.rb
+- lib/wukong/spec_helpers/integration_tests.rb
+- lib/wukong/spec_helpers/integration_tests/integration_test_matchers.rb
+- lib/wukong/spec_helpers/integration_tests/integration_test_runner.rb
+- lib/wukong/spec_helpers/shared_examples.rb
+- lib/wukong/spec_helpers/unit_tests.rb
+- lib/wukong/spec_helpers/unit_tests/unit_test_driver.rb
+- lib/wukong/spec_helpers/unit_tests/unit_test_matchers.rb
+- lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb
+- lib/wukong/version.rb
+- lib/wukong/widget/echo.rb
+- lib/wukong/widget/extract.rb
+- lib/wukong/widget/filters.rb
+- lib/wukong/widget/logger.rb
+- lib/wukong/widget/operators.rb
+- lib/wukong/widget/reducers.rb
+- lib/wukong/widget/reducers/accumulator.rb
+- lib/wukong/widget/reducers/bin.rb
+- lib/wukong/widget/reducers/count.rb
+- lib/wukong/widget/reducers/group.rb
+- lib/wukong/widget/reducers/group_concat.rb
+- lib/wukong/widget/reducers/improver.rb
+- lib/wukong/widget/reducers/join_xml.rb
+- lib/wukong/widget/reducers/moments.rb
+- lib/wukong/widget/reducers/sort.rb
+- lib/wukong/widget/reducers/uniq.rb
+- lib/wukong/widget/serializers.rb
+- lib/wukong/widget/utils.rb
+- lib/wukong/widgets.rb
+- spec/examples/dataflow/fibonacci_series_spec.rb
+- spec/examples/dataflow/parse_apache_logs_spec.rb
+- spec/examples/dataflow/parsing_spec.rb
+- spec/examples/dataflow/simple_spec.rb
+- spec/examples/dataflow/telegram_spec.rb
+- spec/examples/graph/minimum_spanning_tree_spec.rb
+- spec/examples/munging/airline_flights/identifiers_spec.rb
+- spec/examples/munging/airline_flights_spec.rb
+- spec/examples/text/pig_latin_spec.rb
+- spec/examples/workflow/cherry_pie_spec.rb
+- spec/hanuman/graph_spec.rb
+- spec/hanuman/hanuman_spec.rb
+- spec/hanuman/registry_spec.rb
+- spec/hanuman/stage_spec.rb
+- spec/hanuman/tree_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/support/example_test_helpers.rb
+- spec/support/hanuman_test_helpers.rb
+- spec/support/integration_helper.rb
+- spec/support/model_test_helpers.rb
+- spec/support/shared_context_for_graphs.rb
+- spec/support/shared_context_for_reducers.rb
+- spec/support/shared_examples_for_builders.rb
+- spec/support/shared_examples_for_shortcuts.rb
+- spec/wu/model/reconcilable_spec.rb
+- spec/wukong/dataflow_spec.rb
+- spec/wukong/driver_spec.rb
+- spec/wukong/local/runner_spec.rb
+- spec/wukong/local/stdio_driver_spec.rb
+- spec/wukong/local_spec.rb
+- spec/wukong/logger_spec.rb
+- spec/wukong/model/faker_spec.rb
+- spec/wukong/processor_spec.rb
+- spec/wukong/runner_spec.rb
+- spec/wukong/source_spec.rb
+- spec/wukong/widget/extract_spec.rb
+- spec/wukong/widget/filters_spec.rb
+- spec/wukong/widget/logger_spec.rb
+- spec/wukong/widget/operators_spec.rb
+- spec/wukong/widget/reducers/bin_spec.rb
+- spec/wukong/widget/reducers/count_spec.rb
+- spec/wukong/widget/reducers/group_spec.rb
+- spec/wukong/widget/reducers/join_xml_spec.rb
+- spec/wukong/widget/reducers/moments_spec.rb
+- spec/wukong/widget/reducers/sort_spec.rb
+- spec/wukong/widget/reducers/uniq_spec.rb
+- spec/wukong/widget/serializers_spec.rb
+- spec/wukong/widget/sink_spec.rb
+- spec/wukong/widget/source_spec.rb
+- spec/wukong/wu-local_spec.rb
+- spec/wukong/wu-source_spec.rb
+- spec/wukong/wu_spec.rb
+- spec/wukong/wukong_spec.rb
+- wukong.gemspec
+homepage: https://github.com/undeadlabs/wukong
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fork of Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee
+  can use it, yet handles terabyte-scale computation with ease.
+test_files:
+- spec/examples/dataflow/fibonacci_series_spec.rb
+- spec/examples/dataflow/parse_apache_logs_spec.rb
+- spec/examples/dataflow/parsing_spec.rb
+- spec/examples/dataflow/simple_spec.rb
+- spec/examples/dataflow/telegram_spec.rb
+- spec/examples/graph/minimum_spanning_tree_spec.rb
+- spec/examples/munging/airline_flights/identifiers_spec.rb
+- spec/examples/munging/airline_flights_spec.rb
+- spec/examples/text/pig_latin_spec.rb
+- spec/examples/workflow/cherry_pie_spec.rb
+- spec/hanuman/graph_spec.rb
+- spec/hanuman/hanuman_spec.rb
+- spec/hanuman/registry_spec.rb
+- spec/hanuman/stage_spec.rb
+- spec/hanuman/tree_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/support/example_test_helpers.rb
+- spec/support/hanuman_test_helpers.rb
+- spec/support/integration_helper.rb
+- spec/support/model_test_helpers.rb
+- spec/support/shared_context_for_graphs.rb
+- spec/support/shared_context_for_reducers.rb
+- spec/support/shared_examples_for_builders.rb
+- spec/support/shared_examples_for_shortcuts.rb
+- spec/wu/model/reconcilable_spec.rb
+- spec/wukong/dataflow_spec.rb
+- spec/wukong/driver_spec.rb
+- spec/wukong/local/runner_spec.rb
+- spec/wukong/local/stdio_driver_spec.rb
+- spec/wukong/local_spec.rb
+- spec/wukong/logger_spec.rb
+- spec/wukong/model/faker_spec.rb
+- spec/wukong/processor_spec.rb
+- spec/wukong/runner_spec.rb
+- spec/wukong/source_spec.rb
+- spec/wukong/widget/extract_spec.rb
+- spec/wukong/widget/filters_spec.rb
+- spec/wukong/widget/logger_spec.rb
+- spec/wukong/widget/operators_spec.rb
+- spec/wukong/widget/reducers/bin_spec.rb
+- spec/wukong/widget/reducers/count_spec.rb
+- spec/wukong/widget/reducers/group_spec.rb
+- spec/wukong/widget/reducers/join_xml_spec.rb
+- spec/wukong/widget/reducers/moments_spec.rb
+- spec/wukong/widget/reducers/sort_spec.rb
+- spec/wukong/widget/reducers/uniq_spec.rb
+- spec/wukong/widget/serializers_spec.rb
+- spec/wukong/widget/sink_spec.rb
+- spec/wukong/widget/source_spec.rb
+- spec/wukong/wu-local_spec.rb
+- spec/wukong/wu-source_spec.rb
+- spec/wukong/wu_spec.rb
+- spec/wukong/wukong_spec.rb
+has_rdoc: