wukong 3.0.0.pre → 3.0.0.pre2

data/spec/wukong/widget/sink_spec.rb

@@ -1,19 +1,19 @@
-require 'spec_helper'
+# require 'spec_helper'
 
-describe :sinks, :helpers => true do
+# describe :sinks, :helpers => true do
 
-  describe Wukong::Sink::Stdout do
-    it 'dumps records to $stdout' do
-      $stdout.should_receive(:puts).with(mock_record)
-      subject.process(mock_record)
-    end
-  end
+#   describe Wukong::Sink::Stdout do
+#     it 'dumps records to $stdout' do
+#       $stdout.should_receive(:puts).with(mock_record)
+#       subject.process(mock_record)
+#     end
+#   end
 
-  describe Wukong::Sink::Stderr do
-    it 'dumps records to $stderr' do
-      $stderr.should_receive(:puts).with(mock_record)
-      subject.process(mock_record)
-    end
+#   describe Wukong::Sink::Stderr do
+#     it 'dumps records to $stderr' do
+#       $stderr.should_receive(:puts).with(mock_record)
+#       subject.process(mock_record)
+#     end
 
-  end
-end
+#   end
+# end

data/spec/wukong/widget/source_spec.rb

@@ -1,41 +1,65 @@
-require 'spec_helper'
-require 'wukong/widget/gibberish'
-
-describe :sources, :helpers => true do
-  describe Wukong::Source::Integers do
-    subject{ described_class.receive(:size => 10) }
-    before{ subject.setup }
-
-    it 'generates integers up to the given limit' do
-      subject.to_enum.to_a.should == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-    end
-    it 'generates nothing if the initial range is void' do
-      subject.size = 0
-      subject.to_enum.to_a.should == []
-    end
-    it 'generates one thing if the min and max are equal' do
-      subject.size = 1
-      subject.to_enum.to_a.should == [0]
-    end
-
-    context 'dataflow method' do
-      it 'is defined' do
-      end
-      it 'takes simplified args' do
-        described_class.should_receive(:receive).with({:size => 99}).and_return(described_class.new)
-        test_dataflow.integers(99)
-      end
-    end
-  end
-
-  describe Wukong::Widget::Gibberish do
-    subject{ described_class.receive(:size => 3) }
-    before{ subject.setup }
-
-    it 'generates integers up to the given limit' do
-      subject.rng = Random.new(8675309)
-      subject.to_enum.to_a.should == ["loaiaeiaeo neidgfo heeume sptfmeec naet sttptlm waaaioh detov elrrltv nii ulcsnn", "set ensr poeleaa seqi tmnreoee boooral oczncgp deaia rykcoao leo rim mmibpbfii", "artrru sto quuu doo peoehrile nto esl tia gaili tuiooey lkedotp sts kaiy weeeia", "crhi tyiiae mieubmbooa teeae roi ednz taieh zaloy syhe ret kuoa deeeo xittipl mo"]
-    end
-
-  end
-end
+# require 'spec_helper'
+# require 'wukong/widget/gibberish'
+
+# describe :sources, :helpers => true do
+
+#   describe Wukong::Source::Iter do
+#     subject{ described_class.new(obj: (9 .. 14), owner: test_dataflow) }
+#     it 'iterates over a given collection' do
+#       subject.to_enum.to_a.should == [9, 10, 11, 12, 13, 14]
+#     end
+#     context 'dataflow method' do
+#       it 'simplified args' do
+#         test_dataflow.iter(9 .. 14).should == subject
+#       end
+#     end
+#   end
+
+#   # describe Wukong::Source::FileSource do
+#   #   let(:example_filename){ Pathname.path_to(:data, 'text/jabberwocky.txt') }
+#   #   subject{ described_class.receive(filename: example_filename, owner: test_dataflow) }
+#   #   before{ subject.setup }
+#   #   it 'iterates over a given collection' do
+#   #     subject.to_enum.to_a[6, 4].should == ["`Twas brillig, and the slithy toves", "  Did gyre and gimble in the wabe:", "All mimsy were the borogoves,", "  And the mome raths outgrabe.",]
+#   #   end
+#   #   context 'dataflow method' do
+#   #     it 'simplified args' do
+#   #       test_dataflow.file_source(example_filename).should == subject
+#   #     end
+#   #   end
+#   # end
+
+#   describe Wukong::Source::Integers do
+#     subject{ described_class.receive(qty: 10, owner: test_dataflow) }
+#     before{ subject.setup }
+
+#     it 'generates integers up to the given limit' do
+#       subject.to_enum.to_a.should == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+#     end
+#     it 'generates nothing if the initial range is void' do
+#       subject.qty = 0
+#       subject.to_enum.to_a.should == []
+#     end
+#     it 'generates one thing if the min and max are equal' do
+#       subject.qty = 1
+#       subject.to_enum.to_a.should == [0]
+#     end
+
+#     context 'dataflow method' do
+#       it 'takes simplified args' do
+#         test_dataflow.integers(10).should == subject
+#       end
+#     end
+#   end
+
+#   describe Wukong::Widget::Gibberish do
+#     subject{ described_class.receive(:qty => 4) }
+#     before{ subject.setup }
+
+#     it 'generates integers up to the given limit' do
+#       subject.rng = Random.new(8675309)
+#       subject.to_enum.to_a.should == ["loaiaeiaeo neidgfo heeume sptfmeec naet sttptlm waaaioh detov elrrltv nii ulcsnn", "set ensr poeleaa seqi tmnreoee boooral oczncgp deaia rykcoao leo rim mmibpbfii", "artrru sto quuu doo peoehrile nto esl tia gaili tuiooey lkedotp sts kaiy weeeia", "crhi tyiiae mieubmbooa teeae roi ednz taieh zaloy syhe ret kuoa deeeo xittipl mo"]
+#     end
+
+#   end
+# end

data/spec/wukong/wukong_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe Wukong do
+
+  it_behaves_like Hanuman::Shortcuts
+
+  it{ should respond_to(:processor) }
+  it{ should respond_to(:dataflow)  }
+
+end

data/wukong.gemspec

@@ -2,12 +2,15 @@
 require File.expand_path('../lib/wukong/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.name          = 'wukong'
-  gem.version       = Wukong::VERSION
-  gem.authors       = ['Philip (flip) Kromer', 'Travis Dempsey']
-  gem.homepage      = 'https://github.com/infochimps-labs/wukong'
-  gem.summary       = 'Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use it, yet handles terabyte-scale computation with ease.'
-  gem.description   = <<DESC
+  gem.name        = 'wukong'
+  gem.homepage    = 'https://github.com/infochimps-labs/wukong'
+  gem.licenses    = ["Apache 2.0"]
+  gem.email       = 'coders@infochimps.org'
+  gem.authors     = ['Infochimps', 'Philip (flip) Kromer', 'Travis Dempsey']
+  gem.version     = Wukong::VERSION
+
+  gem.summary     = 'Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use it, yet handles terabyte-scale computation with ease.'
+  gem.description = <<-EOF
   Treat your dataset like a:
 
       * stream of lines when it's efficient to process by lines
@@ -15,21 +18,23 @@ Gem::Specification.new do |gem|
       * stream of lightweight objects when it's efficient to deal with objects
 
   Wukong is friends with Hadoop the elephant, Pig the query language, and the cat on your command line.
-DESC
+EOF
 
-  gem.files         = `git ls-files`.split("\n")
-  gem.executables   = [] # gem.files.grep(/^bin/).map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n").reject { |path| path =~ /^(data|docpages|notes|old)/ }
+  gem.executables   = ['wu-local']
   gem.test_files    = gem.files.grep(/^spec/)
   gem.require_paths = ['lib']
 
-  gem.add_dependency('bundler',                 '~> 1.1')
   gem.add_dependency('configliere',             '~> 0.4')
   gem.add_dependency('multi_json',              '>= 1.3.6')
   gem.add_dependency('vayacondios-client',      '>= 0.0.3')
   gem.add_dependency('gorillib',                '>= 0.4.2')
   gem.add_dependency('forgery')
   gem.add_dependency('uuidtools')
+  gem.add_dependency('eventmachine')
+  gem.add_dependency('log4r')
 
+  gem.add_development_dependency('bundler',     '~> 1.1')
   gem.add_development_dependency('rake',        '>= 0.9')
   gem.add_development_dependency('rspec',       '>= 2.8')
   gem.add_development_dependency('guard',       '>= 1.0')
@@ -38,5 +43,7 @@ DESC
   gem.add_development_dependency('pry')
   gem.add_development_dependency('yard')
   gem.add_development_dependency('redcarpet')
+  gem.add_development_dependency('addressable')
+  gem.add_development_dependency('htmlentities')
 
 end

metadata

@@ -1,42 +1,37 @@
 --- !ruby/object:Gem::Specification
 name: wukong
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre
+  version: 3.0.0.pre2
   prerelease: 6
 platform: ruby
 authors:
+- Infochimps
 - Philip (flip) Kromer
 - Travis Dempsey
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-20 00:00:00.000000000Z
+date: 2012-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: &2162964540 !ruby/object:Gem::Requirement
+  name: configliere
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '0.4'
   type: :runtime
   prerelease: false
-  version_requirements: *2162964540
-- !ruby/object:Gem::Dependency
-  name: configliere
-  requirement: &2162964020 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.4'
-  type: :runtime
-  prerelease: false
-  version_requirements: *2162964020
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &2162963500 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +39,15 @@ dependencies:
         version: 1.3.6
   type: :runtime
   prerelease: false
-  version_requirements: *2162963500
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.6
 - !ruby/object:Gem::Dependency
   name: vayacondios-client
-  requirement: &2162962920 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +55,15 @@ dependencies:
         version: 0.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *2162962920
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.3
 - !ruby/object:Gem::Dependency
   name: gorillib
-  requirement: &2162962320 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,10 +71,15 @@ dependencies:
         version: 0.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *2162962320
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.2
 - !ruby/object:Gem::Dependency
   name: forgery
-  requirement: &2162961820 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -77,10 +87,31 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2162961820
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: uuidtools
-  requirement: &2162961220 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -88,10 +119,47 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2162961220
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: log4r
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2162960580 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -99,10 +167,15 @@ dependencies:
         version: '0.9'
   type: :development
   prerelease: false
-  version_requirements: *2162960580
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2162959980 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -110,10 +183,15 @@ dependencies:
         version: '2.8'
   type: :development
   prerelease: false
-  version_requirements: *2162959980
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: guard
-  requirement: &2162959400 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -121,10 +199,15 @@ dependencies:
         version: '1.0'
   type: :development
   prerelease: false
-  version_requirements: *2162959400
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &2162958780 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -132,10 +215,15 @@ dependencies:
         version: '0.6'
   type: :development
   prerelease: false
-  version_requirements: *2162958780
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &2162958140 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -143,10 +231,15 @@ dependencies:
         version: '0.5'
   type: :development
   prerelease: false
-  version_requirements: *2162958140
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &2162957700 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -154,10 +247,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2162957700
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &2162957140 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -165,10 +263,47 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2162957140
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: redcarpet
-  requirement: &2162956680 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -176,18 +311,23 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2162956680
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ! "  Treat your dataset like a:\n\n      * stream of lines when it's
   efficient to process by lines\n      * stream of field arrays when it's efficient
   to deal directly with fields\n      * stream of lightweight objects when it's efficient
   to deal with objects\n\n  Wukong is friends with Hadoop the elephant, Pig the query
   language, and the cat on your command line.\n"
-email: 
-executables: []
+email: coders@infochimps.org
+executables:
+- wu-local
 extensions: []
 extra_rdoc_files: []
 files:
-- .document
 - .gitignore
 - .gitmodules
 - .rspec
@@ -196,142 +336,137 @@ files:
 - CHANGELOG.md
 - Gemfile
 - Guardfile
-- LICENSE.textile
+- LICENSE.md
 - NOTES-travis.md
+- README-old.md
 - README.md
 - Rakefile
 - TODO.md
-- VERSION
 - bin/cutc
 - bin/cuttab
 - bin/greptrue
-- bin/hdp-bin
-- bin/hdp-bzip
-- bin/hdp-cat
-- bin/hdp-catd
-- bin/hdp-cp
-- bin/hdp-du
-- bin/hdp-get
-- bin/hdp-kill
-- bin/hdp-kill-task
-- bin/hdp-ls
-- bin/hdp-mkdir
-- bin/hdp-mkdirp
-- bin/hdp-mv
-- bin/hdp-parts_to_keys.rb
-- bin/hdp-ps
-- bin/hdp-put
-- bin/hdp-rm
-- bin/hdp-sort
-- bin/hdp-stream
-- bin/hdp-stream-flat
-- bin/hdp-stream2
-- bin/hdp-sync
-- bin/hdp-wc
 - bin/md5sort
 - bin/setcat
 - bin/tabchar
 - bin/uniq-ord
 - bin/uniqc
+- bin/wu-clean-encoding
 - bin/wu-date
 - bin/wu-datetime
-- bin/wu-flow
 - bin/wu-hist
 - bin/wu-lign
-- bin/wu-map
+- bin/wu-local
 - bin/wu-plus
-- bin/wu-red
+- bin/wu-server
 - bin/wu-sum
-- bin/wukong
-- data/CREDITS.md
-- data/graph/airfares.tsv
-- data/log/sample_apache_log.log
-- data/text/gift_of_the_magi.txt
-- data/text/jabberwocky.txt
-- data/text/rectification_of_names.txt
-- data/twitter/a_atsigns_b.tsv
-- data/twitter/a_follows_b.tsv
-- data/twitter/tweet.tsv
-- data/twitter/twitter_user.tsv
-- data/wikipedia/dbpedia-sentences.tsv
-- docpages/INSTALL.textile
-- docpages/LICENSE.textile
-- docpages/README-elastic_map_reduce.textile
-- docpages/README-performance.textile
-- docpages/README-wulign.textile
-- docpages/UsingWukong-part1-get_ready.textile
-- docpages/UsingWukong-part2-ThinkingBigData.textile
-- docpages/UsingWukong-part3-parsing.textile
-- docpages/_config.yml
-- docpages/avro/avro_notes.textile
-- docpages/avro/performance.textile
-- docpages/avro/tethering.textile
-- docpages/bigdata-tips.textile
-- docpages/code/api_response_example.txt
-- docpages/code/parser_skeleton.rb
-- docpages/diagrams/MapReduceDiagram.graffle
-- docpages/favicon.ico
-- docpages/gem.css
-- docpages/hadoop-tips.textile
-- docpages/index.textile
-- docpages/intro.textile
-- docpages/moreinfo.textile
-- docpages/news.html
-- docpages/pig/PigLatinExpressionsList.txt
-- docpages/pig/PigLatinReferenceManual.txt
-- docpages/pig/commandline_params.txt
-- docpages/pig/cookbook.html
-- docpages/pig/images/hadoop-logo.jpg
-- docpages/pig/images/instruction_arrow.png
-- docpages/pig/images/pig-logo.gif
-- docpages/pig/piglatin_ref1.html
-- docpages/pig/piglatin_ref2.html
-- docpages/pig/setup.html
-- docpages/pig/skin/basic.css
-- docpages/pig/skin/breadcrumbs.js
-- docpages/pig/skin/fontsize.js
-- docpages/pig/skin/getBlank.js
-- docpages/pig/skin/getMenu.js
-- docpages/pig/skin/images/chapter.gif
-- docpages/pig/skin/images/chapter_open.gif
-- docpages/pig/skin/images/current.gif
-- docpages/pig/skin/images/external-link.gif
-- docpages/pig/skin/images/header_white_line.gif
-- docpages/pig/skin/images/page.gif
-- docpages/pig/skin/images/pdfdoc.gif
-- docpages/pig/skin/images/rc-b-l-15-1body-2menu-3menu.png
-- docpages/pig/skin/images/rc-b-r-15-1body-2menu-3menu.png
-- docpages/pig/skin/images/rc-b-r-5-1header-2tab-selected-3tab-selected.png
-- docpages/pig/skin/images/rc-t-l-5-1header-2searchbox-3searchbox.png
-- docpages/pig/skin/images/rc-t-l-5-1header-2tab-selected-3tab-selected.png
-- docpages/pig/skin/images/rc-t-l-5-1header-2tab-unselected-3tab-unselected.png
-- docpages/pig/skin/images/rc-t-r-15-1body-2menu-3menu.png
-- docpages/pig/skin/images/rc-t-r-5-1header-2searchbox-3searchbox.png
-- docpages/pig/skin/images/rc-t-r-5-1header-2tab-selected-3tab-selected.png
-- docpages/pig/skin/images/rc-t-r-5-1header-2tab-unselected-3tab-unselected.png
-- docpages/pig/skin/print.css
-- docpages/pig/skin/profile.css
-- docpages/pig/skin/screen.css
-- docpages/pig/tutorial.html
-- docpages/pig/udf.html
-- docpages/tutorial.textile
-- docpages/usage.textile
-- docpages/wutils.textile
-- examples/dataflow.rb
+- examples/Gemfile
+- examples/README.md
 - examples/dataflow/apache_log_line.rb
-- examples/dataflow/complex.rb
-- examples/dataflow/donuts.rb
+- examples/dataflow/fibonacci_series.rb
 - examples/dataflow/parse_apache_logs.rb
 - examples/dataflow/pig_latinizer.rb
+- examples/dataflow/scraper_macro_flow.rb
 - examples/dataflow/simple.rb
 - examples/dataflow/telegram.rb
 - examples/examples_helper.rb
+- examples/geo.rb
+- examples/geo/geo_grids.numbers
+- examples/geo/geolocated.rb
+- examples/geo/quadtile.rb
+- examples/geo/spec/geolocated_spec.rb
+- examples/geo/tile_fetcher.rb
 - examples/graph/minimum_spanning_tree.rb
 - examples/graph/union_find.rb
+- examples/jabberwocky.txt
+- examples/models/wikipedia.rb
+- examples/munging/Gemfile
+- examples/munging/airline_flights/airline.rb
+- examples/munging/airline_flights/airline_flights.rake
+- examples/munging/airline_flights/airplane.rb
+- examples/munging/airline_flights/airport.rb
+- examples/munging/airline_flights/airport_id_unification.rb
+- examples/munging/airline_flights/airport_ok_chars.rb
+- examples/munging/airline_flights/flight.rb
+- examples/munging/airline_flights/models.rb
+- examples/munging/airline_flights/parse.rb
+- examples/munging/airline_flights/reconcile_airports.rb
+- examples/munging/airline_flights/route.rb
+- examples/munging/airline_flights/tasks.rake
+- examples/munging/airline_flights/timezone_fixup.rb
+- examples/munging/airline_flights/topcities.rb
+- examples/munging/airports/40_wbans.txt
+- examples/munging/airports/filter_weather_reports.rb
+- examples/munging/airports/join.pig
+- examples/munging/airports/to_tsv.rb
+- examples/munging/airports/usa_wbans.pig
+- examples/munging/airports/usa_wbans.txt
+- examples/munging/airports/wbans.pig
+- examples/munging/airports/wbans.txt
+- examples/munging/geo/geo_json.rb
+- examples/munging/geo/geo_models.rb
+- examples/munging/geo/geonames_models.rb
+- examples/munging/geo/iso_codes.rb
+- examples/munging/geo/reconcile_countries.rb
+- examples/munging/geo/tasks.rake
+- examples/munging/rake_helper.rb
+- examples/munging/weather/.gitignore
+- examples/munging/weather/Gemfile
+- examples/munging/weather/Rakefile
+- examples/munging/weather/extract_ish.rb
+- examples/munging/weather/models/weather.rb
+- examples/munging/weather/utils/noaa_downloader.rb
+- examples/munging/wikipedia/README.md
+- examples/munging/wikipedia/Rakefile
+- examples/munging/wikipedia/articles/extract_articles-parsed.rb
+- examples/munging/wikipedia/articles/extract_articles-templated.rb
+- examples/munging/wikipedia/articles/textualize_articles.rb
+- examples/munging/wikipedia/articles/verify_structure.rb
+- examples/munging/wikipedia/articles/wp2txt-LICENSE.txt
+- examples/munging/wikipedia/articles/wp2txt_article.rb
+- examples/munging/wikipedia/articles/wp2txt_utils.rb
+- examples/munging/wikipedia/dbpedia/dbpedia_common.rb
+- examples/munging/wikipedia/dbpedia/dbpedia_extract_geocoordinates.rb
+- examples/munging/wikipedia/dbpedia/extract_links.rb
+- examples/munging/wikipedia/dbpedia/sameas_extractor.rb
+- examples/munging/wikipedia/n1_subuniverse/n1_nodes.pig
+- examples/munging/wikipedia/page_metadata/extract_page_metadata.rb
+- examples/munging/wikipedia/page_metadata/extract_page_metadata.rb.old
+- examples/munging/wikipedia/pagelinks/augment_pagelinks.pig
+- examples/munging/wikipedia/pagelinks/extract_pagelinks.rb
+- examples/munging/wikipedia/pagelinks/extract_pagelinks.rb.old
+- examples/munging/wikipedia/pagelinks/undirect_pagelinks.pig
+- examples/munging/wikipedia/pageviews/augment_pageviews.pig
+- examples/munging/wikipedia/pageviews/extract_pageviews.rb
+- examples/munging/wikipedia/pig_style_guide.md
+- examples/munging/wikipedia/redirects/redirects_page_metadata.pig
+- examples/munging/wikipedia/subuniverse/sub_articles.pig
+- examples/munging/wikipedia/subuniverse/sub_page_metadata.pig
+- examples/munging/wikipedia/subuniverse/sub_pagelinks_from.pig
+- examples/munging/wikipedia/subuniverse/sub_pagelinks_into.pig
+- examples/munging/wikipedia/subuniverse/sub_pagelinks_within.pig
+- examples/munging/wikipedia/subuniverse/sub_pageviews.pig
+- examples/munging/wikipedia/subuniverse/sub_undirected_pagelinks_within.pig
+- examples/munging/wikipedia/utils/get_namespaces.rb
+- examples/munging/wikipedia/utils/munging_utils.rb
+- examples/munging/wikipedia/utils/namespaces.json
+- examples/rake_helper.rb
+- examples/server_logs/geo_ip_mapping/munge_geolite.rb
+- examples/server_logs/logline.rb
+- examples/server_logs/models.rb
+- examples/server_logs/page_counts.pig
+- examples/server_logs/server_logs-01-parse-script.rb
+- examples/server_logs/server_logs-02-histograms-full.rb
+- examples/server_logs/server_logs-02-histograms-mapper.rb
+- examples/server_logs/server_logs-03-breadcrumbs-full.rb
+- examples/server_logs/server_logs-04-page_page_edges-full.rb
+- examples/string_reverser.rb
 - examples/text/latinize_text.rb
 - examples/text/pig_latin.rb
+- examples/text/regional_flavor/README.md
+- examples/text/regional_flavor/article_wordbags.pig
+- examples/text/regional_flavor/j01-article_wordbags.rb
+- examples/text/regional_flavor/simple_pig_script.pig
 - examples/tiny_count.rb
-- examples/tiny_count/jabberwocky_output.tsv
 - examples/twitter/locations.rb
 - examples/twitter/models.rb
 - examples/twitter/pt1-fiddle.pig
@@ -341,211 +476,99 @@ files:
 - examples/twitter/pt4-strong_links.rb
 - examples/twitter/pt5-lnglat_and_strong_links.pig
 - examples/twitter/states.tsv
-- examples/word_count.rb
+- examples/word_count/accumulator.rb
+- examples/word_count/tokenizer.rb
+- examples/word_count/word_count.rb
+- examples/workflow/cherry_pie.dot
 - examples/workflow/cherry_pie.md
+- examples/workflow/cherry_pie.png
 - examples/workflow/cherry_pie.rb
-- examples/workflow/fiddle.rb
 - examples/workflow/package_gem.rb
-- lib/away/escapement.rb
-- lib/away/exe.rb
-- lib/away/experimental.rb
-- lib/away/from_file.rb
-- lib/away/job.rb
-- lib/away/job/rake_compat.rb
-- lib/away/registry.rb
-- lib/away/runner.rb
-- lib/away/runner/execute.rb
-- lib/away/script.rb
-- lib/away/script/hadoop_command.rb
-- lib/away/source/file_list_source.rb
-- lib/away/source/looper.rb
-- lib/away/task.rb
 - lib/hanuman.rb
-- lib/hanuman/action.rb
-- lib/hanuman/chain.rb
 - lib/hanuman/graph.rb
-- lib/hanuman/graphviz.rb
 - lib/hanuman/graphvizzer.rb
-- lib/hanuman/resource.rb
-- lib/hanuman/slot.rb
-- lib/hanuman/slottable.rb
+- lib/hanuman/graphvizzer/gv_models.rb
+- lib/hanuman/graphvizzer/gv_presenter.rb
+- lib/hanuman/link.rb
+- lib/hanuman/registry.rb
 - lib/hanuman/stage.rb
 - lib/wukong.rb
-- lib/wukong/bad_record.rb
+- lib/wukong/boot.rb
+- lib/wukong/configuration.rb
 - lib/wukong/dataflow.rb
-- lib/wukong/event.rb
-- lib/wukong/local_runner.rb
-- lib/wukong/mapred.rb
+- lib/wukong/driver.rb
+- lib/wukong/emitter.rb
 - lib/wukong/model/faker.rb
+- lib/wukong/model/flatpack_parser/flat.rb
+- lib/wukong/model/flatpack_parser/flatpack.rb
+- lib/wukong/model/flatpack_parser/lang.rb
+- lib/wukong/model/flatpack_parser/parser.rb
+- lib/wukong/model/flatpack_parser/tokens.rb
 - lib/wukong/processor.rb
-- lib/wukong/settings.rb
-- lib/wukong/universe.rb
+- lib/wukong/spec_helpers.rb
+- lib/wukong/spec_helpers/integration_driver.rb
+- lib/wukong/spec_helpers/integration_driver_matchers.rb
+- lib/wukong/spec_helpers/processor_helpers.rb
+- lib/wukong/spec_helpers/processor_methods.rb
+- lib/wukong/spec_helpers/shared_examples.rb
+- lib/wukong/spec_helpers/spec_driver.rb
+- lib/wukong/spec_helpers/spec_driver_matchers.rb
 - lib/wukong/version.rb
-- lib/wukong/widget/filter.rb
-- lib/wukong/widget/gibberish.rb
-- lib/wukong/widget/monitor.rb
-- lib/wukong/widget/reducer.rb
+- lib/wukong/widget/filters.rb
+- lib/wukong/widget/processors.rb
+- lib/wukong/widget/reducers.rb
+- lib/wukong/widget/reducers/accumulator.rb
+- lib/wukong/widget/reducers/bin.rb
+- lib/wukong/widget/reducers/count.rb
+- lib/wukong/widget/reducers/group.rb
+- lib/wukong/widget/reducers/group_concat.rb
+- lib/wukong/widget/reducers/moments.rb
+- lib/wukong/widget/reducers/sort.rb
+- lib/wukong/widget/serializers.rb
 - lib/wukong/widget/sink.rb
 - lib/wukong/widget/source.rb
-- lib/wukong/widget/stringifier.rb
-- lib/wukong/workflow.rb
-- lib/wukong/workflow/command.rb
-- old/config/emr-example.yaml
-- old/examples/README.txt
-- old/examples/contrib/jeans/README.markdown
-- old/examples/contrib/jeans/data/normalized_sizes
-- old/examples/contrib/jeans/data/orders.tsv
-- old/examples/contrib/jeans/data/sizes
-- old/examples/contrib/jeans/normalize.rb
-- old/examples/contrib/jeans/sizes.rb
-- old/examples/corpus/bnc_word_freq.rb
-- old/examples/corpus/bucket_counter.rb
-- old/examples/corpus/dbpedia_abstract_to_sentences.rb
-- old/examples/corpus/sentence_bigrams.rb
-- old/examples/corpus/sentence_coocurrence.rb
-- old/examples/corpus/stopwords.rb
-- old/examples/corpus/words_to_bigrams.rb
-- old/examples/emr/README.textile
-- old/examples/emr/dot_wukong_dir/credentials.json
-- old/examples/emr/dot_wukong_dir/emr.yaml
-- old/examples/emr/dot_wukong_dir/emr_bootstrap.sh
-- old/examples/emr/elastic_mapreduce_example.rb
-- old/examples/network_graph/adjacency_list.rb
-- old/examples/network_graph/breadth_first_search.rb
-- old/examples/network_graph/gen_2paths.rb
-- old/examples/network_graph/gen_multi_edge.rb
-- old/examples/network_graph/gen_symmetric_links.rb
-- old/examples/pagerank/README.textile
-- old/examples/pagerank/gen_initial_pagerank_graph.pig
-- old/examples/pagerank/pagerank.rb
-- old/examples/pagerank/pagerank_initialize.rb
-- old/examples/pagerank/run_pagerank.sh
-- old/examples/sample_records.rb
-- old/examples/server_logs/apache_log_parser.rb
-- old/examples/server_logs/breadcrumbs.rb
-- old/examples/server_logs/nook.rb
-- old/examples/server_logs/nook/faraday_dummy_adapter.rb
-- old/examples/server_logs/user_agent.rb
-- old/examples/simple_word_count.rb
-- old/examples/size.rb
-- old/examples/stats/avg_value_frequency.rb
-- old/examples/stats/binning_percentile_estimator.rb
-- old/examples/stats/data/avg_value_frequency.tsv
-- old/examples/stats/rank_and_bin.rb
-- old/examples/stupidly_simple_filter.rb
-- old/examples/word_count.rb
-- old/graph/graphviz_builder.rb
-- old/graph_easy/Attributes.pm
-- old/graph_easy/Graphviz.pm
-- old/wukong.rb
-- old/wukong/and_pig.rb
-- old/wukong/bad_record.rb
-- old/wukong/datatypes.rb
-- old/wukong/datatypes/enum.rb
-- old/wukong/datatypes/fake_types.rb
-- old/wukong/decorator.rb
-- old/wukong/encoding/asciize.rb
-- old/wukong/extensions.rb
-- old/wukong/extensions/array.rb
-- old/wukong/extensions/blank.rb
-- old/wukong/extensions/class.rb
-- old/wukong/extensions/date_time.rb
-- old/wukong/extensions/emittable.rb
-- old/wukong/extensions/enumerable.rb
-- old/wukong/extensions/hash.rb
-- old/wukong/extensions/hash_keys.rb
-- old/wukong/extensions/hash_like.rb
-- old/wukong/extensions/hashlike_class.rb
-- old/wukong/extensions/module.rb
-- old/wukong/extensions/pathname.rb
-- old/wukong/extensions/string.rb
-- old/wukong/extensions/struct.rb
-- old/wukong/extensions/symbol.rb
-- old/wukong/filename_pattern.rb
-- old/wukong/helper.rb
-- old/wukong/helper/stopwords.rb
-- old/wukong/helper/tokenize.rb
-- old/wukong/logger.rb
-- old/wukong/periodic_monitor.rb
-- old/wukong/schema.rb
-- old/wukong/script.rb
-- old/wukong/script/avro_command.rb
-- old/wukong/script/cassandra_loader_script.rb
-- old/wukong/script/emr_command.rb
-- old/wukong/script/hadoop_command.rb
-- old/wukong/script/local_command.rb
-- old/wukong/store.rb
-- old/wukong/store/base.rb
-- old/wukong/store/cassandra.rb
-- old/wukong/store/cassandra/streaming.rb
-- old/wukong/store/cassandra/struct_loader.rb
-- old/wukong/store/cassandra_model.rb
-- old/wukong/store/chh_chunked_flat_file_store.rb
-- old/wukong/store/chunked_flat_file_store.rb
-- old/wukong/store/conditional_store.rb
-- old/wukong/store/factory.rb
-- old/wukong/store/flat_file_store.rb
-- old/wukong/store/key_store.rb
-- old/wukong/store/null_store.rb
-- old/wukong/store/read_thru_store.rb
-- old/wukong/store/tokyo_tdb_key_store.rb
-- old/wukong/store/tyrant_rdb_key_store.rb
-- old/wukong/store/tyrant_tdb_key_store.rb
-- old/wukong/streamer.rb
-- old/wukong/streamer/accumulating_reducer.rb
-- old/wukong/streamer/base.rb
-- old/wukong/streamer/counting_reducer.rb
-- old/wukong/streamer/filter.rb
-- old/wukong/streamer/instance_streamer.rb
-- old/wukong/streamer/json_streamer.rb
-- old/wukong/streamer/line_streamer.rb
-- old/wukong/streamer/list_reducer.rb
-- old/wukong/streamer/rank_and_bin_reducer.rb
-- old/wukong/streamer/record_streamer.rb
-- old/wukong/streamer/reducer.rb
-- old/wukong/streamer/set_reducer.rb
-- old/wukong/streamer/struct_streamer.rb
-- old/wukong/streamer/summing_reducer.rb
-- old/wukong/streamer/uniq_by_last_reducer.rb
-- old/wukong/typed_struct.rb
-- spec/away/encoding_spec.rb
-- spec/away/exe_spec.rb
-- spec/away/flow_spec.rb
-- spec/away/graph_spec.rb
-- spec/away/job_spec.rb
-- spec/away/rake_compat_spec.rb
-- spec/away/script_spec.rb
+- lib/wukong/widget/utils.rb
+- lib/wukong/widgets.rb
+- spec/examples/dataflow/fibonacci_series_spec.rb
 - spec/examples/dataflow/parse_apache_logs_spec.rb
 - spec/examples/dataflow/parsing_spec.rb
 - spec/examples/dataflow/simple_spec.rb
 - spec/examples/dataflow/telegram_spec.rb
 - spec/examples/graph/minimum_spanning_tree_spec.rb
+- spec/examples/munging/airline_flights/identifiers_spec.rb
+- spec/examples/munging/airline_flights_spec.rb
 - spec/examples/text/pig_latin_spec.rb
 - spec/examples/workflow/cherry_pie_spec.rb
 - spec/hanuman/graph_spec.rb
-- spec/hanuman/graphviz_spec.rb
-- spec/hanuman/slot_spec.rb
+- spec/hanuman/hanuman_spec.rb
+- spec/hanuman/registry_spec.rb
 - spec/hanuman/stage_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-- spec/support/examples_helper.rb
 - spec/support/hanuman_test_helpers.rb
-- spec/support/streamer_test_helpers.rb
+- spec/support/shared_context_for_reducers.rb
+- spec/support/shared_examples_for_builders.rb
+- spec/support/shared_examples_for_shortcuts.rb
 - spec/support/wukong_test_helpers.rb
-- spec/support/wukong_widget_helpers.rb
 - spec/wukong/dataflow_spec.rb
 - spec/wukong/local_runner_spec.rb
 - spec/wukong/model/faker_spec.rb
-- spec/wukong/processor_spec.rb
 - spec/wukong/runner_spec.rb
-- spec/wukong/widget/filter_spec.rb
+- spec/wukong/widget/filters_spec.rb
+- spec/wukong/widget/processors_spec.rb
+- spec/wukong/widget/reducers/bin_spec.rb
+- spec/wukong/widget/reducers/count_spec.rb
+- spec/wukong/widget/reducers/group_spec.rb
+- spec/wukong/widget/reducers/moments_spec.rb
+- spec/wukong/widget/reducers/sort_spec.rb
+- spec/wukong/widget/serializers_spec.rb
 - spec/wukong/widget/sink_spec.rb
 - spec/wukong/widget/source_spec.rb
-- spec/wukong/widget/stringifier_spec.rb
-- spec/wukong/workflow/command_spec.rb
+- spec/wukong/wukong_spec.rb
 - wukong.gemspec
 homepage: https://github.com/infochimps-labs/wukong
-licenses: []
+licenses:
+- Apache 2.0
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -564,45 +587,46 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use
   it, yet handles terabyte-scale computation with ease.
 test_files:
-- spec/away/encoding_spec.rb
-- spec/away/exe_spec.rb
-- spec/away/flow_spec.rb
-- spec/away/graph_spec.rb
-- spec/away/job_spec.rb
-- spec/away/rake_compat_spec.rb
-- spec/away/script_spec.rb
+- spec/examples/dataflow/fibonacci_series_spec.rb
 - spec/examples/dataflow/parse_apache_logs_spec.rb
 - spec/examples/dataflow/parsing_spec.rb
 - spec/examples/dataflow/simple_spec.rb
 - spec/examples/dataflow/telegram_spec.rb
 - spec/examples/graph/minimum_spanning_tree_spec.rb
+- spec/examples/munging/airline_flights/identifiers_spec.rb
+- spec/examples/munging/airline_flights_spec.rb
 - spec/examples/text/pig_latin_spec.rb
 - spec/examples/workflow/cherry_pie_spec.rb
 - spec/hanuman/graph_spec.rb
-- spec/hanuman/graphviz_spec.rb
-- spec/hanuman/slot_spec.rb
+- spec/hanuman/hanuman_spec.rb
+- spec/hanuman/registry_spec.rb
 - spec/hanuman/stage_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-- spec/support/examples_helper.rb
 - spec/support/hanuman_test_helpers.rb
-- spec/support/streamer_test_helpers.rb
+- spec/support/shared_context_for_reducers.rb
+- spec/support/shared_examples_for_builders.rb
+- spec/support/shared_examples_for_shortcuts.rb
 - spec/support/wukong_test_helpers.rb
-- spec/support/wukong_widget_helpers.rb
 - spec/wukong/dataflow_spec.rb
 - spec/wukong/local_runner_spec.rb
 - spec/wukong/model/faker_spec.rb
-- spec/wukong/processor_spec.rb
 - spec/wukong/runner_spec.rb
-- spec/wukong/widget/filter_spec.rb
+- spec/wukong/widget/filters_spec.rb
+- spec/wukong/widget/processors_spec.rb
+- spec/wukong/widget/reducers/bin_spec.rb
+- spec/wukong/widget/reducers/count_spec.rb
+- spec/wukong/widget/reducers/group_spec.rb
+- spec/wukong/widget/reducers/moments_spec.rb
+- spec/wukong/widget/reducers/sort_spec.rb
+- spec/wukong/widget/serializers_spec.rb
 - spec/wukong/widget/sink_spec.rb
 - spec/wukong/widget/source_spec.rb
-- spec/wukong/widget/stringifier_spec.rb
-- spec/wukong/workflow/command_spec.rb
+- spec/wukong/wukong_spec.rb
 has_rdoc: