wukong 3.0.0 → 3.0.1

data/README.md

@@ -19,8 +19,8 @@ Here is a list of various other projects which you may also want to
 peruse when trying to understand the full Wukong experience:
 
 * <a href="http://github.com/infochimps-labs/wukong-hadoop">wukong-hadoop</a>: Run Wukong processors as mappers and reducers within the Hadoop framework.  Model Hadoop jobs locally before you run them.
-* <a href="http://github.com/infochimps-labs/wukong-storm>wukong-storm</a>: Run Wukong processors within the Storm framework.  Model flows locally before you run them.
-* <a href="http://github.com/infochimps-labs/wukong-load>wukong-load</a>: Load the output data from your local Wukong jobs and flows into a variety of different data stores.
+* <a href="http://github.com/infochimps-labs/wukong-storm">wukong-storm</a>: Run Wukong processors within the Storm framework.  Model flows locally before you run them.
+* <a href="http://github.com/infochimps-labs/wukong-load">wukong-load</a>: Load the output data from your local Wukong jobs and flows into a variety of different data stores.
 * <a href="http://github.com/infochimps-labs/wonderdog">wonderdog</a>: Connect Wukong processors running within Hadoop to Elasticsearch as either a source or sink for data.
 * <a href="http://github.com/infochimps-labs/wukong-deploy">wukong-deploy</a>: Orchestrate Wukong and other wu-tools together to support an application running on the Infochimps Platform.
 

data/lib/wukong.rb

@@ -35,7 +35,7 @@ module Wukong
   
   add_shortcut_method_for(:processor, ProcessorBuilder)
   add_shortcut_method_for(:dataflow,  DataflowBuilder)
-
+  
 end
 
 # Alias module name for shorter namespaces
@@ -44,4 +44,6 @@ Wu = Wukong
 require_relative 'wukong/widgets'
 require_relative 'wukong/local'
 
-
+module Wukong
+  BUILTINS = Set.new(Wukong.registry.show.keys)
+end

data/lib/wukong/dataflow.rb

@@ -33,6 +33,11 @@ module Wukong
   
   class Dataflow < Hanuman::Graph
 
+    def self.description desc=nil
+      @description = desc if desc
+      @description
+    end
+    
     def has_input?(stage)
       links.any?{ |link| link.into == stage }
     end

data/lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb

@@ -44,6 +44,13 @@ module Wukong
         @driver ||= UnitTestDriver.new(processor, settings)
       end
 
+      # No need to load commandline arguments when we are testing
+      # There are other mechanisms for passing them in, plus
+      # RSpec goes into an infinite loop if you load a spec file
+      # from within a spec file
+      def load_args
+      end
+
       # Do nothing.  This prevents control flow within the Ruby
       # interpreter from staying within this runner, as it would
       # ordinarly do for `wu-local`.

data/lib/wukong/version.rb

@@ -1,4 +1,3 @@
 module Wukong
-  # The current version of Wukong.
-  VERSION = '3.0.0'
+  VERSION = '3.0.1'
 end

data/lib/wukong/widget/reducers.rb

@@ -5,3 +5,4 @@ require_relative("reducers/group")
 require_relative("reducers/group_concat")
 require_relative("reducers/moments")
 require_relative("reducers/bin")
+require_relative("reducers/uniq")

data/lib/wukong/widget/reducers/uniq.rb

@@ -0,0 +1,90 @@
+require_relative("accumulator")
+
+module Wukong
+  class Processor
+
+    # A processor which emits only unique records from its input.
+    # It's intended to work just like `uniq`.
+    #
+    # @example Emit unique elements from the input (like `uniq`).
+    #
+    #   $ uniq input
+    #   apple
+    #   banana
+    #   pear
+    #   $ cat input | wu-local uniq
+    #   apple
+    #   banana
+    #   pear
+    #
+    # @example Emit unique elements from the input with counts (like `uniq -c`).
+    #
+    #   $ uniq -c input
+    #        3 apple
+    #        2 banana
+    #        3 pear
+    #   $ cat input | wu-local uniq --count --to=tsv
+    #   apple	3
+    #   banana	5
+    #   pear	8
+    
+    class Uniq < Accumulator
+
+      field :count, :boolean, doc: "Emit a count for each group of input records", default: false
+
+      description <<EOF
+This processor uniq's its inputs.
+
+    $ uniq input
+    apple
+    banana
+    pear
+    $ cat input | wu-local uniq
+    apple
+    banana
+    pear
+  
+And it can count as well:
+  
+    $ uniq -c input
+         3 apple
+         2 banana
+         3 pear
+    $ cat input | wu-local uniq --count --to=tsv
+    apple	3
+    banana	5
+    pear	8
+EOF
+
+      # The total size of the input recors.
+      attr_accessor :size
+
+      # Initializes the count to 0.
+      def setup
+        super()
+        @size = 0
+      end
+
+      # Accumulate a `record` by incrmenting the total size.
+      #
+      # @param [Object] record
+      def accumulate record
+        self.size += 1
+      end
+
+      # Yields the total size.
+      #
+      # @yield [size]
+      # @yieldparam [Integer] size
+      def finalize
+        if count
+          yield [key, self.size]
+        else
+          yield key
+        end
+      end
+
+      register
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wukong
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-20 00:00:00.000000000 Z
+date: 2013-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: configliere
@@ -337,6 +337,7 @@ files:
 - lib/wukong/widget/reducers/group_concat.rb
 - lib/wukong/widget/reducers/moments.rb
 - lib/wukong/widget/reducers/sort.rb
+- lib/wukong/widget/reducers/uniq.rb
 - lib/wukong/widget/serializers.rb
 - lib/wukong/widget/utils.rb
 - lib/wukong/widgets.rb
@@ -399,15 +400,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 719389029987495852
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 719389029987495852
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use