wukong 3.0.0.pre3 → 3.0.0

data/lib/wukong/version.rb

@@ -1,4 +1,4 @@
 module Wukong
   # The current version of Wukong.
-  VERSION = '3.0.0.pre3'
+  VERSION = '3.0.0'
 end

data/lib/wukong/widget/filters.rb

@@ -5,6 +5,10 @@ module Wukong
     # criterion.
     class Filter < Processor
 
+      description <<EOF
+A processor which filters input records according to some criterion.
+EOF
+
       # Process a `record` by yielding it only if it should be
       # selected by this filter.
       #
@@ -61,6 +65,7 @@ module Wukong
     # @see Filter
     # @see Null
     class Identity < Filter
+      description "This processor passes all records unmodified."
       register
     end
     
@@ -84,6 +89,9 @@ module Wukong
     # @see Filter
     # @see All
     class Null < Filter
+
+      description "This processor acts as a filter which passes no records at all."
+      
       # Prevents any records from passing because it always returns
       # `false`.
       #
@@ -117,8 +125,21 @@ module Wukong
     # @see NotRegexpFilter
     class RegexpFilter < Filter
 
-      # The regular expression to use to match records.
-      field :match, Regexp
+      description <<EOF
+This processor only passes records which match against a given regular
+expression.
+
+  $ cat input
+  apple
+  banana
+  cat
+  $ cat input | wu-local regexp --match='^a'
+  apple
+
+If no --match argument is given, all records will be passed.
+EOF
+
+      field :match, Regexp, :doc => "Regular expression to match against"
       
       # Selects a `record` only if it matches this widget's `match`
       # field.
@@ -154,6 +175,22 @@ module Wukong
     # @see Filter
     # @see NotRegexpFilter
     class NotRegexpFilter < RegexpFilter
+
+      description <<EOF
+This processor only passes records which fail to match against a given
+regular expression.
+
+  $ cat input
+  apple
+  banana
+  cat
+  $ cat input | wu-local not_regexp --match='^a'
+  banana
+  cat
+
+If no --match argument is given, all records will be passed.
+EOF
+      
       # Select a `record` only if it <b>doesn't</b> match this
       # widget's `match` field.
       #
@@ -189,8 +226,22 @@ module Wukong
     # @see Filter
     class Limit < Filter
 
-      # The maximum number of records to let pass.
-      field :max, Integer, :default => Float::INFINITY
+      description <<EOF
+This processor passes a certain number of records and then stops
+passing any, acting as a limit.
+
+  $ cat input
+  1
+  2
+  3
+  $ cat input | wu-local limit --max=2
+  1
+  2
+
+If no --max argument is given, all records will be passed.
+EOF
+
+      field :max, Integer, :doc => "Maximum number of records to let pass"
 
       # The current record count.
       attr_accessor :count
@@ -206,7 +257,7 @@ module Wukong
       # @param [Object] record
       # @return [true, false]
       def select?(record)
-        keep = @count < max
+        keep = (max ? @count < max : true)
         @count += 1
         keep
       end
@@ -236,9 +287,23 @@ module Wukong
     # @see Limit
     class Sample < Filter
 
-      # The fraction of records to let pass.  Must be between 0.0 and
-      # 10.0
-      field :fraction, Float, :default => 1.0
+      description <<EOF
+This processor will pass input records with a certain frequency,
+acting as a random sampler.
+
+  $ cat input
+  1
+  2
+  3
+  4
+  $ cat input | wu-local sample --fraction=0.5
+  1
+  4
+
+If no --fraction is given, all records will be passed.
+EOF
+
+      field :fraction, Float, :default => 1.0, :doc => "Fraction of records to let pass.  Must be between 0 and 1.0"
 
       # Selects a `record` randomly, with a probability given the the
       # `fraction` for this widget.
@@ -307,5 +372,66 @@ module Wukong
       register
     end
 
+    # Emit only the first `n` records.
+    #
+    # @see Filter
+    class Head < Filter
+
+      field :n, Integer, :default => 10, :doc => "Number of records to let pass"
+
+      # The current record count.
+      attr_accessor :count
+
+      # Initializes the record count to zero.
+      def setup
+        self.count = 0
+      end
+
+      # Select a record only if we're below the maximum number of
+      # records.
+      #
+      # @param [Object] record
+      # @return [true, false]
+      def select?(record)
+        keep = @count < n
+        @count += 1
+        keep
+      end
+      register
+    end
+
+    # Skip the first `n` records.
+    #
+    # Works slightly differently than the UNIX `tail` command which
+    # prints the last `n` records.  This notion is less useful in a
+    # streaming context, so think of this filter as the equivalent of
+    # `tail -n+`.
+    #
+    # @see Filter
+    class Tail < Filter
+      
+      field :n, Integer, :default => 0, :doc => "Number of records to skip before letting records pass"
+
+      # The current record count
+      attr_accessor :count
+
+      # Initializes the record count to zero.
+      def setup
+        self.count = 0
+      end
+
+      # Select a record only if we've already skipped the first `n`
+      # records.
+      #
+      # @param [Object]
+      # @return [true, false]
+      def select?(record)
+        keep = (@count >= n)
+        @count += 1
+        keep
+      end
+      register
+    end
+    
   end
 end

data/lib/wukong/widget/processors.rb

@@ -22,8 +22,29 @@ module Wukong
     #     ... | logger
     #   end
     class Logger < Processor
-      # The level to use for logging.
-      field :level, Symbol, :default => :info
+      field :level, Symbol, :default => :info, :doc => "Log level priority"
+
+      description <<EOF
+This processor passes all input records unmodified, making a log
+statement on each one.
+
+  $ cat input
+  1
+  2
+  3
+  $ cat input | wu-local logger
+  INFO 2013-01-04 17:10:59 [Logger              ] -- 1
+  INFO 2013-01-04 17:10:59 [Logger              ] -- 2
+  INFO 2013-01-04 17:10:59 [Logger              ] -- 3
+
+You can set the priority level of the log messages with the --level
+flag.
+
+  $ cat input | wu-local logger --level=debug
+  DEBUG 2013-01-04 17:10:59 [Logger              ] -- 1
+  DEBUG 2013-01-04 17:10:59 [Logger              ] -- 2
+  DEBUG 2013-01-04 17:10:59 [Logger              ] -- 3
+EOF
 
       # Process a given `record` by logging it.
       #
@@ -100,8 +121,44 @@ module Wukong
     class Extract < Processor
       include DynamicGet
 
-      # The part to extract.
-      field :part, Whatever, :default => nil
+      description <<EOF
+This processor will pass extracted parts of input records.
+
+It can be used to extract a field from a delimited input
+
+  $ cat input
+  snap	crackle	pop
+  a	b	c
+  $ cat input | wu-local extract --part=2
+  crackle
+  b
+
+The default separator is a tab character but you can specify this as
+well
+
+  $ cat input
+  snap,crackle,pop
+  a,b,c
+  $ cat input | wu-local extract --part=2 --separator=,
+  crackle
+  b
+
+It can also be used on JSON records, even those with nested fields
+
+  $ cat input
+  {"id": 1, {"data": {"text": "hi there"}}
+  {"id": 2, {"data": {"text": "goodbye"}}
+  $ cat input | wu-local extract --part=id
+  1
+  2
+  $ cat input | wu-local extract --part=data.text
+  hi there
+  goodbye
+
+If no --part argument is given, the original record will be yielded.
+EOF
+
+      field :part, Whatever, :default => nil, :doc => "Part of the record to extract"
 
       # Extract a `part` of a `record`.
       #
@@ -115,7 +172,9 @@ module Wukong
     end
     
     class Topic < Processor
-      field :topic, Symbol
+      
+      field :topic, Symbol, :doc => "Topic to label the record with"
+      
       def process(record)
         yield perform_action(record)
       end

data/lib/wukong/widget/reducers/bin.rb

@@ -12,7 +12,7 @@ module Wukong
     #   0.03480
     #   0.74418
     #   ...
-    #   $ cat input | wu-local bin
+    #   $ cat input | wu-local bin --to=tsv
     #
     #   0.02935	0.12638500000000003	7
     #   0.12638500000000003	0.22342000000000004	11
@@ -20,7 +20,7 @@ module Wukong
     #
     # @example Control how the bins are defined and displayed
     #
-    #   $ cat input | wu-local bin --min=0.0 --max=1.0 --num_bins=10 --precision=1
+    #   $ cat input | wu-local bin --min=0.0 --max=1.0 --num_bins=10 --precision=1 --to=tsv
     #   0.0	0.1	10.0
     #   0.1	0.2	12.0
     #   0.2	0.3	8.0
@@ -28,7 +28,7 @@ module Wukong
     #
     # @example Include an additional column of normalized (fractional) counts
     # 
-    #   $ cat input | wu-local bin --min=0.0 --max=1.0 --num_bins=10 --precision=1 --normalize
+    #   $ cat input | wu-local bin --min=0.0 --max=1.0 --num_bins=10 --precision=1 --normalize --to=tsv
     #   0.0	0.1	10.0	0.3
     #   0.1	0.2	12.0	0.36
     #   0.2	0.3	8.0	0.24
@@ -36,7 +36,7 @@ module Wukong
     #
     # @example Make a log-log histogram
     #
-    #   $ cat input | wu-local bin --log_bins --log_counts
+    #   $ cat input | wu-local bin --log_bins --log_counts --to=tsv
     #   1.000	3.162	1.099
     #   3.162	10.000	1.946
     #   10.000	31.623	3.045
@@ -48,29 +48,79 @@ module Wukong
     # @example Use the bin at the end of a dataflow
     #
     #   Wukong.processor(:bins_at_end) do
-    #     ... | extract(part: 'age') | bin(num_bins: 10)
+    #     ... | extract(part: 'age') | bin(num_bins: 10) | to_tsv
     #   end
     #
     # @see Accumulator
     # @see Extract
     class Bin < Accumulator
+
+      description <<EOF
+This processor can be used to create a set of bins defining the
+frequency distribution of the input records (or some part of each
+input record).
+
+Here's a simple example:
+
+  $ cat input.dat
+  1
+  2
+  3
+  ...
+  100
+
+  $ cat input.dat | wu-local bin --to=tsv
+  1.000	10.900	10.000
+  10.900	20.800	10.000
+  20.800	30.700	10.000
+  30.700	40.600	10.000
+  ...
+  90.100	100.000	10.000
+
+By default, all the input values are included and the number of bins
+used corresponds to the square root of the number of input values.
+You can customize the domain for the distribution, the number of bins,
+or the explicit bin edges themselves, via the --min, --max,
+--num_bins, and --edges flags.
+
+You can control the display of numbers with the --format_string and
+--precision options.
+
+  $ cat input.dat | wu-local bin --num_bins=4 --min=0 --max=100 --precision=0 --to=tsv
+  0.0	25	24
+  25	50	25
+  50	75	25
+  75	100	26
+
+You can use the --log_bins, --log_counts, and --base options to use
+logarithmically spaced bins or logarithmic counts within each bin to
+the given base.
+
+You can also normalize the distribution using the --normalize option.
+
+  $ cat input.dat | wu-local bin --num_bins=4 --log_bins --normalize --to=tsv
+  1.000	3.162	3.000	0.030
+  3.162	10.000	7.000	0.070
+  10.000	31.623	21.000	0.210
+  31.623	100.000	69.000	0.690
+EOF
       
-      field :num_bins,    Integer
-      field :edges,       Array
-      field :min,         Float
-      field :max,         Float
+      field :num_bins,    Integer, :doc => "Number of bins to use"
+      field :edges,       Array,   :doc => "Number of edges to use"
+      field :min,         Float,   :doc => "Smallest bin starting point"
+      field :max,         Float,   :doc => "Largest bin ending point"
 
-      field :format_string, String
-      field :precision,     Integer, :default => 3
+      field :format_string, String,  :doc => "Format string used when printing numerical values"
+      field :precision,     Integer, :doc => "Precision used when printing numerical values", :default => 3
 
       include DynamicGet
-      field :by,          Whatever
+      field :by,          Whatever, :doc => "Bin the values extracted by this label"
 
-      field :log_bins,    :boolean, :default => false
-      field :log_counts,  :boolean, :default => false
-      field :base,        Float,    :default => Math::E
+      field :log_bins,    :boolean, :default => false,   :doc => "Use logarithmically spaced bins"
+      field :log_counts,  :boolean, :default => false,   :doc => "Use logarithmic bin counts"
+      field :base,        Float,    :default => Math::E, :doc => "Base for logarithms"
       
-      field :normalize,   :boolean, :default => false
+      field :normalize,   :boolean, :default => false, :doc => "Normalize bin counts so they sum to 1.0"
 
       # The accumulated values
       attr_accessor :values
@@ -148,7 +198,7 @@ module Wukong
           if normalize && total_count > 0
             bin << log_count_if_necessary((count.to_f / total_count.to_f))
           end
-          yield bin.map { |n| format(n) }.join("\t")
+          yield bin.map { |n| format(n) }
         end
       end
 
@@ -169,7 +219,7 @@ module Wukong
         when format_string
           format_string % n
         when n == 0.0
-          0.0
+          '0.0'
         when n.abs > 1000 || n.abs < 0.001
           "%#{precision}.#{precision}E" % n
         else

data/lib/wukong/widget/reducers/count.rb

@@ -17,6 +17,18 @@ module Wukong
     #   283
     class Count < Accumulator
 
+      description <<EOF
+This processor counts the number of input records it receives.
+
+  $ wc -l input
+  283 input
+  $ cat input | wu-local count
+  283
+
+This processor will not output any records until it receives its final
+input record.
+EOF
+
       # The total size of the input recors.
       attr_accessor :size