wukong 3.0.0.pre3 → 3.0.0

data/lib/wukong/widget/reducers/group.rb

@@ -21,7 +21,7 @@ module Wukong
     #   banana
     #   apple
     #   ...
-    #   $ cat input | wu-local sort | wu-local group
+    #   $ cat input | wu-local sort | wu-local group --to=tsv
     #   apple	4
     #   banana	2
     #   cat	5
@@ -34,7 +34,7 @@ module Wukong
     #   {"id": 2, "word": "cat"   }
     #   {"id": 3, "word": "banana"}
     #   ...
-    #   $ cat input | wu-local sort --on==word | wu-local group --by=word
+    #   $ cat input | wu-local sort --on=word | wu-local group --by=word --to=tsv
     #   apple	4
     #   banana	2
     #   cat	5
@@ -46,14 +46,57 @@ module Wukong
     # @example Using a group at the end of a dataflow
     #
     #   Wukong.dataflow(:makes_groups) do
-    #     ... | sort(on: 'field') | group(by: 'field')
+    #     ... | sort(on: 'field') | group(by: 'field') | to_tsv
     #   end
     #
     # @see Sort
     class Group < Count
 
+      description <<EOF
+This processor groups consecutive input records that share the same
+"group key".  There are several ways to extract this group key from a
+record.
+
+NOTE: The input records must be previously sorted by the
+same key used for grouping in order to ensure that groups are
+not split up.
+
+By default the input records themselves are used as their own group
+keys, allowing to count identical values, a la `uniq -c`:
+
+  $ cat input
+  apple
+  cat
+  banana
+  apple
+  ...
+
+  $ cat input | wu-local sort | wu-local group --to=tsv
+  apple	4
+  banana	2
+  cat	5
+  ...
+
+You can also group by some part of in input record:
+
+  $ cat input
+  {"id": 1, "word": "apple" }
+  {"id": 2, "word": "cat"   }
+  {"id": 3, "word": "banana"}
+  ...
+
+  $ cat input | wu-local sort --on==word | wu-local group --by=word --to=tsv
+  apple	4
+  banana	2
+  cat	5
+  ...
+
+This processor will not produce any output for a given group until it
+sees the last record of that group.
+EOF
+
       include DynamicGet
-      field :by, Whatever
+      field :by, Whatever, :doc => "Part of the record to group by"
 
       # Get the key which defines the group for this `record`.
       #
@@ -76,7 +119,7 @@ module Wukong
       # @yieldparam [Object] key the key defining the group
       # @yieldparam [Integer] size the size of the group
       def finalize
-        yield [key, size].map(&:to_s).join("\t")
+        yield [key, size]
       end
 
       register

data/lib/wukong/widget/reducers/group_concat.rb

@@ -13,7 +13,7 @@ module Wukong
     #   {"id": 2, "parent_id": 3}
     #   {"id": 3, "parent_id": 3}
     #   ...
-    #   $ cat input | wu-local group_concat --by=parent_id
+    #   $ cat input | wu-local group_concat --by=parent_id --to=tsv
     #   4	1	{"id": 1, "parent_id": 4}
     #   3	2	{"id": 2, "parent_id": 3}	{"id": 3, "parent_id": 3}
     #   ...
@@ -23,6 +23,34 @@ module Wukong
     # @see Group
     class GroupConcat < Group
 
+      description <<EOF
+This processor concatenates records of a consecutive group of records
+into a single record.
+
+  $ cat input
+  {"id": 1, "parent_id": 4}
+  {"id": 2, "parent_id": 3}
+  {"id": 3, "parent_id": 3}
+  ...
+
+  $ cat input | wu-local group_concat --by=parent_id --to=tsv
+  4	1	{"id": 1, "parent_id": 4}
+  3	2	{"id": 2, "parent_id": 3}	{"id": 3, "parent_id": 3}
+  ...
+
+Each output record consists of tab-separated fields in the following
+order:
+
+  1) The key defining the group of input records in this output record
+  2) The number of input records in the group
+  3) Each input record in the group
+  ...
+
+This processor will not produce any output for a given group until it
+sees the last record of that group. See the documentation for the
+'group' processor for more information.
+EOF
+
       # The members of the current group.
       attr_accessor :members
 
@@ -58,7 +86,7 @@ module Wukong
       def finalize
         group = [key, size]
         group.concat(members)
-        yield group.map(&:to_s).join("\t")
+        yield group
       end
 
       register

data/lib/wukong/widget/reducers/moments.rb

@@ -4,12 +4,12 @@ module Wukong
   class Processor
     class Moments < Group
 
-      field :group_by, Whatever
+      field :group_by, Whatever, :doc => "Part of the record to group by"
 
       attr_accessor :measurements
 
-      field :of,      Array,    :default => []
-      field :std_dev, :boolean, :default => true
+      field :of,         Array,    :default => [],   :doc => "Parts of the record to measure moments of"
+      field :no_std_dev, :boolean, :doc => "Don't compute standard deviations"
 
       def get_key record
         super(record) unless (self.group_by || self.by)
@@ -52,7 +52,7 @@ module Wukong
             
             mean               = values.inject(0.0) { |sum, value| sum += value } / count
             r[property][:mean] = mean
-            if std_dev
+            unless no_std_dev
               variance    = values.inject(0.0) { |sum, value| diff = (value - mean) ; sum += diff * diff } / count
               std         = Math.sqrt(variance)
               r[property][:std_dev] = std

data/lib/wukong/widget/reducers/sort.rb

@@ -62,11 +62,61 @@ module Wukong
     #     group
     #   end
     class Sort < Accumulator
+
+      description <<EOF
+This processor sorts input records alphabetically or numerically based
+on their value or the value of one of their parts.
+
+NOTE: For many use cases you're better off using native tools like
+`/bin/sort` because they are faster and already do what you
+need.
+
+You can sort simple inputs
+
+  $ cat input
+  1	apple
+  2	banana
+  3	cat
+  4	banana
+  ...
+
+  $ cat input | wu-local sort --on=2
+  1	apple
+  2	banana
+  4	banana
+  3	cat
+  ...
+
+as well as complicated ones
+
+  $ cat input
+  {"id": 1, "word": "apple" }
+  {"id": 2, "word": "cat"   }
+  {"id": 3, "word": "banana"}
+  ...
+
+  $ cat input | wu-local sort --on=word
+  {"id": 1, "word": "apple" }
+  {"id": 3, "word": "banana"}    
+  {"id": 2, "word": "cat"   }
+  ...
+
+You can also sort in --reverse or using --numeric order instead of
+lexical.
+
+The sort widget is useful for modeling Hadoop jobs, but don't
+forget that [Hadoop does its own
+sorting](http://hadoop.apache.org/docs/r0.20.2/mapred_tutorial.html#Sort),
+so the sort widget doesn't belong in your map/reduce jobs.
+
+This processor will not produce any output till it has received all
+its input records.
+EOF
       
       include DynamicGet
-      field :on,        Whatever
-      field :reverse,   :boolean, :default => false
-      field :numeric,   :boolean, :default => false
+      field :on,        Whatever, :doc => "Part of the record to sort on"
+      field :reverse,   :boolean, :default => false, :doc => "Sort in reverse order"
+      field :numeric,   :boolean, :default => false, :doc => "Sort numerically instead of lexically"
 
       # Intializes the array of records that will hold all the values.
       def setup

data/lib/wukong/widget/serializers.rb

@@ -1,14 +1,14 @@
 module Wukong
   class Processor
 
-    SerializerError = Class.new(StandardError)
+    SerializerError = Class.new(Error)
 
     class Serializer < Processor
-      field :on_error, String, default: 'log'
+      field :on_error, String, default: 'log', :doc => "Action to take upon an error, either 'log' or 'notify'"
 
       def handle_error(record, err)
         case on_error
-        when 'log'    then log.warn "Bad record: #{record}"
+        when 'log'    then log.warn "#{err.class}: #{err.message}"
         when 'notify' then notify('error', record: record, error: err)
         end          
       end
@@ -25,7 +25,14 @@ module Wukong
     #
     # @see FromJson
     class ToJson < Serializer
-      field :pretty, :boolean, default: false
+
+      description <<EOF
+Turns input records into JSON strings.
+
+Pretty print input with the --pretty flag.
+EOF
+      
+      field :pretty, :boolean, default: false, :doc => "Pretty-print output"
 
       # Yields the input `record` serialized as JSON.
       #
@@ -56,6 +63,16 @@ module Wukong
     #
     # @see ToJson
     class FromJson < Serializer
+
+      description <<EOF
+Parse JSON input records into native Ruby objects.
+
+  $ cat input.json
+  {"hi": "there"}
+  $ cat input.json | wu-local from_json
+  {"hi"=>"there"}
+EOF
+      
       # Yields the input `record` deserialized from JSON.
       #
       # @param [String] json
@@ -196,7 +213,7 @@ module Wukong
     #
     # @see FromDelimited
     class ToDelimited < Serializer
-      field :delimiter, String, :default => "\t"
+      field :delimiter, String, :default => "\t", :doc => "Delimiter to use between fields in a record"
       # Yields the input `record` serialized in a delimited format..
       #
       # @param [Object] record
@@ -224,7 +241,7 @@ module Wukong
     #
     # @see ToDelimited
     class FromDelimited < Serializer
-      field :delimiter, String, :default => "\t"
+      field :delimiter, String, :default => "\t", :doc => "Delimiter to use between fields in a record"
       # Yields the input `record` deserialized from a delimited format.
       #
       # @param [String] delimited
@@ -260,18 +277,26 @@ module Wukong
       end
       register
     end
-    
+
+    # A widget for turning a record into an instance of some class.
+    # The class must provide a "class method" `receive` which accepts
+    # a Hash argument.
     class Recordize < Serializer
-      field :model, Whatever
+      field :model, Whatever, :doc => "Model class to turn records into"
 
+      # Turn the given `record` into an instance of the class named
+      # with the `model` field.
+      #
+      # @param [Hash, #to_wire] record
+      # @return [Object]
       def process(record)
         wire_format = record.try(:to_wire) || record
-        raise SerializerError.new("Record must be in hash format to be recordized") unless wire_format.is_a?(Hash)
+        raise SerializerError.new("Can only recordize a Hash-like record") unless wire_format.is_a?(Hash)
         yield model.receive(wire_format)
-      rescue => e        
-        handle_error(record, e)  
+      rescue => e
+        handle_error(record, e)
       end
       register
-    end    
+    end
   end
 end

data/lib/wukong/widget/utils.rb

@@ -7,7 +7,7 @@ module Wukong
 
     # :nodoc:
     def self.included klass
-      klass.send(:field, :separator, String,   :default => "\t")
+      klass.send(:field, :separator, String, :default => "\t", :doc => "The default separator between fields on a single line")
     end
 
     # :nodoc:

data/spec/spec_helper.rb

@@ -2,12 +2,30 @@ require 'wukong'
 require 'wukong/spec_helpers'
 require_relative './support/shared_examples_for_shortcuts'
 require_relative './support/shared_examples_for_builders'
-require_relative './support/integration_helper'
 require_relative './support/shared_context_for_reducers'
 
 RSpec.configure do |config|
+  
   config.mock_with :rspec
+  
   include Wukong::SpecHelpers
-  include Wukong::Local::IntegrationHelper
+  def root
+    @root ||= Pathname.new(File.expand_path('../..', __FILE__))
+  end
+
+  def local_runner *args
+    runner(Wukong::Local::LocalRunner, 'wu-local', *args)
+  end
+
+  def generic_runner *args
+    runner(Wukong::Runner, 'wu-generic', *args)
+  end
+
+  def wu_local *args
+    command('wu-local', *args)
+  end
+  
+  # FIXME Why is this here?
   config.treat_symbols_as_metadata_keys_with_true_values = true
+  
 end

data/spec/wukong/driver_spec.rb

@@ -0,0 +1,2 @@
+require 'spec_helper'
+

data/spec/wukong/local/runner_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe Wukong::Local::LocalRunner do
+  before { EM.stub!(:run) }
+
+  describe "choosing a processor name" do
+
+    it "raises an error without any arguments" do
+      expect { local_runner() }.to raise_error(Wukong::Error, /must provide.*processor.*run.*argument/i)
+    end
+    
+    it "raises an error when passed the name of a processor that isn't registered" do
+      expect { local_runner('some_proc_that_dont_exit') }.to raise_error(Wukong::Error, /no such processor.*some_proc.*/i)
+    end
+    
+    it "accepts an explicit --run argument" do
+      local_runner('--run=identity').processor.should == 'identity'
+    end
+    
+    it "accepts a registered processor name from the first argument" do
+      local_runner('identity').processor.should == 'identity'
+    end
+    
+    it "accepts a registerd processor name from the the basename of the first file argument" do
+      local_runner(examples_dir('string_reverser.rb')).processor.should == 'string_reverser'
+    end
+  end
+
+  describe "uses a" do
+    it "StdioDriver by default" do
+      local_runner('identity').driver.should == Wukong::Local::StdioDriver
+    end
+
+    it "TCPDriver when given a --port argument" do
+      local_runner('identity','--tcp_port=6000').driver.should == Wukong::Local::TCPDriver
+    end
+    
+  end
+  
+end

data/spec/wukong/local_spec.rb

@@ -0,0 +1,6 @@
+require 'spec_helper'
+
+describe Wukong::Local do
+  it_behaves_like 'a plugin'
+end
+

data/spec/wukong/logger_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+describe Wukong::Logging do
+
+  let(:loggable) { Class.new.class_eval { include Wukong::Logging } }
+  let(:model)    { Class.new.class_eval { include Gorillib::Model } }
+  
+  describe "a class including Wukong::Logging" do
+    subject { loggable }
+
+    it { should respond_to(:log) }
+
+    describe "has instances that" do
+      let(:loggable_instance) { loggable.new      }
+      subject                 { loggable_instance }
+      
+      it   { should respond_to(:log)   }
+      describe "have an instance method #log that" do
+        let(:log) { loggable_instance.log }
+        subject   { log                   }
+
+        it { should respond_to(:debug) }
+        it { should respond_to(:info)  }
+        it { should respond_to(:warn)  }
+      end
+    end
+
+    describe "has subclasses" do
+      let(:child)  { Class.new(loggable)    }
+      
+      describe "with instances that" do
+        subject { child.new }
+        
+        it      { should respond_to(:log)   }
+      end
+    end
+  end
+
+  describe "a class including Gorillib::Model and then Wukong::Logging" do
+    let(:loggable_model) { model.class_eval { include Wukong::Logging } }
+    subject              { loggable_model                               }
+    
+    describe "has fields that" do
+      subject { loggable_model.fields }
+      it      { should include(:log) }
+    end
+  end
+end
+