wukong 3.0.0.pre → 3.0.0.pre2

data/lib/wukong/widget/reducers/count.rb

@@ -0,0 +1,61 @@
+require_relative("accumulator")
+
+module Wukong
+  class Processor
+
+    # A processor which counts the total number of its input records.
+    #
+    # On it's own, this widget is really just a poor man's `wc -l`.
+    # It's really intended to serve as a superclass for more complex
+    # accumulators.
+    #
+    # @example Count the total number of input records on the command-line.
+    #
+    #   $ wc -l input
+    #   283 input
+    #   $ cat input | wu-local count
+    #   283
+    class Count < Accumulator
+
+      # The total size of the input recors.
+      attr_accessor :size
+
+      # Initializes the count to 0.
+      def setup
+        super()
+        @size = 0
+      end
+
+      # Accumulate a `record` by incrmenting the total size.
+      #
+      # @param [Object] record
+      def accumulate record
+        self.size += 1
+      end
+
+      # Keeps all records in the same group so that one count is
+      # emitted at the end.
+      #
+      # Overriding this method and returning different keys for
+      # different records is the beginning of constructing a "group
+      # by" type widget.
+      #
+      # @param [Object] record
+      # @return [:__first__group__]
+      # @see Group
+      def get_key record
+        :__first_group__
+      end
+
+      # Yields the total size.
+      #
+      # @yield [size]
+      # @yieldparam [Integer] size
+      def finalize
+        yield self.size
+      end
+
+      register
+    end
+  end
+end

data/lib/wukong/widget/reducers/group.rb

@@ -0,0 +1,85 @@
+require_relative("../utils")
+require_relative("count")
+
+module Wukong
+  class Processor
+
+    # Groups sorted input records and emits each group with a count.
+    #
+    # Allows you to use several ways of extracting the key that
+    # defines the group.
+    #
+    # **Note:** The input records must be previously sorted by the
+    # same key used for grouping in order to ensure that groups are
+    # not split up.
+    #
+    # @example Group simple string values on the command-line.
+    #
+    #   $ cat input
+    #   apple
+    #   cat
+    #   banana
+    #   apple
+    #   ...
+    #   $ cat input | wu-local sort | wu-local group
+    #   apple	4
+    #   banana	2
+    #   cat	5
+    #   ...
+    #
+    # @example Group using a nested key within a JSON string on the command-line
+    #
+    #   $ cat input
+    #   {"id": 1, "word": "apple" }
+    #   {"id": 2, "word": "cat"   }
+    #   {"id": 3, "word": "banana"}
+    #   ...
+    #   $ cat input | wu-local sort --on==word | wu-local group --by=word
+    #   apple	4
+    #   banana	2
+    #   cat	5
+    #   ...
+    #
+    # A group fits nicely at the end of a dataflow.  Since it requires
+    # a sort, it is blocking.
+    #
+    # @example Using a group at the end of a dataflow
+    #
+    #   Wukong.dataflow(:makes_groups) do
+    #     ... | sort(on: 'field') | group(by: 'field')
+    #   end
+    #
+    # @see Sort
+    class Group < Count
+
+      include DynamicGet
+      field :by, Whatever
+
+      # Get the key which defines the group for this `record`.
+      #
+      # @param [Object] record
+      # @return [Object]
+      def get_key(record)
+        get(self.by, record)
+      end
+
+      # Reset the size counter for new group.
+      #
+      # @param [Object] record
+      def start record
+        self.size = 0
+      end
+      
+      # Yields the current group along with its size
+      #
+      # @yield [key, size]
+      # @yieldparam [Object] key the key defining the group
+      # @yieldparam [Integer] size the size of the group
+      def finalize
+        yield [key, size].map(&:to_s).join("\t")
+      end
+
+      register
+    end
+  end
+end

data/lib/wukong/widget/reducers/group_concat.rb

@@ -0,0 +1,70 @@
+require_relative("group")
+
+module Wukong
+  class Processor
+
+    # Concatenates the elements of a group, yielding the group key,
+    # the count, and its members.
+    #
+    # @example Concatenating elements of a group on the command-line.
+    #
+    #   $ cat input
+    #   {"id": 1, "parent_id": 4}
+    #   {"id": 2, "parent_id": 3}
+    #   {"id": 3, "parent_id": 3}
+    #   ...
+    #   $ cat input | wu-local group_concat --by=parent_id
+    #   4	1	{"id": 1, "parent_id": 4}
+    #   3	2	{"id": 2, "parent_id": 3}	{"id": 3, "parent_id": 3}
+    #   ...
+    #
+    # GroupConcat takes all the same options as Group.
+    #
+    # @see Group
+    class GroupConcat < Group
+
+      # The members of the current group.
+      attr_accessor :members
+
+      # Initializes the empty members array.
+      def setup
+        super()
+        @members = []
+      end
+
+      # Initializes the empty members array.
+      #
+      # @param [Object] record
+      def start record
+        super(record)
+        self.members = []
+      end
+
+      # Accumulate each record, adding it to the current members.
+      #
+      # @param [Object] record
+      def accumulate record
+        super(record)
+        self.members << record
+      end
+      
+      # Yields the group, including its key, its size, and each
+      # member.
+      #
+      # @yield [key, size, *members]
+      # @yieldparam [Object] key the key defining the group
+      # @yieldparam [Integer] size the number of members in the group
+      # @yieldparam [Array<Object>] the members of the group
+      def finalize
+        group = [key, size]
+        group.concat(members)
+        yield group.map(&:to_s).join("\t")
+      end
+
+      register
+    end
+  end
+end
+
+
+    

data/lib/wukong/widget/reducers/moments.rb

@@ -0,0 +1,72 @@
+require_relative("group")
+
+module Wukong
+  class Processor
+    class Moments < Group
+
+      field :group_by, Whatever
+
+      attr_accessor :measurements
+
+      field :of,      Array,    :default => []
+      field :std_dev, :boolean, :default => true
+
+      def get_key record
+        super(record) unless (self.group_by || self.by)
+        get(self.group_by || self.by, record)
+      end
+
+      def receive_of o
+        @of = case o
+        when String then o.split(',')
+        when Array  then o
+        else []
+        end
+      end
+
+      def start record
+        super(record)
+        @measurements = {}.tap do |m|
+          self.of.each do |property|
+            m[property] = []
+          end
+        end
+      end
+
+      def accumulate record
+        super(record)
+        self.of.each do |property|
+          if raw = get(property, record)
+            self.measurements[property] << (raw.to_f rescue next)
+          end
+        end
+      end
+      
+      def results
+        {}.tap do |r|
+          measurements.each_pair do |property, values|
+            r[property] = {}
+            next if values.empty?
+            count               = values.size.to_f
+            r[property][:count] = count.to_i
+            
+            mean               = values.inject(0.0) { |sum, value| sum += value } / count
+            r[property][:mean] = mean
+            if std_dev
+              variance    = values.inject(0.0) { |sum, value| diff = (value - mean) ; sum += diff * diff } / count
+              std         = Math.sqrt(variance)
+              r[property][:std_dev] = std
+            end
+          end
+        end
+      end
+
+      def finalize
+        yield({:group => key, :count => size}.merge(:results => results))
+      end
+      
+      register
+    end
+  end
+end
+

data/lib/wukong/widget/reducers/sort.rb

@@ -0,0 +1,130 @@
+require_relative("accumulator")
+require_relative("../utils")
+
+module Wukong
+  class Processor
+
+    # Sorts input records.
+    #
+    # For many use cases you're better off using native tools like
+    # `/bin/sort` because they are faster and already do what you
+    # need.
+    #
+    # @example When /bin/sort is more than enough on the command-line
+    #
+    #   $ cat input
+    #   1	apple
+    #   2	banana
+    #   3	cat
+    #   4	banana
+    #   ...
+    #   $ cat input | sort -k2
+    #   1	apple
+    #   2	banana
+    #   4	banana
+    #   3	cat
+    #   ...
+    #
+    # Other times, you need something that can introspect more on its
+    # input:
+    #
+    # @example When you may prefer the sort widget on the command-line
+    #
+    #   $ cat input
+    #   {"id": 1, "word": "apple" }
+    #   {"id": 2, "word": "cat"   }
+    #   {"id": 3, "word": "banana"}
+    #   ...
+    #   $ cat input | wu-local sort --on word
+    #   {"id": 1, "word": "apple" }
+    #   {"id": 3, "word": "banana"}    
+    #   {"id": 2, "word": "cat"   }
+    #   ...
+    #
+    # The sort widget is useful for modeling Hadoop jobs, but don't
+    # forget that [Hadoop does its own
+    # sorting](http://hadoop.apache.org/docs/r0.20.2/mapred_tutorial.html#Sort),
+    # so the sort widget doesn't belong in your map/reduce jobs.
+    #
+    # @example The wrong way to model a Hadoop map/reduce job
+    #
+    #   Wukong.dataflow(:my_incorrect_job_dataflow) do
+    #     parse | extract(part: 'country') | sort | group
+    #   end
+    #
+    # @example The right way to model a Hadoop map/reduce job
+    #
+    #   Wukong.dataflow(:mapper) do
+    #     parse | extract(part: 'country')
+    #   end
+    #   
+    #   Wukong.dataflow(:reducer) do
+    #     group
+    #   end
+    class Sort < Accumulator
+      
+      include DynamicGet
+      field :on,        Whatever
+      field :reverse,   :boolean, :default => false
+      field :numeric,   :boolean, :default => false
+
+      # Intializes the array of records that will hold all the values.
+      def setup
+        super()
+        @records = []
+      end
+
+      # Keeps all the records in a single group so they can be sorted.
+      #
+      # @param [Object] record
+      # @return [:__first__group__]
+      def get_key(record)
+        :__first_group__
+      end
+
+      # Stores the `record` for later sorting.
+      #
+      # @param [Object] record
+      def accumulate record
+        @records << record
+      end
+      
+      # Sorts all the stored records and yields in one sorted
+      # according to the field in the right order.
+      #
+      # @yield [record] each record in correct sort order
+      # @yeildparam [Object] record
+      def finalize
+        sorted = @records.sort{ |x, y| compare(x, y) }
+        sorted.reverse! if reverse
+        sorted.each{ |record| yield record }
+      end
+
+      # Extracts the sortable part of the input `record`.
+      #
+      # @param [Object] record
+      # @return [Object] the part of the record to sort on
+      def sortable(record)
+        get(self.on, record)
+      end
+
+      # Compare records `x` and `y` using their sortable parts.
+      #
+      # Will use numeric sorting when asked.
+      #
+      # @param [Object] x
+      # @param [Object] y
+      # @return [1,0,-1] depends on which of x or y is considered greater
+      def compare(x, y)
+        a = (sortable(x) or return -1) 
+        b = (sortable(y) or return  1)
+        if numeric
+          a = a.to_f ; b = b.to_f
+        end
+        a <=> b
+      end
+
+      register
+    end
+  end
+end

data/lib/wukong/widget/serializers.rb

@@ -0,0 +1,287 @@
+module Wukong
+  class Processor
+
+    # An empty parent class for all Serializers to subclass.
+    class Serializer < Processor
+    end
+
+    # A widget for serializing inputs to JSON.
+    #
+    # @example Serializing to JSON at the end of a data flow
+    #
+    #   Wukong.dataflow(:emits_json) do
+    #     ... | to_json
+    #   end
+    #
+    # @see FromJson
+    class ToJson < Serializer
+      # Yields the input `record` serialized as JSON.
+      #
+      # @param [Object] record
+      # @yield [json] the serialized json output
+      # @yieldparam [String] json
+      def process(record)
+        begin
+          json = ::MultiJson.dump(record)
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield json
+      end
+      register
+    end
+
+    # A widget for deserializing inputs from JSON.
+    #
+    # @example Deserializing from JSON at the beginning of a data flow
+    #
+    #   Wukong.dataflow(:consumes_json) do
+    #     from_json | ...
+    #   end
+    #
+    # @see ToJson
+    class FromJson < Serializer
+      # Yields the input `record` deserialized from JSON.
+      #
+      # @param [String] json
+      # @yield [obj] the deserialized object
+      # @yieldparam [Object] obj
+      def process(json)
+        begin
+          obj = ::MultiJson.load(json)
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield obj
+      end
+      register
+    end
+
+    # A widget for serializing inputs to TSV.
+    #
+    # @example Serializing to TSV at the end of a data flow
+    #
+    #   Wukong.dataflow(:emits_tsv) do
+    #     ... | to_tsv
+    #   end
+    #
+    # @see FromTsv
+    class ToTsv < Serializer
+      # Yields the input `record` serialized as TSV.
+      #
+      # @param [Object] record
+      # @yield [tsv] the serialized TSV output
+      # @yieldparam [String] tsv
+      def process(record)
+        begin
+          tsv = record.map(&:to_s).join("\t")
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield tsv
+      end
+      register
+    end
+
+    # A widget for deserializing inputs from TSV.
+    #
+    # @example Deserializing from TSV at the beginning of a data flow
+    #
+    #   Wukong.dataflow(:consumes_tsv) do
+    #     from_tsv | ...
+    #   end
+    #
+    # @see ToTsv
+    class FromTsv < Serializer
+      # Yields the input `record` deserialized from TSV.
+      #
+      # @param [String] tsv
+      # @yield [obj] the deserialized object
+      # @yieldparam [Object] obj
+      def process(tsv)
+        begin
+          record = tsv.split(/\t/)
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield record
+      end
+      register
+    end
+
+    # A widget for serializing inputs to CSV.
+    #
+    # @example Serializing to CSV at the end of a data flow
+    #
+    #   Wukong.dataflow(:emits_csv) do
+    #     ... | to_csv
+    #   end
+    #
+    # @see FromCsv
+    class ToCsv < Serializer
+      # Yields the input `record` serialized as CSV.
+      #
+      # @param [Object] record
+      # @yield [csv] the serialized CSV output
+      # @yieldparam [String] csv
+      def process(record)
+        begin
+          csv = record.map(&:to_s).join(",")
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield csv
+      end
+      register
+    end
+
+    # A widget for deserializing inputs from CSV.
+    #
+    # @example Deserializing from CSV at the beginning of a data flow
+    #
+    #   Wukong.dataflow(:consumes_csv) do
+    #     from_csv | ...
+    #   end
+    #
+    # @see ToCsv
+    class FromCsv < Serializer
+      # Yields the input `record` deserialized from CSV.
+      #
+      # @param [String] csv
+      # @yield [obj] the deserialized object
+      # @yieldparam [Object] obj
+      def process(csv)
+        begin
+          record = csv.split(/,/)
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield record
+      end
+      register
+    end
+
+    # A widget for serializing inputs to a delimited format.
+    #
+    # @example Serializing to a delimited format at the end of a data flow
+    #
+    #   Wukong.dataflow(:emits_delimited) do
+    #     ... | to_delimited(delimiter: "--")
+    #   end
+    #
+    # @see FromDelimited
+    class ToDelimited < Serializer
+      field :delimiter, String, :default => "\t"
+      # Yields the input `record` serialized in a delimited format..
+      #
+      # @param [Object] record
+      # @yield [delimited] the serialized delimited output
+      # @yieldparam [String] delimited
+      def process(record)
+        begin
+          delimited = record.map(&:to_s).join(delimiter)
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield delimited
+      end
+      register
+    end
+
+    # A widget for deserializing inputs from a delimited format.
+    #
+    # @example Deserializing from a delimited format at the beginning of a data flow
+    #
+    #   Wukong.dataflow(:consumes_delimited) do
+    #     from_delimited(delimiter: "--") | ...
+    #   end
+    #
+    # @see ToDelimited
+    class FromDelimited < Serializer
+      field :delimiter, String, :default => "\t"
+      # Yields the input `record` deserialized from a delimited format.
+      #
+      # @param [String] delimited
+      # @yield [obj] the deserialized object
+      # @yieldparam [Object] obj
+      def process(delimited)
+        begin
+          record = delimited.split(delimiter)
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield record
+      end
+      register
+    end
+
+    # A widget for serializing inputs to Ruby's `inspect` format.
+    #
+    # @example Serializing to Ruby's inspect format at the end of a data flow
+    #
+    #   Wukong.dataflow(:emits_inspected) do
+    #     ... | to_inspect
+    #   end
+    class ToInspect < Serializer
+      # Yields the input record(s) passed through Ruby's `inspect`.
+      #
+      # @param [Array<Object>]
+      # @yield [inspected]
+      # @yieldparam [String] inspected
+      def process(*args)
+        yield args.size == 1 ? args.first.inspect : args.inspect
+      end
+      register
+    end
+
+    # A widget for pretty printing input records.
+    #
+    # @example Pretty printing JSON on the command-line
+    #
+    #   $ cat input
+    #   {"id": 1, "word": "apple" }
+    #   $ cat input | wu-local pretty
+    #   {
+    #     "id":2,
+    #     "parent_id":3
+    #   }
+    class Pretty < Serializer
+      # Pretty print `record` if we can.
+      #
+      # @param [Object] record
+      # @yield [pretty]
+      # @yieldparam [String] pretty the pretty-printed record
+      def process record
+        if record.is_a?(String) && record =~ /^\s*\{/
+          yield pretty_json(record)
+        else
+          yield record.to_s
+        end
+      end
+
+      # Attempt to pretty-print the given `json`, returning the
+      # original on an error.
+      #
+      # @param [String] json ugly JSON
+      # @return [String] prettier JSON
+      def pretty_json json
+        begin
+          MultiJson.dump(MultiJson.load(json), :pretty => true)
+        rescue => e
+          json
+        end
+      end
+      
+      register
+    end
+
+  end
+end