ul-wukong 4.1.0

data/lib/wukong/widget/reducers/sort.rb

@@ -0,0 +1,180 @@
+require_relative("accumulator")
+require_relative("../utils")
+
+module Wukong
+  class Processor
+
+    # Sorts input records.
+    #
+    # For many use cases you're better off using native tools like
+    # `/bin/sort` because they are faster and already do what you
+    # need.
+    #
+    # @example When /bin/sort is more than enough on the command-line
+    #
+    #   $ cat input
+    #   1	apple
+    #   2	banana
+    #   3	cat
+    #   4	banana
+    #   ...
+    #   $ cat input | sort -k2
+    #   1	apple
+    #   2	banana
+    #   4	banana
+    #   3	cat
+    #   ...
+    #
+    # Other times, you need something that can introspect more on its
+    # input:
+    #
+    # @example When you may prefer the sort widget on the command-line
+    #
+    #   $ cat input
+    #   {"id": 1, "word": "apple" }
+    #   {"id": 2, "word": "cat"   }
+    #   {"id": 3, "word": "banana"}
+    #   ...
+    #   $ cat input | wu-local sort --on word
+    #   {"id": 1, "word": "apple" }
+    #   {"id": 3, "word": "banana"}    
+    #   {"id": 2, "word": "cat"   }
+    #   ...
+    #
+    # The sort widget is useful for modeling Hadoop jobs, but don't
+    # forget that [Hadoop does its own
+    # sorting](http://hadoop.apache.org/docs/r0.20.2/mapred_tutorial.html#Sort),
+    # so the sort widget doesn't belong in your map/reduce jobs.
+    #
+    # @example The wrong way to model a Hadoop map/reduce job
+    #
+    #   Wukong.dataflow(:my_incorrect_job_dataflow) do
+    #     parse | extract(part: 'country') | sort | group
+    #   end
+    #
+    # @example The right way to model a Hadoop map/reduce job
+    #
+    #   Wukong.dataflow(:mapper) do
+    #     parse | extract(part: 'country')
+    #   end
+    #   
+    #   Wukong.dataflow(:reducer) do
+    #     group
+    #   end
+    class Sort < Accumulator
+
+      description <<EOF
+This processor sorts input records alphabetically or numerically based
+on their value or the value of one of their parts.
+
+NOTE: For many use cases you're better off using native tools like
+`/bin/sort` because they are faster and already do what you
+need.
+
+You can sort simple inputs
+
+  $ cat input
+  1	apple
+  2	banana
+  3	cat
+  4	banana
+  ...
+
+  $ cat input | wu-local sort --on=2
+  1	apple
+  2	banana
+  4	banana
+  3	cat
+  ...
+
+as well as complicated ones
+
+  $ cat input
+  {"id": 1, "word": "apple" }
+  {"id": 2, "word": "cat"   }
+  {"id": 3, "word": "banana"}
+  ...
+
+  $ cat input | wu-local sort --on=word
+  {"id": 1, "word": "apple" }
+  {"id": 3, "word": "banana"}    
+  {"id": 2, "word": "cat"   }
+  ...
+
+You can also sort in --reverse or using --numeric order instead of
+lexical.
+
+The sort widget is useful for modeling Hadoop jobs, but don't
+forget that [Hadoop does its own
+sorting](http://hadoop.apache.org/docs/r0.20.2/mapred_tutorial.html#Sort),
+so the sort widget doesn't belong in your map/reduce jobs.
+
+This processor will not produce any output till it has received all
+its input records.
+EOF
+      
+      include DynamicGet
+      field :on,        Whatever, :doc => "Part of the record to sort on"
+      field :reverse,   :boolean, :default => false, :doc => "Sort in reverse order"
+      field :numeric,   :boolean, :default => false, :doc => "Sort numerically instead of lexically"
+
+      # Intializes the array of records that will hold all the values.
+      def setup
+        super()
+        @records = []
+      end
+
+      # Keeps all the records in a single group so they can be sorted.
+      #
+      # @param [Object] record
+      # @return [:__first__group__]
+      def get_key(record)
+        :__first_group__
+      end
+
+      # Stores the `record` for later sorting.
+      #
+      # @param [Object] record
+      def accumulate record
+        @records << record
+      end
+      
+      # Sorts all the stored records and yields in one sorted
+      # according to the field in the right order.
+      #
+      # @yield [record] each record in correct sort order
+      # @yeildparam [Object] record
+      def finalize
+        sorted = @records.sort{ |x, y| compare(x, y) }
+        sorted.reverse! if reverse
+        sorted.each{ |record| yield record }
+      end
+
+      # Extracts the sortable part of the input `record`.
+      #
+      # @param [Object] record
+      # @return [Object] the part of the record to sort on
+      def sortable(record)
+        get(self.on, record)
+      end
+
+      # Compare records `x` and `y` using their sortable parts.
+      #
+      # Will use numeric sorting when asked.
+      #
+      # @param [Object] x
+      # @param [Object] y
+      # @return [1,0,-1] depends on which of x or y is considered greater
+      def compare(x, y)
+        a = (sortable(x) or return -1) 
+        b = (sortable(y) or return  1)
+        if numeric
+          a = a.to_f ; b = b.to_f
+        end
+        a <=> b
+      end
+
+      register
+    end
+  end
+end

data/lib/wukong/widget/reducers/uniq.rb

@@ -0,0 +1,91 @@
+require_relative("accumulator")
+
+module Wukong
+  class Processor
+
+    # A processor which emits only unique records from its input.
+    # It's intended to work just like `uniq`.
+    #
+    # @example Emit unique elements from the input (like `uniq`).
+    #
+    #   $ uniq input
+    #   apple
+    #   banana
+    #   pear
+    #   $ cat input | wu-local uniq
+    #   apple
+    #   banana
+    #   pear
+    #
+    # @example Emit unique elements from the input with counts (like `uniq -c`).
+    #
+    #   $ uniq -c input
+    #        3 apple
+    #        2 banana
+    #        3 pear
+    #   $ cat input | wu-local uniq --count --to=tsv
+    #   apple	3
+    #   banana	5
+    #   pear	8
+    
+    class Uniq < Accumulator
+
+      field :count, :boolean, doc: "Emit a count for each group of input records", default: false
+
+      description <<EOF
+This processor uniq's its inputs.
+
+    $ uniq input
+    apple
+    banana
+    pear
+    $ cat input | wu-local uniq
+    apple
+    banana
+    pear
+  
+And it can count as well:
+  
+    $ uniq -c input
+         3 apple
+         2 banana
+         3 pear
+    $ cat input | wu-local uniq --count --to=tsv
+    apple	3
+    banana	5
+    pear	8
+EOF
+
+      # The total size of the input recors.
+      attr_accessor :size
+
+      # Initializes the count to 0.
+      def setup
+        super()
+        @size = 0
+      end
+
+      # Accumulate a `record` by incrmenting the total size.
+      #
+      # @param [Object] record
+      def accumulate record
+        self.size += 1
+      end
+
+      # Yields the total size.
+      #
+      # @yield [size]
+      # @yieldparam [Integer] size
+      def finalize
+        return if key == :__first_group__
+        if count
+          yield [key, self.size]
+        else
+          yield key
+        end
+      end
+
+      register
+    end
+  end
+end

data/lib/wukong/widget/serializers.rb

@@ -0,0 +1,317 @@
+module Wukong
+  class Processor
+
+    SerializerError = Class.new(Error)
+
+    class Serializer < Processor
+
+      def handle_error(record, err)
+        return if err.class == Errno::EPIPE
+        log.error "#{err.class}: #{err.message}"
+        err.backtrace.each { |line| log.debug(line) }
+      end
+
+    end
+
+    # A widget for serializing inputs to JSON.
+    #
+    # @example Serializing to JSON at the end of a data flow
+    #
+    #   Wukong.dataflow(:emits_json) do
+    #     ... | to_json
+    #   end
+    #
+    # @see FromJson
+    class ToJson < Serializer
+
+      description <<EOF
+Turns input records into JSON strings.
+
+Pretty print input with the --pretty flag.
+EOF
+      
+      field :pretty, :boolean, default: false, :doc => "Pretty-print output"
+
+      # Yields the input `record` serialized as JSON.
+      #
+      # @param [Object] record
+      # @yield [json] the serialized json output
+      # @yieldparam [String] json
+      def process(record)
+        raise SerializerError.new("Cannot serialize: <nil>") if record.nil?
+        if record.respond_to?(:to_json) && !record.is_a?(Hash) # We only want to invoke to_json if it has been explicitly defined
+          json = record.to_json(pretty: pretty)
+        else
+          json = MultiJson.dump(record.try(:to_wire) || record, pretty: pretty)
+        end
+        yield json
+      rescue => e
+        handle_error(record, e)
+      end
+      register
+    end
+
+    # A widget for deserializing inputs from JSON.
+    #
+    # @example Deserializing from JSON at the beginning of a data flow
+    #
+    #   Wukong.dataflow(:consumes_json) do
+    #     from_json | ...
+    #   end
+    #
+    # @see ToJson
+    class FromJson < Serializer
+
+      description <<EOF
+Parse JSON input records into native Ruby objects.
+
+  $ cat input.json
+  {"hi": "there"}
+  $ cat input.json | wu-local from_json
+  {"hi"=>"there"}
+EOF
+      
+      # Yields the input `record` deserialized from JSON.
+      #
+      # @param [String] json
+      # @yield [obj] the deserialized object
+      # @yieldparam [Object] obj
+      def process(record)
+        if record.respond_to?(:from_json)
+          obj = record.from_json
+        else
+          obj = MultiJson.load(record)
+        end
+        yield obj
+      rescue => e
+        handle_error(record, e)       
+      end
+      register
+    end
+
+    # A widget for serializing inputs to TSV.
+    #
+    # @example Serializing to TSV at the end of a data flow
+    #
+    #   Wukong.dataflow(:emits_tsv) do
+    #     ... | to_tsv
+    #   end
+    #
+    # @see FromTsv
+    class ToTsv < Serializer
+      # Yields the input `record` serialized as TSV.
+      #
+      # @param [Object] record
+      # @yield [tsv] the serialized TSV output
+      # @yieldparam [String] tsv
+      def process(record)
+        if record.respond_to?(:to_tsv)
+          tsv = record.to_tsv
+        else         
+          wire_format = record.try(:to_wire) || record
+          raise SerializerError.new("Record must be in Array format to be serialized as TSV") unless wire_format.respond_to?(:map)
+          tsv = wire_format.map(&:to_s).join("\t")
+        end
+        yield tsv
+      rescue => e
+        handle_error(record, e)
+      end
+      register
+    end
+
+    # A widget for deserializing inputs from TSV.
+    #
+    # @example Deserializing from TSV at the beginning of a data flow
+    #
+    #   Wukong.dataflow(:consumes_tsv) do
+    #     from_tsv | ...
+    #   end
+    #
+    # @see ToTsv
+    class FromTsv < Serializer
+      # Yields the input `record` deserialized from TSV.
+      #
+      # @param [String] tsv
+      # @yield [obj] the deserialized object
+      # @yieldparam [Object] obj
+      def process(record)
+        if record.respond_to?(:from_tsv)
+          obj = record.from_tsv
+        else
+          obj = record.split(/\t/, -1)
+        end
+        yield obj
+      rescue => e        
+        handle_error(record, e)
+      end
+      register
+    end
+
+    # A widget for serializing inputs to CSV.
+    #
+    # @example Serializing to CSV at the end of a data flow
+    #
+    #   Wukong.dataflow(:emits_csv) do
+    #     ... | to_csv
+    #   end
+    #
+    # @see FromCsv
+    class ToCsv < Serializer
+      # Yields the input `record` serialized as CSV.
+      #
+      # @param [Object] record
+      # @yield [csv] the serialized CSV output
+      # @yieldparam [String] csv
+      def process(record)
+        begin
+          csv = record.map(&:to_s).join(",")
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield csv
+      end
+      register
+    end
+
+    # A widget for deserializing inputs from CSV.
+    #
+    # @example Deserializing from CSV at the beginning of a data flow
+    #
+    #   Wukong.dataflow(:consumes_csv) do
+    #     from_csv | ...
+    #   end
+    #
+    # @see ToCsv
+    class FromCsv < Serializer
+      # Yields the input `record` deserialized from CSV.
+      #
+      # @param [String] csv
+      # @yield [obj] the deserialized object
+      # @yieldparam [Object] obj
+      def process(csv)
+        begin
+          record = csv.split(/,/)
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield record
+      end
+      register
+    end
+
+    # A widget for serializing inputs to a delimited format.
+    #
+    # @example Serializing to a delimited format at the end of a data flow
+    #
+    #   Wukong.dataflow(:emits_delimited) do
+    #     ... | to_delimited(delimiter: "--")
+    #   end
+    #
+    # @see FromDelimited
+    class ToDelimited < Serializer
+      field :delimiter, String, :default => "\t", :doc => "Delimiter to use between fields in a record"
+      # Yields the input `record` serialized in a delimited format..
+      #
+      # @param [Object] record
+      # @yield [delimited] the serialized delimited output
+      # @yieldparam [String] delimited
+      def process(record)
+        begin
+          delimited = record.map(&:to_s).join(delimiter)
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield delimited
+      end
+      register
+    end
+
+    # A widget for deserializing inputs from a delimited format.
+    #
+    # @example Deserializing from a delimited format at the beginning of a data flow
+    #
+    #   Wukong.dataflow(:consumes_delimited) do
+    #     from_delimited(delimiter: "--") | ...
+    #   end
+    #
+    # @see ToDelimited
+    class FromDelimited < Serializer
+      field :delimiter, String, :default => "\t", :doc => "Delimiter to use between fields in a record"
+      # Yields the input `record` deserialized from a delimited format.
+      #
+      # @param [String] delimited
+      # @yield [obj] the deserialized object
+      # @yieldparam [Object] obj
+      def process(delimited)
+        begin
+          record = delimited.split(delimiter)
+        rescue => e
+          # FIXME -- should we log here or what?
+          return
+        end
+        yield record
+      end
+      register
+    end
+
+    # A widget for serializing inputs to Ruby's `inspect` format.
+    #
+    # @example Serializing to Ruby's inspect format at the end of a data flow
+    #
+    #   Wukong.dataflow(:emits_inspected) do
+    #     ... | to_inspect
+    #   end
+    class ToInspect < Serializer
+      # Yields the input record(s) passed through Ruby's `inspect`.
+      #
+      # @param [Array<Object>]
+      # @yield [inspected]
+      # @yieldparam [String] inspected
+      def process(record)
+        yield record.inspect
+      end
+      register
+    end
+
+    # A widget for turning a record into an instance of some class.
+    # The class must provide a "class method" `receive` which accepts
+    # a Hash argument.
+    class Recordize < Serializer
+      field :model, Whatever, :doc => "Model class to turn records into"
+
+      # Turn the given `record` into an instance of the class named
+      # with the `model` field.
+      #
+      # @param [Hash, #to_wire] record
+      # @return [Object]
+      def process(record)
+        wire_format = record.try(:to_wire) || record
+        raise SerializerError.new("Can only recordize a Hash-like record") unless wire_format.is_a?(Hash)
+        klass = model_class_for(wire_format)
+        if klass
+          yield klass.receive(wire_format)
+        else
+          log.error("No default model class and no explicit model for: #{wire_format.inspect}")
+        end
+      rescue => e
+        handle_error(record, e)
+      end
+
+      def model_class_for(record)
+        if explicit_type = (record[:_type] || record["_type"])
+          begin
+            return explicit_type.constantize
+          rescue NameError => e
+            log.warn("Could not find a class for <#{explicit_type}>")
+          end
+        end
+        return model if model
+      end
+      
+      register
+    end
+  end
+end