ul-wukong 4.1.0

data/lib/wukong/widget/reducers/count.rb

@@ -0,0 +1,73 @@
+require_relative("accumulator")
+
+module Wukong
+  class Processor
+
+    # A processor which counts the total number of its input records.
+    #
+    # On it's own, this widget is really just a poor man's `wc -l`.
+    # It's really intended to serve as a superclass for more complex
+    # accumulators.
+    #
+    # @example Count the total number of input records on the command-line.
+    #
+    #   $ wc -l input
+    #   283 input
+    #   $ cat input | wu-local count
+    #   283
+    class Count < Accumulator
+
+      description <<EOF
+This processor counts the number of input records it receives.
+
+  $ wc -l input
+  283 input
+  $ cat input | wu-local count
+  283
+
+This processor will not output any records until it receives its final
+input record.
+EOF
+
+      # The total size of the input recors.
+      attr_accessor :size
+
+      # Initializes the count to 0.
+      def setup
+        super()
+        @size = 0
+      end
+
+      # Accumulate a `record` by incrmenting the total size.
+      #
+      # @param [Object] record
+      def accumulate record
+        self.size += 1
+      end
+
+      # Keeps all records in the same group so that one count is
+      # emitted at the end.
+      #
+      # Overriding this method and returning different keys for
+      # different records is the beginning of constructing a "group
+      # by" type widget.
+      #
+      # @param [Object] record
+      # @return [:__first__group__]
+      # @see Group
+      def get_key record
+        :__first_group__
+      end
+
+      # Yields the total size.
+      #
+      # @yield [size]
+      # @yieldparam [Integer] size
+      def finalize
+        yield self.size
+      end
+
+      register
+    end
+  end
+end

data/lib/wukong/widget/reducers/group.rb

@@ -0,0 +1,128 @@
+require_relative("../utils")
+require_relative("count")
+
+module Wukong
+  class Processor
+
+    # Groups sorted input records and emits each group with a count.
+    #
+    # Allows you to use several ways of extracting the key that
+    # defines the group.
+    #
+    # **Note:** The input records must be previously sorted by the
+    # same key used for grouping in order to ensure that groups are
+    # not split up.
+    #
+    # @example Group simple string values on the command-line.
+    #
+    #   $ cat input
+    #   apple
+    #   cat
+    #   banana
+    #   apple
+    #   ...
+    #   $ cat input | wu-local sort | wu-local group --to=tsv
+    #   apple	4
+    #   banana	2
+    #   cat	5
+    #   ...
+    #
+    # @example Group using a nested key within a JSON string on the command-line
+    #
+    #   $ cat input
+    #   {"id": 1, "word": "apple" }
+    #   {"id": 2, "word": "cat"   }
+    #   {"id": 3, "word": "banana"}
+    #   ...
+    #   $ cat input | wu-local sort --on=word | wu-local group --by=word --to=tsv
+    #   apple	4
+    #   banana	2
+    #   cat	5
+    #   ...
+    #
+    # A group fits nicely at the end of a dataflow.  Since it requires
+    # a sort, it is blocking.
+    #
+    # @example Using a group at the end of a dataflow
+    #
+    #   Wukong.dataflow(:makes_groups) do
+    #     ... | sort(on: 'field') | group(by: 'field') | to_tsv
+    #   end
+    #
+    # @see Sort
+    class Group < Count
+
+      description <<EOF
+This processor groups consecutive input records that share the same
+"group key".  There are several ways to extract this group key from a
+record.
+
+NOTE: The input records must be previously sorted by the
+same key used for grouping in order to ensure that groups are
+not split up.
+
+By default the input records themselves are used as their own group
+keys, allowing to count identical values, a la `uniq -c`:
+
+  $ cat input
+  apple
+  cat
+  banana
+  apple
+  ...
+
+  $ cat input | wu-local sort | wu-local group --to=tsv
+  apple	4
+  banana	2
+  cat	5
+  ...
+
+You can also group by some part of in input record:
+
+  $ cat input
+  {"id": 1, "word": "apple" }
+  {"id": 2, "word": "cat"   }
+  {"id": 3, "word": "banana"}
+  ...
+
+  $ cat input | wu-local sort --on==word | wu-local group --by=word --to=tsv
+  apple	4
+  banana	2
+  cat	5
+  ...
+
+This processor will not produce any output for a given group until it
+sees the last record of that group.
+EOF
+
+      include DynamicGet
+      field :by, Whatever, :doc => "Part of the record to group by"
+
+      # Get the key which defines the group for this `record`.
+      #
+      # @param [Object] record
+      # @return [Object]
+      def get_key(record)
+        get(self.by, record)
+      end
+
+      # Reset the size counter for new group.
+      #
+      # @param [Object] record
+      def start record
+        self.size = 0
+      end
+      
+      # Yields the current group along with its size
+      #
+      # @yield [key, size]
+      # @yieldparam [Object] key the key defining the group
+      # @yieldparam [Integer] size the size of the group
+      def finalize
+        yield [key, size]
+      end
+
+      register
+    end
+  end
+end

data/lib/wukong/widget/reducers/group_concat.rb

@@ -0,0 +1,98 @@
+require_relative("group")
+
+module Wukong
+  class Processor
+
+    # Concatenates the elements of a group, yielding the group key,
+    # the count, and its members.
+    #
+    # @example Concatenating elements of a group on the command-line.
+    #
+    #   $ cat input
+    #   {"id": 1, "parent_id": 4}
+    #   {"id": 2, "parent_id": 3}
+    #   {"id": 3, "parent_id": 3}
+    #   ...
+    #   $ cat input | wu-local group_concat --by=parent_id --to=tsv
+    #   4	1	{"id": 1, "parent_id": 4}
+    #   3	2	{"id": 2, "parent_id": 3}	{"id": 3, "parent_id": 3}
+    #   ...
+    #
+    # GroupConcat takes all the same options as Group.
+    #
+    # @see Group
+    class GroupConcat < Group
+
+      description <<EOF
+This processor concatenates records of a consecutive group of records
+into a single record.
+
+  $ cat input
+  {"id": 1, "parent_id": 4}
+  {"id": 2, "parent_id": 3}
+  {"id": 3, "parent_id": 3}
+  ...
+
+  $ cat input | wu-local group_concat --by=parent_id --to=tsv
+  4	1	{"id": 1, "parent_id": 4}
+  3	2	{"id": 2, "parent_id": 3}	{"id": 3, "parent_id": 3}
+  ...
+
+Each output record consists of tab-separated fields in the following
+order:
+
+  1) The key defining the group of input records in this output record
+  2) The number of input records in the group
+  3) Each input record in the group
+  ...
+
+This processor will not produce any output for a given group until it
+sees the last record of that group. See the documentation for the
+'group' processor for more information.
+EOF
+
+      # The members of the current group.
+      attr_accessor :members
+
+      # Initializes the empty members array.
+      def setup
+        super()
+        @members = []
+      end
+
+      # Initializes the empty members array.
+      #
+      # @param [Object] record
+      def start record
+        super(record)
+        self.members = []
+      end
+
+      # Accumulate each record, adding it to the current members.
+      #
+      # @param [Object] record
+      def accumulate record
+        super(record)
+        self.members << record
+      end
+      
+      # Yields the group, including its key, its size, and each
+      # member.
+      #
+      # @yield [key, size, *members]
+      # @yieldparam [Object] key the key defining the group
+      # @yieldparam [Integer] size the number of members in the group
+      # @yieldparam [Array<Object>] the members of the group
+      def finalize
+        group = [key, size]
+        group.concat(members)
+        yield group
+      end
+
+      register
+    end
+  end
+end
+
+
+    

data/lib/wukong/widget/reducers/improver.rb

@@ -0,0 +1,71 @@
+module Wukong
+  class Processor
+
+    # A base widget for building more complex improver widgets.
+    class Improver < Processor
+
+      # The current group of records.
+      attr_accessor :group
+
+      # Sets up this improver by defining an initial key (with a
+      # value that is unlikely to be found in real data) and calling
+      # `#zero` with no record.
+      def setup
+        @key = :__first_group__
+        zero
+      end
+
+      def recordize record
+        record.split("\t")
+      end
+      
+      #
+      # All kinds of assumptions here,
+      # record is tab-delimited and the
+      # first field is a name of a function
+      # to call
+      #
+      def get_function record
+        record.first
+      end
+      
+      # Processes the `record`.
+      def process(record)
+        fields = recordize(record)
+        func   = get_function(fields)
+        case func
+        when 'zero' then
+          yield zero
+        when 'accumulate' then
+          accumulate(fields[1..-1])
+        when 'improve' then
+          yield improve(fields[1], self.group)
+          self.group = []
+        else
+          raise NoMethodError, "undefined method #{func} for Improver"
+        end
+        STDOUT.flush # WHY? Because.
+      end
+
+      # Starts accumulation for a new key. Return what you would
+      # with no improvements.
+      def zero
+        self.group = []
+      end
+
+      # Accumulates another +record+.
+      #
+      # @param [Object] record
+      def accumulate record
+        self.group << record
+      end
+
+      # Improve prev with group
+      #
+      #
+      def improve prev, group
+      end
+      
+    end
+  end
+end

data/lib/wukong/widget/reducers/join_xml.rb

@@ -0,0 +1,37 @@
+module Wukong
+  class Processor
+
+    # Joins XML input data based on a root tag.
+    class JoinXML < Processor
+
+      field :root, String, default: 'xml', doc: "Name of the root XML element"
+
+      def setup
+        @lines = []
+      end
+
+      def process line
+        if match = terminator.match(line)
+          if match.end(0) == line.size
+            @lines << line
+          else
+            @lines << line[0...match.end(0)]
+          end
+          yield @lines.join("\n")
+          @lines = []
+          @lines << line[match.end(0)..-1] unless match.end(0) == line.size
+        else
+          @lines << line
+        end
+      end
+    
+      def terminator
+        %r{<\s*/\s*#{root}\s*>}i
+      end
+      
+      register :join_xml
+    end
+  end
+end
+
+

data/lib/wukong/widget/reducers/moments.rb

@@ -0,0 +1,72 @@
+require_relative("group")
+
+module Wukong
+  class Processor
+    class Moments < Group
+
+      field :group_by, Whatever, :doc => "Part of the record to group by"
+
+      attr_accessor :measurements
+
+      field :of,         Array,    :default => [],   :doc => "Parts of the record to measure moments of"
+      field :no_std_dev, :boolean, :doc => "Don't compute standard deviations"
+
+      def get_key record
+        super(record) unless (self.group_by || self.by)
+        get(self.group_by || self.by, record)
+      end
+
+      def receive_of o
+        @of = case o
+        when String then o.split(',')
+        when Array  then o
+        else []
+        end
+      end
+
+      def start record
+        super(record)
+        @measurements = {}.tap do |m|
+          self.of.each do |property|
+            m[property] = []
+          end
+        end
+      end
+
+      def accumulate record
+        super(record)
+        self.of.each do |property|
+          if raw = get(property, record)
+            self.measurements[property] << (raw.to_f rescue next)
+          end
+        end
+      end
+      
+      def results
+        {}.tap do |r|
+          measurements.each_pair do |property, values|
+            r[property] = {}
+            next if values.empty?
+            count               = values.size.to_f
+            r[property][:count] = count.to_i
+            
+            mean               = values.inject(0.0) { |sum, value| sum += value } / count
+            r[property][:mean] = mean
+            unless no_std_dev
+              variance    = values.inject(0.0) { |sum, value| diff = (value - mean) ; sum += diff * diff } / count
+              std         = Math.sqrt(variance)
+              r[property][:std_dev] = std
+            end
+          end
+        end
+      end
+
+      def finalize
+        yield({:group => key, :count => size}.merge(:results => results))
+      end
+      
+      register
+    end
+  end
+end
+