wukong 3.0.0.pre → 3.0.0.pre2

data/old/wukong/streamer/counting_reducer.rb

@@ -1,25 +0,0 @@
-module Wukong
-  module Streamer
-    #
-    # Emit each unique key and the count of its occurrences
-    #
-    class CountingReducer < Wukong::Streamer::AccumulatingReducer
-
-      # reset the counter to zero
-      def start! *args
-        @count = 0
-      end
-
-      # record one more for this key
-      def accumulate *vals
-        @count += 1
-      end
-
-      # emit each key field and the count, tab-separated.
-      def finalize
-        yield [key, @count]
-      end
-    end
-
-  end
-end

data/old/wukong/streamer/filter.rb

@@ -1,20 +0,0 @@
-module Wukong
-  module Streamer
-    #
-    # emit only some records, as dictated by the #emit? method
-    #
-    # This is a mixin: including this module in your streamer
-    # implements its +#process+ method.
-    #
-    module Filter
-      #
-      # Filter out a subset of record/lines
-      #
-      # Subclass and re-define the emit? method
-      #
-      def process *record
-        yield record if emit?(*record)
-      end
-    end
-  end
-end

data/old/wukong/streamer/instance_streamer.rb

@@ -1,15 +0,0 @@
-module Wukong
-  module Streamer
-    #
-    # Instantiate an instance of 'record_model' for each line
-    class InstanceStreamer < Wukong::Streamer::RecordStreamer
-      class_attribute :record_model
-
-      def recordize(raw_record)
-        fields = super(raw_record)
-        [ record_model.new(*fields) ] if fields
-      end
-
-    end
-  end
-end

data/old/wukong/streamer/json_streamer.rb

@@ -1,21 +0,0 @@
-module Wukong
-  module Streamer
-    #
-    #
-    # Note: it's on you to `require 'json'` somewhere (there's too many 
-    class JsonStreamer < Wukong::Streamer::Base
-
-      #
-      # Parses the incoming record as JSON, returns a single arg to #process
-      #
-      def recordize line
-        begin
-          [JSON.parse(line)]
-        rescue StandardError => boom
-          bad_record!(boom, line.to_s)
-        end
-      end
-
-    end
-  end
-end

data/old/wukong/streamer/line_streamer.rb

@@ -1,12 +0,0 @@
-module Wukong
-  module Streamer
-    class LineStreamer < Wukong::Streamer::Base
-      #
-      # Turns a flat line into a record for +#process+ to consume
-      #
-      def recordize line
-        [line]
-      end
-    end
-  end
-end

data/old/wukong/streamer/list_reducer.rb

@@ -1,31 +0,0 @@
-module Wukong
-  module Streamer
-    #
-    # Roll up all records from a given key into a single list.
-    #
-    class ListReducer < Wukong::Streamer::AccumulatingReducer
-      attr_accessor :values
-
-      # start with an empty list
-      def start! *args
-        @values = []
-      end
-
-      # aggregate all records.
-      # note that this accumulates the full *record* -- key, value, everything.
-      def accumulate *record
-        @values << record
-      end
-
-      # emit the key and all records, tab-separated
-      #
-      # you will almost certainly want to override this method to do something
-      # interesting with the values (or override accumulate to gather scalar
-      # values)
-      #
-      def finalize
-        yield [key, @values.to_flat.join(";")].flatten
-      end
-    end
-  end
-end

data/old/wukong/streamer/rank_and_bin_reducer.rb

@@ -1,145 +0,0 @@
-module Wukong
-  module Streamer
-
-    #
-    # Bin and order a partitioned subset of keys
-    #
-    # For each record, appends a
-    #
-    # * numbering, from 0..(n-1).  Each element gets a distinct numbering based on
-    #   the order seen at the reducer; elements with identical keys might have
-    #   different numbering on different runs.
-    #
-    # * rank, a number within 1..n giving the "place" of each value. Each element
-    #   receives a successive (and thus unique) numbering, but all elements with
-    #   the same key share the same rank. The first element for a given rank has
-    #
-    #     (rank == numbering + 1)
-    #
-    # * bin, a number assigning keys by rank into a smaller number of groups.  You
-    #   must supply command line arguments
-    #
-    #     --bins=[number] --total_count=[number]
-    #
-    #   giving the number of groups and predicting in advance the total number of
-    #   records. (Or override the bin assignment method to use your own damn
-    #   strategy).
-    #
-    # If your data looked (in order) as follows, and 4 bins were requested:
-    #
-    #   data:       1   1   1 2.3   7  69  79  79  80  81  81
-    #   numbering:  0   1   2   3   4   5   6   7   8   9  10
-    #   rank:       1   1   1   4   5   6   7   7   9  10  10
-    #   4-bin:      1   1   1   2   2   3   3   3   4   4   4
-    #
-    # If instead 100 bins were requested,
-    #
-    #   data:       1   1   1 2.3   7  69  79  79  80  81  81
-    #   numbering:  0   1   2   3   4   5   6   7   8   9  10
-    #   rank:       1   1   1   4   5   6   7   7   9  10  10
-    #   100-bin:    1   1   1  28  37  46  55  55  73  82  91
-    #
-    # Note most of the bins are empty, and that the
-    #
-    # --------------------------------------------------------------------------
-    #
-    # Note that in this implementation each reducer numbers its own subset of
-    # elements from 1..total_count. If you want to number your whole dataset,
-    # you'll have to set @:reduce_tasks => 1@ in your Script's
-    # Script#default_options.
-    #
-    # You might feel a bit better about yourself if you can bin several fields
-    # (or subsets) at once. The :partition_fields option to Wukong::Script
-    # (which requests a KeyFieldBasedPartitioner) can be used to route different
-    # subsets to (possibly) distinct reducers.
-    #
-    # See the [examples/rank_and_bin_fields.rb] example script for an
-    # implementation of this. (And note the thing you have to do in case one
-    # reducer sees multiple partitions).
-    #
-    # It would surely be best to use a total sort and supply each reducer with the
-    # initial rank of its run.
-    #
-    class RankAndBinReducer < Wukong::Streamer::Base
-      attr_accessor :bin_size
-      def initialize options
-        super options
-        configure_bins! options
-        reset_order_params!
-      end
-
-      # ===========================================================================
-      #
-      # Order parameters (numbering, bin and rank)
-      #
-
-      #
-      # Key used to assign ranking -- elements with identical keys have identical
-      # rank.
-      #
-      def get_key *args
-        args.first
-      end
-
-      def reset_order_params!
-        @last_key  = nil
-        @numbering = 0
-        @rank      = 1
-      end
-
-      #
-      # The ranking is the "place" of each value: each element receives a
-      # successive (and thus unique) numbering, but all elements with the same key
-      # share the same rank. The first element for a given rank has
-      #
-      #   (rank == numbering + 1)
-      #
-      def get_rank key
-        if @last_key != key
-          @rank       = @numbering + 1
-          @last_key   = key
-        end
-        @rank
-      end
-
-      #
-      # Set the bin from the current rank
-      # elements with identical keys land in identical bins.
-      #
-      def get_bin rank
-        ((rank-0.5) / bin_size ).floor + 1
-      end
-
-      #
-      # Return the numbering, rank and bin for the given key
-      #
-      def get_order_params key
-        numbering   = @numbering # use un-incremented value
-        rank        = get_rank key
-        bin         = get_bin  rank
-        @numbering += 1
-        [numbering, rank, bin]
-      end
-
-      def configure_bins! options
-        case
-        when options[:bins]
-          total_count = options[:total_count].to_f
-          bins        = options[:bins].to_i
-          unless total_count && (total_count != 0) then raise "To set the bin (%ile) size using --bins, we need to know the total count in advance. Please supply the total_count option." end
-          self.bin_size = (total_count / bins)
-          # $stderr.puts "Splitting %s records into %s bins of size %f. First element gets bin %d, last gets bin %d, median gets bin %d/%d" %
-          #   [total_count, bins, bin_size, get_bin(1), get_bin(total_count), get_bin(((total_count+1)/2.0).floor), get_bin(((total_count+1)/2.0).ceil)]
-        else
-          raise "Please specify a number of --bins= and a --total_count= or your own strategy to bin the ranked items."
-        end
-      end
-
-      def process *fields
-        numbering, rank, bin = get_order_params(get_key(*fields))
-        yield fields.to_flat + [numbering, rank, bin]
-      end
-    end
-
-  end
-end

data/old/wukong/streamer/record_streamer.rb

@@ -1,14 +0,0 @@
-module Wukong
-  module Streamer
-    class RecordStreamer < Wukong::Streamer::Base
-
-      #
-      # Default recordizer: returns array of fields by splitting at tabs
-      #
-      def recordize line
-        line.split("\t") rescue nil
-      end
-
-    end
-  end
-end

data/old/wukong/streamer/reducer.rb

@@ -1,11 +0,0 @@
- module Wukong
-   module Streamer
-     class Reducer < Wukong::Streamer::ListReducer
-
-       def finalize &block
-         reduce @values, &block
-       end
-     end
-
-   end
-end

data/old/wukong/streamer/set_reducer.rb

@@ -1,14 +0,0 @@
-require 'wukong/streamer/list_reducer'
-module Wukong
-  module Streamer
-    #
-    # Emit each unique key and the count of its occurrences
-    #
-    class SetReducer < Wukong::Streamer::ListReducer
-      # Begin with an empty set
-      def start! *args
-        self.values = Set.new
-      end
-    end
-  end
-end

data/old/wukong/streamer/struct_streamer.rb

@@ -1,48 +0,0 @@
-# -*- coding: utf-8 -*-
-module Wukong
-  module Streamer
-    #
-    # Mix StructRecordizer into any streamer to make it accept a stream of
-    # objects -- the first field in each line is turned into a class and used to
-    # instantiate an object using the remaining fields on that line.
-    #
-    module StructRecordizer
-
-      #
-      # Turned the first field into a class name, then use the remaining fields
-      # on that line to instantiate the object to process.
-      #
-      def self.recordize rsrc, *fields
-        klass_name, suffix = rsrc.split('-', 2)
-        klass = Wukong.class_from_resource(klass_name) or return
-        # instantiate the class using the remaining fields on that line
-        begin
-          [ klass.new(*fields), suffix ]
-        rescue ArgumentError => e
-          warn "Couldn't instantiate: #{e} (#{[klass, fields].inspect})"
-          return
-        rescue Exception => e
-          raise [e, rsrc, fields].inspect
-        end
-      end
-
-      #
-      #
-      #
-      def recordize line
-        StructRecordizer.recordize *line.split("\t") unless line.blank?
-      end
-    end
-
-    #
-    # Processes file as a stream of objects -- the first field in each line is
-    # turned into a class and used to instantiate an object using the remaining
-    # fields on that line.
-    #
-    # See [StructRecordizer] for more.
-    #
-    class StructStreamer < Wukong::Streamer::Base
-      include StructRecordizer
-    end
-  end
-end

data/old/wukong/streamer/summing_reducer.rb

@@ -1,29 +0,0 @@
-module Wukong
-  module Streamer
-    #
-    # Emit each unique key and the count of its occurrences
-    #
-    class SummingReducer < Wukong::Streamer::AccumulatingReducer
-      attr_accessor :summing_elements
-      attr_accessor :sums      
-
-      # reset the counter to zero
-      def start! *args
-        self.sums = summing_elements.map{ 0 }
-      end
-
-      # record one more for this key
-      def accumulate *fields
-        vals = fields.values_at( *summing_elements )
-        vals.each_with_index{|val,idx| self.sums[idx] += val.to_i }
-      end
-
-      # emit each key field and the count, tab-separated.
-      def finalize
-        yield [key, sums].flatten
-      end
-    end
-
-  end
-end
-

data/old/wukong/streamer/uniq_by_last_reducer.rb

@@ -1,51 +0,0 @@
-module Wukong
-  module Streamer
-    #
-    # UniqByLastReducer accepts all records for a given key and emits only the
-    # last-seen.
-    #
-    # It acts like an insecure high-school kid: for each record of a given key
-    # it discards whatever record it's holding and adopts this new value. When a
-    # new key comes on the scene it emits the last record, like an older brother
-    # handing off his Depeche Mode collection.
-    #
-    # For example, to extract the *latest* value for each property, emit your
-    # records as
-    #
-    #    [resource_type, key, timestamp, ... fields ...]
-    #
-    # then set :sort_fields to 3 and :partition_fields to 2.
-    #
-    class UniqByLastReducer < Wukong::Streamer::AccumulatingReducer
-      attr_accessor :final_value
-
-      #
-      # Use first two fields as keys by default
-      #
-      def get_key *vals
-        vals[0..1]
-      end
-
-      #
-      # Adopt each value in turn: the last one's the one you want.
-      #
-      def accumulate *vals
-        self.final_value = vals
-      end
-
-      #
-      # Emit the last-seen value
-      #
-      def finalize
-        yield final_value if final_value
-      end
-
-      #
-      # Clear state on reset
-      #
-      def start! *args
-        self.final_value = nil
-      end
-    end
-  end
-end