wukong 0.1.1

data/lib/wukong/streamer/rank_and_bin_reducer.rb

@@ -0,0 +1,145 @@
+module Wukong
+  module Streamer
+
+    #
+    # Bin and order a partitioned subset of keys
+    #
+    # For each record, appends a
+    #
+    # * numbering, from 0..(n-1).  Each element gets a distinct numbering based on
+    #   the order seen at the reducer; elements with identical keys might have
+    #   different numbering on different runs.
+    #
+    # * rank, a number within 1..n giving the "place" of each value. Each element
+    #   receives a successive (and thus unique) numbering, but all elements with
+    #   the same key share the same rank. The first element for a given rank has
+    #
+    #     (rank == numbering + 1)
+    #
+    # * bin, a number assigning keys by rank into a smaller number of groups.  You
+    #   must supply command line arguments
+    #
+    #     --bins=[number] --total_count=[number]
+    #
+    #   giving the number of groups and predicting in advance the total number of
+    #   records. (Or override the bin assignment method to use your own damn
+    #   strategy).
+    #
+    # If your data looked (in order) as follows, and 4 bins were requested:
+    #
+    #   data:       1   1   1 2.3   7  69  79  79  80  81  81
+    #   numbering:  0   1   2   3   4   5   6   7   8   9  10
+    #   rank:       1   1   1   4   5   6   7   7   9  10  10
+    #   4-bin:      1   1   1   2   2   3   3   3   4   4   4
+    #
+    # If instead 100 bins were requested,
+    #
+    #   data:       1   1   1 2.3   7  69  79  79  80  81  81
+    #   numbering:  0   1   2   3   4   5   6   7   8   9  10
+    #   rank:       1   1   1   4   5   6   7   7   9  10  10
+    #   100-bin:    1   1   1  28  37  46  55  55  73  82  91
+    #
+    # Note most of the bins are empty, and that the
+    #
+    # --------------------------------------------------------------------------
+    #
+    # Note that in this implementation each reducer numbers its own subset of
+    # elements from 1..total_count. If you want to number your whole dataset,
+    # you'll have to set @:reduce_tasks => 1@ in your Script's
+    # Script#default_options.
+    #
+    # You might feel a bit better about yourself if you can bin several fields
+    # (or subsets) at once. The :partition_fields option to Wukong::Script
+    # (which requests a KeyFieldBasedPartitioner) can be used to route different
+    # subsets to (possibly) distinct reducers.
+    #
+    # See the [examples/rank_and_bin_fields.rb] example script for an
+    # implementation of this. (And note the thing you have to do in case one
+    # reducer sees multiple partitions).
+    #
+    # It would surely be best to use a total sort and supply each reducer with the
+    # initial rank of its run.
+    #
+    class RankAndBinReducer < Wukong::Streamer::Base
+      attr_accessor :bin_size
+      def initialize options
+        super options
+        configure_bins! options
+        reset_order_params!
+      end
+
+      # ===========================================================================
+      #
+      # Order parameters (numbering, bin and rank)
+      #
+
+      #
+      # Key used to assign ranking -- elements with identical keys have identical
+      # rank.
+      #
+      def get_key *args
+        args.first
+      end
+
+      def reset_order_params!
+        @last_key  = nil
+        @numbering = 0
+        @rank      = 1
+      end
+
+      #
+      # The ranking is the "place" of each value: each element receives a
+      # successive (and thus unique) numbering, but all elements with the same key
+      # share the same rank. The first element for a given rank has
+      #
+      #   (rank == numbering + 1)
+      #
+      def get_rank key
+        if @last_key != key
+          @rank       = @numbering + 1
+          @last_key   = key
+        end
+        @rank
+      end
+
+      #
+      # Set the bin from the current rank
+      # elements with identical keys land in identical bins.
+      #
+      def get_bin rank
+        ((rank-0.5) / bin_size ).floor + 1
+      end
+
+      #
+      # Return the numbering, rank and bin for the given key
+      #
+      def get_order_params key
+        numbering   = @numbering # use un-incremented value
+        rank        = get_rank key
+        bin         = get_bin  rank
+        @numbering += 1
+        [numbering, rank, bin]
+      end
+
+      def configure_bins! options
+        case
+        when options[:bins]
+          total_count = options[:total_count].to_f
+          bins        = options[:bins].to_i
+          unless total_count && (total_count != 0) then raise "To set the bin (%ile) size using --bins, we need to know the total count in advance. Please supply the total_count option." end
+          self.bin_size = (total_count / bins)
+          # $stderr.puts "Splitting %s records into %s bins of size %f. First element gets bin %d, last gets bin %d, median gets bin %d/%d" %
+          #   [total_count, bins, bin_size, get_bin(1), get_bin(total_count), get_bin(((total_count+1)/2.0).floor), get_bin(((total_count+1)/2.0).ceil)]
+        else
+          raise "Please specify a number of --bins= and a --total_count= or your own strategy to bin the ranked items."
+        end
+      end
+
+      def process *fields
+        numbering, rank, bin = get_order_params(get_key(*fields))
+        yield fields.to_flat + [numbering, rank, bin]
+      end
+    end
+
+  end
+end

data/lib/wukong/streamer/set_reducer.rb

@@ -0,0 +1,14 @@
+require 'wukong/streamer/list_reducer'
+module Wukong
+  module Streamer
+    #
+    # Emit each unique key and the count of its occurrences
+    #
+    class SetReducer < Wukong::Streamer::ListReducer
+      # Begin with an empty set
+      def start! *args
+        self.values = Set.new
+      end
+    end
+  end
+end

data/lib/wukong/streamer/struct_streamer.rb

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+module Wukong
+  module Streamer
+    #
+    # Mix StructRecordizer into any streamer to make it accept a stream of
+    # objects -- the first field in each line is turned into a class and used to
+    # instantiate an object using the remaining fields on that line.
+    #
+    module StructRecordizer
+
+      #
+      # Turned the first field into a class name, then use the remaining fields
+      # on that line to instantiate the object to process.
+      #
+      def self.recordize rsrc, *fields
+        klass_name, suffix = rsrc.split('-', 2)
+        klass = Wukong.class_from_resource(klass_name) or return
+        # instantiate the class using the remaining fields on that line
+        begin
+          [ klass.new(*fields), suffix ]
+        rescue ArgumentError => e
+          warn "Couldn't instantiate: #{e} (#{[rsrc, fields].inspect})"
+          return
+        rescue Exception => e
+          raise [e, rsrc, fields].inspect
+        end
+      end
+
+      #
+      #
+      #
+      def recordize line
+        StructRecordizer.recordize *line.split("\t")
+      end
+    end
+
+    #
+    # Processes file as a stream of objects -- the first field in each line is
+    # turned into a class and used to instantiate an object using the remaining
+    # fields on that line.
+    #
+    # See [StructRecordizer] for more.
+    #
+    class StructStreamer < Wukong::Streamer::Base
+      include StructRecordizer
+    end
+  end
+end

data/lib/wukong/streamer/summing_reducer.rb

@@ -0,0 +1,29 @@
+module Wukong
+  module Streamer
+    #
+    # Emit each unique key and the count of its occurrences
+    #
+    class SummingReducer < Wukong::Streamer::AccumulatingReducer
+      attr_accessor :summing_elements
+      attr_accessor :sums      
+
+      # reset the counter to zero
+      def start! *args
+        self.sums = summing_elements.map{ 0 }
+      end
+
+      # record one more for this key
+      def accumulate *fields
+        vals = fields.values_at( *summing_elements )
+        vals.each_with_index{|val,idx| self.sums[idx] += val.to_i }
+      end
+
+      # emit each key field and the count, tab-separated.
+      def finalize
+        yield [key, sums].flatten
+      end
+    end
+
+  end
+end
+

data/lib/wukong/streamer/uniq_by_last_reducer.rb

@@ -0,0 +1,44 @@
+module Wukong
+  module Streamer
+    #
+    # Accumulate acts like an insecure high-school kid, for each key adopting in
+    # turn the latest value seen. It then emits the last (in sort order) value
+    # for that key.
+    #
+    # For example, to extract the *latest* value for each property, set hadoop
+    # to use <resource, item_id, timestamp> as sort fields and <resource,
+    # item_id> as key fields.
+    #
+    class UniqByLastReducer < Wukong::Streamer::AccumulatingReducer
+      attr_accessor :final_value
+
+      #
+      # Use first two fields as keys by default
+      #
+      def get_key *vals
+        vals[0..1]
+      end
+
+      #
+      # Adopt each value in turn: the last one's the one you want.
+      #
+      def accumulate *vals
+        self.final_value = vals
+      end
+
+      #
+      # Emit the last-seen value
+      #
+      def finalize
+        yield final_value if final_value
+      end
+
+      #
+      # Clear state on reset
+      #
+      def start! *args
+        self.final_value = nil
+      end
+    end
+  end
+end

data/lib/wukong/typed_struct.rb

@@ -0,0 +1,12 @@
+class TypedStruct < Struct
+  def self.new *members_types
+    members, mtypes = members_types.transpose
+    klass           = Struct.new *members.map(&:to_sym)
+    klass.class_eval do
+      cattr_accessor :mtypes, :members_types
+      self.mtypes        = mtypes
+      self.members_types = Hash.zip(members, mtypes)
+    end
+    klass
+  end
+end

data/lib/wukong/wukong_class.rb

@@ -0,0 +1,21 @@
+# require 'active_support/core_ext/class/inheritable_attributes.rb'
+require 'extlib/class'
+
+module Wukong
+  #
+  # Use to instrument an actual class to behave
+  #
+  module WukongClass
+
+
+    def [](attr)
+      self.send attr
+    end
+    def []=(attr, val)
+      self.send("#{attr}=", val)
+    end
+
+  end
+
+
+end

data/spec/bin/hdp-wc_spec.rb

@@ -0,0 +1,4 @@
+# -*- coding: undecided -*-
+
+
+I18N_STR = "Iñtërnâtiônàlizætiøn hasn't happened to ruby 1.8 yet"

data/wukong.gemspec

@@ -0,0 +1,179 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{wukong}
+  s.version = "0.1.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Philip (flip) Kromer"]
+  s.date = %q{2009-09-28}
+  s.description = %q{  Treat your dataset like a:
+
+      * stream of lines when it’s efficient to process by lines
+      * stream of field arrays when it’s efficient to deal directly with fields
+      * stream of lightweight objects when it’s efficient to deal with objects
+
+  Wukong is friends with Hadoop the elephant, Pig the query language, and the cat on your command line.
+}
+  s.email = %q{flip@infochimps.org}
+  s.executables = ["cutc", "cuttab", "greptrue", "hdp-cat", "hdp-catd", "hdp-du", "hdp-get", "hdp-kill", "hdp-ls", "hdp-mkdir", "hdp-mv", "hdp-parts_to_keys.rb", "hdp-ps", "hdp-put", "hdp-rm", "hdp-sort", "hdp-stream", "hdp-stream-flat", "hdp-sync", "hdp-wc", "md5sort", "tabchar", "uniqc", "wu-hist", "wu-lign", "wu-sum"]
+  s.extra_rdoc_files = [
+    "LICENSE.textile",
+     "README.textile"
+  ]
+  s.files = [
+    "doc/INSTALL.textile",
+     "doc/LICENSE.textile",
+     "doc/README-tutorial.textile",
+     "doc/README-wulign.textile",
+     "doc/README-wutils.textile",
+     "doc/TODO.textile",
+     "doc/UsingWukong-part1-setup.textile",
+     "doc/UsingWukong-part2-scraping.textile",
+     "doc/UsingWukong-part3-parsing.textile",
+     "doc/code/api_response_example.txt",
+     "doc/code/parser_skeleton.rb",
+     "doc/hadoop-nfs.textile",
+     "doc/hadoop-setup.textile",
+     "doc/index.textile",
+     "doc/intro_to_map_reduce/MapReduceDiagram.graffle",
+     "doc/links.textile",
+     "doc/overview.textile",
+     "doc/pig/PigLatinExpressionsList.txt",
+     "doc/pig/PigLatinReferenceManual.html",
+     "doc/pig/PigLatinReferenceManual.txt",
+     "doc/tips.textile",
+     "doc/usage.textile",
+     "doc/utils.textile",
+     "examples/README.txt",
+     "examples/and_pig/sample_queries.rb",
+     "examples/apache_log_parser.rb",
+     "examples/count_keys.rb",
+     "examples/count_keys_at_mapper.rb",
+     "examples/graph/adjacency_list.rb",
+     "examples/graph/breadth_first_search.rb",
+     "examples/graph/gen_2paths.rb",
+     "examples/graph/gen_multi_edge.rb",
+     "examples/graph/gen_symmetric_links.rb",
+     "examples/package-local.rb",
+     "examples/package.rb",
+     "examples/pagerank/README.textile",
+     "examples/pagerank/gen_initial_pagerank_graph.pig",
+     "examples/pagerank/pagerank.rb",
+     "examples/pagerank/pagerank_initialize.rb",
+     "examples/pagerank/run_pagerank.sh",
+     "examples/rank_and_bin.rb",
+     "examples/run_all.sh",
+     "examples/sample_records.rb",
+     "examples/size.rb",
+     "examples/word_count.rb",
+     "lib/wukong.rb",
+     "lib/wukong/and_pig.rb",
+     "lib/wukong/and_pig/README.textile",
+     "lib/wukong/and_pig/as.rb",
+     "lib/wukong/and_pig/data_types.rb",
+     "lib/wukong/and_pig/functions.rb",
+     "lib/wukong/and_pig/generate.rb",
+     "lib/wukong/and_pig/generate/variable_inflections.rb",
+     "lib/wukong/and_pig/junk.rb",
+     "lib/wukong/and_pig/operators.rb",
+     "lib/wukong/and_pig/operators/compound.rb",
+     "lib/wukong/and_pig/operators/evaluators.rb",
+     "lib/wukong/and_pig/operators/execution.rb",
+     "lib/wukong/and_pig/operators/file_methods.rb",
+     "lib/wukong/and_pig/operators/foreach.rb",
+     "lib/wukong/and_pig/operators/groupies.rb",
+     "lib/wukong/and_pig/operators/load_store.rb",
+     "lib/wukong/and_pig/operators/meta.rb",
+     "lib/wukong/and_pig/operators/relational.rb",
+     "lib/wukong/and_pig/pig_struct.rb",
+     "lib/wukong/and_pig/pig_var.rb",
+     "lib/wukong/and_pig/symbol.rb",
+     "lib/wukong/and_pig/utils.rb",
+     "lib/wukong/bad_record.rb",
+     "lib/wukong/boot.rb",
+     "lib/wukong/datatypes.rb",
+     "lib/wukong/datatypes/enum.rb",
+     "lib/wukong/dfs.rb",
+     "lib/wukong/encoding.rb",
+     "lib/wukong/extensions.rb",
+     "lib/wukong/extensions/array.rb",
+     "lib/wukong/extensions/blank.rb",
+     "lib/wukong/extensions/class.rb",
+     "lib/wukong/extensions/date_time.rb",
+     "lib/wukong/extensions/emittable.rb",
+     "lib/wukong/extensions/hash.rb",
+     "lib/wukong/extensions/hash_like.rb",
+     "lib/wukong/extensions/hashlike_class.rb",
+     "lib/wukong/extensions/module.rb",
+     "lib/wukong/extensions/pathname.rb",
+     "lib/wukong/extensions/string.rb",
+     "lib/wukong/extensions/struct.rb",
+     "lib/wukong/extensions/symbol.rb",
+     "lib/wukong/logger.rb",
+     "lib/wukong/models/graph.rb",
+     "lib/wukong/rdf.rb",
+     "lib/wukong/schema.rb",
+     "lib/wukong/script.rb",
+     "lib/wukong/script/hadoop_command.rb",
+     "lib/wukong/script/local_command.rb",
+     "lib/wukong/streamer.rb",
+     "lib/wukong/streamer/accumulating_reducer.rb",
+     "lib/wukong/streamer/base.rb",
+     "lib/wukong/streamer/count_keys.rb",
+     "lib/wukong/streamer/count_lines.rb",
+     "lib/wukong/streamer/filter.rb",
+     "lib/wukong/streamer/line_streamer.rb",
+     "lib/wukong/streamer/list_reducer.rb",
+     "lib/wukong/streamer/preprocess_with_pipe_streamer.rb",
+     "lib/wukong/streamer/rank_and_bin_reducer.rb",
+     "lib/wukong/streamer/set_reducer.rb",
+     "lib/wukong/streamer/struct_streamer.rb",
+     "lib/wukong/streamer/summing_reducer.rb",
+     "lib/wukong/streamer/uniq_by_last_reducer.rb",
+     "lib/wukong/typed_struct.rb",
+     "lib/wukong/wukong_class.rb",
+     "spec/bin/hdp-wc_spec.rb",
+     "spec/spec_helper.rb",
+     "wukong.gemspec"
+  ]
+  s.homepage = %q{http://github.com/mrflip/wukong}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Wukong makes Hadoop so easy a chimpanzee can use it.}
+  s.test_files = [
+    "spec/bin/hdp-wc_spec.rb",
+     "spec/spec_helper.rb",
+     "examples/and_pig/sample_queries.rb",
+     "examples/apache_log_parser.rb",
+     "examples/count_keys.rb",
+     "examples/count_keys_at_mapper.rb",
+     "examples/graph/adjacency_list.rb",
+     "examples/graph/breadth_first_search.rb",
+     "examples/graph/gen_2paths.rb",
+     "examples/graph/gen_multi_edge.rb",
+     "examples/graph/gen_symmetric_links.rb",
+     "examples/package-local.rb",
+     "examples/package.rb",
+     "examples/pagerank/pagerank.rb",
+     "examples/pagerank/pagerank_initialize.rb",
+     "examples/rank_and_bin.rb",
+     "examples/sample_records.rb",
+     "examples/size.rb",
+     "examples/word_count.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end