wukong 1.4.10 → 1.4.11

data/CHANGELOG.textile

@@ -1,3 +1,14 @@
+h2. Wukong v.14.11 2010-07-30
+
+* added the @max_(maps|reduces)_per_(node|cluster)@ jobconfs.
+* added jobconfs for io_job_mb and friends.
+* added a loadable module to convert output data to pig bags and tuples
+* pulled in several methods from active_support, incl. Enumerable#sum
+* Scripts to find percentile rank of elements in a dataset
+* We are starting to move wukong to a model where streaming is from a generic
+  source into a generic sink. Several stores have been landed in the code, but
+  many are in a half- or un-baked state. Please ignore this for the moment.
+
 h2. Wukong v1.4.8 2010-06-05
 
 * made scripts inject a helpful job name using mapred.job.name

data/bin/hdp-mkdirp

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+#
+# Despite arguments and deliberation, this IS a necessary script. Azkaban, should you choose to
+# use it, will fail if (it seems) ANY of its spawned subprocesses fails
+#
+
+hadoop fs -test -e "$@"
+if [ "$?" != "0" ] ; then
+  # echo "File does not exist, making..."
+  exec hadoop fs -mkdir "$@"
+fi

data/bin/hdp-rm

@@ -1,11 +1,32 @@
 #!/usr/bin/env bash
 
+#
+# Documentation for hadoop fs -rmr says "acts just like Unix rm -rf command". If this
+# is true then we need to ignore directories that don't exist and still return 0.
+#
+
+#
+# All the dirty conditional logic here does is test whether a directory exists. If so, remove it
+#
 if [ "$1" == "-r" ] ; then
+  shift
+  if [ "$1" == "-skipTrash" ] ; then
     shift
-    action=rmr
+    hadoop fs -test -e "$@"
+    if [ "$?" == "0" ] ; then
+           # echo "File exists, skipping trash, removing it..."
+      echo hadoop dfs -rmr "$@"
+      exec hadoop dfs -rmr "$@"
+    fi
+  else
+    hadoop fs -test -e "$@"
+    if [ "$?" == "0" ] ; then
+            # echo "File exists, removing it..."
+      echo hadoop dfs -rmr "$@"
+      exec hadoop dfs -rmr "$@"
+    fi
+  fi
 else
-    action=rm
+  echo hadoop dfs -rm "$@"
+  exec hadoop dfs -rm "$@"
 fi
-echo hadoop dfs -$action "$@"
-# read -p "Hit ctrl-C to abort or enter to do this...."
-exec hadoop dfs -$action "$@"

data/bin/hdp-sort

@@ -13,7 +13,7 @@ if [ "$output_file" == "" ] ; then echo "$0 input_file output_file [mapper=/bin/
 HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
 
 cmd="${HADOOP_HOME}/bin/hadoop \
-     jar         ${HADOOP_HOME}/contrib/streaming/hadoop-*-streaming.jar		
+     jar         ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar		
     -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner 			
     -jobconf     num.key.fields.for.partition=\"$partfields\"				
     -jobconf 	 stream.num.map.output.key.fields=\"$sortfields\"

data/bin/hdp-stream

@@ -13,7 +13,7 @@ if [ "$output_file" == "" ] ; then echo "$0 input_file output_file [mapper=/bin/
 HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
 
 cmd="${HADOOP_HOME}/bin/hadoop \
-     jar         ${HADOOP_HOME}/contrib/streaming/hadoop-*-streaming.jar		
+     jar         ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar		
     -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner 			
     -jobconf     num.key.fields.for.partition=\"$partfields\"				
     -jobconf 	 stream.num.map.output.key.fields=\"$sortfields\"

data/bin/hdp-stream-flat

@@ -14,7 +14,7 @@ HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
 # -jobconf mapred.reduce.tasks=3                                                    \
 
 exec ${HADOOP_HOME}/bin/hadoop \
-     jar         ${HADOOP_HOME}/contrib/streaming/hadoop*streaming*.jar		\
+     jar         ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar		\
     "$@"                                                                                \
     -jobconf    "mapred.job.name=`basename $0`-$map_script-$input_file-$output_file"    \
     -mapper  	"$map_script"  								\

data/examples/store/chunked_store_example.rb

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'wukong'
+# require 'wukong/store'
+
+require 'configliere'
+Configliere.use :commandline, :define, :config_file
+Settings.read('foo.yaml')
+
+# store = ChunkedFlatFileStore.new(Settings)
+
+100.times do |iter|
+  # store.save   [iter, Time.now.to_flat].join("\t")
+  $stdout.puts [iter, Time.now.to_flat].join("\t")
+  sleep 2
+end
+
+

data/lib/wukong/and_pig.rb

@@ -0,0 +1,23 @@
+module Enumerable
+  #
+  # Convert an array of values to a string representing it as a pig tuple
+  #
+  def to_pig_tuple
+    map{|*vals| '(' + vals.join(',') + ')' }
+  end
+
+  #
+  # Convert an array of values to a string pig format
+  # Delegates to to_pig_tuple -- see also to_pig_bag
+  #
+  def to_pig *args
+    to_pig_tuple *args
+  end
+
+  #
+  # Convert an array of values to a string representing it as a pig bag
+  #
+  def to_pig_bag
+    '{' + self.join(',') + '}'
+  end
+end

data/lib/wukong/monitor.rb

@@ -0,0 +1,7 @@
+module Monkeyshines
+  module Monitor
+    autoload :PeriodicMonitor, 'monkeyshines/monitor/periodic_monitor'
+    autoload :PeriodicLogger,  'monkeyshines/monitor/periodic_logger'
+  end
+end
+

data/lib/wukong/monitor/chunked_store.rb

@@ -0,0 +1,23 @@
+require 'monkeyshines/monitor/periodic_monitor'
+module Monkeyshines
+  module Monitor
+    module ChunkedStore
+      attr_accessor :file_pattern
+      def initialize file_pattern
+        self.file_pattern = file_pattern
+        super file_pattern.make
+      end
+
+      def close_and_reopen
+        close
+        self.filename = file_pattern.make
+        dump_file
+      end
+
+      def save *args
+        chunk_monitor.periodically{ close_rename_and_open }
+        super *args
+      end
+    end
+  end
+end

data/lib/wukong/monitor/periodic_logger.rb

@@ -0,0 +1,34 @@
+module Monkeyshines
+  module Monitor
+
+    #
+    # Emits a log line but only every +iter_interval+ calls or +time_interval+
+    # lapse.
+    #
+    # Since the contents of the block aren't called until the criteria are met,
+    # you can put relatively expensive operations in the log without killing
+    # your iteration time.
+    #
+    class PeriodicLogger < PeriodicMonitor
+      #
+      # Call with a block that returns a string or array to log.
+      # If you return
+      #
+      # Ex: log if it has been at least 5 minutes since last announcement:
+      #
+      #   periodic_logger = Monkeyshines::Monitor::PeriodicLogger.new(:time => 300)
+      #   loop do
+      #     # ... stuff ...
+      #     periodic_logger.periodically{ [morbenfactor, crunkosity, exuberance] }
+      #   end
+      #
+      def periodically &block
+        super do
+          now = Time.now.utc.to_f
+          result = [ "%10d"%iter, "%7.1f"%since, "%7.1f"%inst_rate(now), (block ? block.call : nil) ].flatten.compact
+          Log.info result.join("\t")
+        end
+      end
+    end
+  end
+end

data/lib/wukong/monitor/periodic_monitor.rb

@@ -0,0 +1,72 @@
+module Monkeyshines
+  module Monitor
+    #
+    # Accepts a lightweight call every iteration.
+    #
+    # Once either a time or an iteration criterion is met, executes the block
+    # and resets the timer until next execution.
+    #
+    # Note that the +time_interval+ is measured *excution to execution* and not
+    # in multiples of iter_interval. Say I set a time_interval of 300s, and
+    # happen to iterate at 297s and 310s after start.  Then the monitor will
+    # execute at 310s, and the next execution will happen on or after 610s.
+    #
+    # Also note that when *either* criterion is met, *both* criteria are
+    # reset. Say I set a time interval of 300s and an +iter_interval+ of 10_000;
+    # and that at 250s I reach iteration 10_000.  Then the monitor will execute
+    # on or after 20_000 iteration or 550s, whichever happens first.
+    #
+    class PeriodicMonitor
+      attr_accessor :time_interval, :iter_interval
+      attr_accessor :last_time, :current_iter, :iter, :started_at
+
+      def initialize options={}
+        self.started_at    = Time.now.utc.to_f
+        self.last_time     = started_at
+        self.iter          = 0
+        self.current_iter  = 0
+        self.time_interval = options[:time]
+        self.iter_interval = options[:iters]
+      end
+
+      # True if more than +iter_interval+ has elapsed since last execution.
+      def enough_iterations?
+        iter % iter_interval == 0 if iter_interval
+      end
+
+      # True if more than +time_interval+ has elapsed since last execution.
+      def enough_time? now
+        (now - last_time) > time_interval if time_interval
+      end
+
+      # Time since monitor was created
+      def since
+        Time.now.utc.to_f - started_at
+      end
+      # Overall iterations per second
+      def rate
+        iter.to_f / since.to_f
+      end
+      # "Instantaneous" iterations per second
+      def inst_rate now
+        current_iter.to_f / (now-last_time).to_f
+      end
+
+      #
+      # if the interval conditions are met, executes block; otherwise just does
+      # bookkeeping and returns.
+      #
+      def periodically &block
+        self.iter += 1
+        self.current_iter += 1
+        now       = Time.now.utc.to_f
+        if enough_iterations? || enough_time?(now)
+          block.call(iter, (now-last_time))
+          self.last_time = now
+          self.current_iter = 0
+        end
+      end
+    end
+
+  end
+end

data/lib/wukong/script/hadoop_command.rb

@@ -32,6 +32,10 @@ module Wukong
     Settings.define :noempty,                                  :description => "don't create zero-byte reduce files (hadoop mode only)", :wukong => true
     Settings.define :job_name,               :jobconf => true, :description => 'mapred.job.name', :wukong => true
     # mapred.linerecordreader.maxlength :description => "Safeguards against corrupted data: lines longer than this (in bytes) are treated as bad records."
+    Settings.define :max_reduces_per_node,   :jobconf => true, :description => 'mapred.max.reduces.per.node',    :wukong => true
+    Settings.define :max_reduces_per_cluster,:jobconf => true, :description => 'mapred.max.reduces.per.cluster', :wukong => true
+    Settings.define :max_maps_per_node,      :jobconf => true, :description => 'mapred.max.maps.per.node',       :wukong => true
+    Settings.define :max_maps_per_cluster,   :jobconf => true, :description => 'mapred.max.maps.per.cluster',    :wukong => true
 
     # emit a -jobconf hadoop option if the simplified command line arg is present
     # if not, the resulting nil will be elided later

data/lib/wukong/store.rb

@@ -0,0 +1,14 @@
+module Monkeyshines
+  module Store
+    extend FactoryModule
+    autoload :Base,                 'monkeyshines/store/base'
+    autoload :FlatFileStore,        'monkeyshines/store/flat_file_store'
+    autoload :ConditionalStore,     'monkeyshines/store/conditional_store'
+    autoload :ChunkedFlatFileStore, 'monkeyshines/store/chunked_flat_file_store'
+    autoload :KeyStore,             'monkeyshines/store/key_store'
+    autoload :TokyoTdbKeyStore,     'monkeyshines/store/tokyo_tdb_key_store'
+    autoload :TyrantTdbKeyStore,    'monkeyshines/store/tyrant_tdb_key_store'
+    autoload :TyrantRdbKeyStore,    'monkeyshines/store/tyrant_rdb_key_store'
+    autoload :ReadThruStore,        'monkeyshines/store/read_thru_store'
+  end
+end

data/lib/wukong/store/base.rb

@@ -0,0 +1,29 @@
+module Monkeyshines
+  module Store
+    class Base
+      attr_accessor :options
+      def initialize _options={}
+        self.options = _options
+        Log.info "Creating #{self.class}"
+      end
+
+      #
+      def each_as klass, &block
+        self.each do |*args|
+          begin
+            item = klass.new *args[1..-1]
+          rescue Exception => e
+            Log.info [args, e.to_s, self].join("\t")
+            raise e
+          end
+          yield item
+        end
+      end
+
+      def log_line
+        nil
+      end
+
+    end
+  end
+end

data/lib/wukong/store/chunked_flat_file_store.rb

@@ -0,0 +1,37 @@
+module Monkeyshines
+  module Store
+    class ChunkedFlatFileStore < Monkeyshines::Store::FlatFileStore
+      attr_accessor :filename_pattern, :chunk_monitor, :handle
+
+      DEFAULT_OPTIONS = {
+        :chunktime    => 4*60*60, # default 4 hours
+        :pattern   => ":rootdir/:date/:handle+:timestamp-:pid.tsv",
+        :rootdir   => nil,
+        :filemode  => 'w',
+      }
+
+      def initialize _options
+        self.options = DEFAULT_OPTIONS.deep_merge(_options)
+        raise "You don't really want a chunk time this small: #{options[:chunktime]}" unless options[:chunktime] > 600
+        self.chunk_monitor    = Monkeyshines::Monitor::PeriodicMonitor.new( :time => options[:chunktime] )
+        self.handle           = options[:handle] || Monkeyshines::CONFIG[:handle]
+        self.filename_pattern = Monkeyshines::Utils::FilenamePattern.new(options[:pattern], :handle => handle, :rootdir => options[:rootdir])
+        super options.merge(:filename => filename_pattern.make())
+        self.mkdir!
+      end
+
+      def save *args
+        result = super *args
+        chunk_monitor.periodically do
+          new_filename = filename_pattern.make()
+          Log.info "Rotating chunked file #{filename} into #{new_filename}"
+          self.close
+          @filename = new_filename
+          self.mkdir!
+        end
+        result
+      end
+
+    end
+  end
+end

data/lib/wukong/store/conditional_store.rb

@@ -0,0 +1,57 @@
+module Monkeyshines
+  module Store
+    class ConditionalStore < Monkeyshines::Store::Base
+      attr_accessor :options, :cache, :store, :misses
+
+      DEFAULT_OPTIONS = {
+        :cache => { :type => :tyrant_rdb_key_store    },
+        :store => { :type => :chunked_flat_file_store },
+      }
+
+      #
+      #
+      # +cache+ must behave like a hash (Hash and
+      #  Monkeyshines::Store::TyrantRdbKeyStore are both cromulent
+      #  choices).
+      #
+      #
+      #
+      def initialize _options
+        self.options = DEFAULT_OPTIONS.deep_merge(_options)
+        self.cache  = Monkeyshines::Store.create(options[:cache])
+        self.store  = Monkeyshines::Store.create(options[:store])
+        self.misses = 0
+      end
+
+      #
+      # If key is absent, save the result of calling the block.
+      # If key is present, block is never called.
+      #
+      # Ex:
+      #   rt_store.set(url) do
+      #     fetcher.get url # will only be called if url isn't in rt_store
+      #   end
+      #
+      def set key, force=nil, &block
+        return if (!force) && cache.include?(key)
+        cache_val, store_val = block.call()
+        return unless cache_val
+        cache.set_nr key, cache_val # update cache
+        store << store_val          # save value
+        self.misses += 1            # track the cache miss
+        store_val
+      end
+
+      def size() cache.size  end
+
+      def log_line
+        [size, "%8d misses"%misses]
+      end
+
+      def close()
+        cache.close
+        store.close
+      end
+    end
+  end
+end

data/lib/wukong/store/factory.rb

@@ -0,0 +1,8 @@
+module Monkeyshines
+  module Store
+    class Factory
+      def self.generate type, opts
+      end
+    end
+  end
+end

data/lib/wukong/store/flat_file_store.rb

@@ -0,0 +1,90 @@
+require 'fileutils'; include FileUtils
+
+module Monkeyshines
+  module Store
+    #
+    class FlatFileStore < Store::Base
+      attr_accessor :filename, :filemode
+
+      #
+      # +filename_root+  : first part of name for files
+      #
+      def initialize options={}
+        Log.debug "New #{self.class} as #{options.inspect}"
+        self.filename = options[:filename] or raise "Missing filename in #{self.class}"
+        self.filemode = options[:filemode] || 'r'
+        skip!(options[:skip]) if options[:skip]
+      end
+
+      #
+      #
+      #
+      def each &block
+        file.each do |line|
+          next if line[0..0] == '#'
+          attrs = line.chomp.split("\t")
+          next if attrs.blank?
+          yield *attrs
+        end
+      end
+
+      #
+      # Read ahead n_lines lines in the file
+      #
+      def skip! n_lines
+        Log.info "Skipping #{n_lines} in #{self.class}:#{filename}"
+        n_lines.times do
+          file.readline
+        end
+      end
+
+      #
+      # Open the timestamped file,
+      # ensuring its directory exists
+      #
+      def file
+        return @file if @file
+        Log.info "Opening file #{filename} with mode #{filemode}"
+        @file = File.open(filename, filemode)
+      end
+
+      # Close the dump file
+      def close
+        @file.close if @file
+        @file = nil
+      end
+
+      # Ensure the file's directory exists
+      def mkdir!
+        dir = File.dirname(filename)
+        return if File.directory?(dir)
+        Log.info "Making directory #{dir}"
+        FileUtils.mkdir_p dir
+      end
+
+      # write to the file
+      def save obj
+        file << obj.to_flat.join("\t")+"\n"
+        obj
+      end
+      
+      # returns the size of the current file
+      def size
+        return 0 if !@file
+        File.size(filename)
+      end
+
+      def set key, *args, &block
+        tok, obj = block.call
+        save obj
+      end
+
+      # delegates to +#save+ -- writes the object to the file
+      def <<(obj)
+        save obj
+      end
+
+    end
+  end
+end
+

data/lib/wukong/store/key_store.rb

@@ -0,0 +1,51 @@
+module Monkeyshines
+  module Store
+    class KeyStore < Monkeyshines::Store::Base
+      # The actual backing store; should respond to #set and #get methods
+      attr_accessor :db
+
+      #
+      # Executes block once for each element in the whole DB, in whatever order
+      # the DB thinks you should see it.
+      #
+      # Your block will see |key, val|
+      #
+      # key_store.each do |key, val|
+      #   # ... stuff ...
+      # end
+      #
+      def each &block
+        db.iterinit
+        loop do
+          key = db.iternext or break
+          val = db[key]
+          yield key, val
+        end
+      end
+
+
+      # Save the value into the database
+      def set(key, val)
+        return unless val
+        db[key] = val
+      end
+
+      alias_method :save, :set
+      def get(key)      db[key]  end
+      def [](key)       db[key]  end
+      def close()       db.close end
+      def size()        db.size  end
+
+      #
+      # Load from standard command-line options
+      #
+      # obvs only works when there's just one store
+      #
+      def self.new_from_command_line cmdline_opts, default_opts={}
+        options = default_opts.merge(cmdline_opts)
+        store = self.new(options[:store_db])
+        store
+      end
+    end
+  end
+end

data/lib/wukong/store/null_store.rb

@@ -0,0 +1,15 @@
+module Monkeyshines
+  module Store
+    class NullStore < Monkeyshines::Store::Base
+
+      def each *args, &block
+      end
+
+
+      # Does nothing!
+      def set *args
+      end
+
+    end
+  end
+end

data/lib/wukong/store/read_thru_store.rb

@@ -0,0 +1,22 @@
+module Monkeyshines
+  module Store
+    class ReadThruStore < Monkeyshines::Store::TyrantTdbKeyStore
+
+      #
+      # If key is absent, save the result of calling the block.
+      # If key is present, block is never called.
+      #
+      # Ex:
+      #   rt_store.set(url) do
+      #     fetcher.get url # will only be called if url isn't in rt_store
+      #   end
+      #
+      def set key, force=nil, &block
+        return if !force && db.has_key?(key)
+        result = block.call() or return
+        super(key, result)
+      end
+
+    end
+  end
+end

data/lib/wukong/store/tokyo_tdb_key_store.rb

@@ -0,0 +1,33 @@
+require 'tokyocabinet'
+module Monkeyshines
+  module Store
+    #
+    # Implementation of KeyStore with a Local TokyoCabinet table database (TDB)
+    #
+    class TokyoTdbKeyStore < Monkeyshines::Store::KeyStore
+
+      # pass in the filename or URI of a tokyo cabinet table-style DB
+      # set create_db = true if you want to create a missing DB file
+      def initialize db_uri, *args
+        self.db = TokyoCabinet::TDB.new
+        db.open(db_uri, TokyoCabinet::TDB::OWRITER) or raise "#{self.class.to_s}: Can't open TokyoCabinet TDB #{db_uri}"
+        super *args
+      end
+
+
+      def each_as klass, &block
+        self.each do |key, hsh|
+          yield klass.from_hash hsh
+        end
+      end
+      # Delegate to store
+      def set(key, val)
+        return unless val
+        db.put key, val.to_hash.compact
+      end
+
+      def size()        db.rnum  end
+
+    end #class
+  end
+end

data/lib/wukong/store/tyrant_rdb_key_store.rb

@@ -0,0 +1,57 @@
+require 'tokyotyrant'
+module Monkeyshines
+  module Store
+
+    #
+    # Implementation of KeyStore with a Local TokyoCabinet hash database (RDB)
+    #
+    class TyrantRdbKeyStore < Monkeyshines::Store::KeyStore
+      attr_accessor :db_host, :db_port
+
+      # pass in the host:port uri of the key store.
+      def initialize options
+        raise "URI for #{self.class} is required" if options[:uri].blank?
+        self.db_host, self.db_port = options[:uri].to_s.split(':')
+        self.db_host.gsub!(/^(localhost|127\.0\.0\.1)$/,'')
+        super options
+      end
+
+      def db
+        return @db if @db
+        @db ||= TokyoTyrant::RDB.new
+        @db.open(db_host, db_port) or raise("Can't open DB at host #{db_host} port #{db_port}. Pass in host:port' #{@db.ecode}: #{@db.errmsg(@db.ecode)}")
+        @db
+      end
+
+      def close
+        @db.close if @db
+        @db = nil
+      end
+
+      # Save the value into the database without waiting for a response.
+      def set_nr(key, val)
+        db.putnr key, val if val
+      end
+
+      def size()        db.rnum  end
+      def include? *args
+        db.has_key? *args
+      end
+
+      # require 'memcache'
+      # def initialize db_uri=nil, *args
+      #   # db_uri ||= ':1978'
+      #   # self.db_host, self.db_port = db_uri.split(':')
+      #   self.db = MemCache.new(db_uri, :no_reply => true)
+      #   if !self.db then raise("Can't open DB #{db_uri}. Pass in host:port, default is ':1978' #{db.ecode}: #{db.errmsg(db.ecode)}") end
+      #   super *args
+      # end
+      #
+      # def size
+      #   db.stats
+      # end
+
+    end #class
+  end
+end
+

data/lib/wukong/store/tyrant_tdb_key_store.rb

@@ -0,0 +1,20 @@
+require 'tokyotyrant'
+require 'tyrant_rdb_key_store'
+module Monkeyshines
+  module Store
+    #
+    # Implementation of KeyStore with a Local TokyoCabinet Table database (RDBTBL)
+    #
+    class TyrantRdbKeyStore < TyrantRdbKeyStore Monkeyshines::Store::KeyStore
+
+      def db
+        return @db if @db
+        @db ||= TokyoTyrant::RDBTBL.new
+        @db.open(db_host, db_port) or raise("Can't open DB #{db_host}:#{db_port}. Pass in host:port' #{@db.ecode}: #{@db.errmsg(@db.ecode)}")
+        @db
+      end
+
+    end #class
+  end
+end
+

data/wukong.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{wukong}
-  s.version = "1.4.10"
+  s.version = "1.4.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer"]
-  s.date = %q{2010-07-19}
+  s.date = %q{2010-07-30}
   s.description = %q{  Treat your dataset like a:
 
       * stream of lines when it's efficient to process by lines
@@ -42,6 +42,7 @@ Gem::Specification.new do |s|
      "bin/hdp-kill-task",
      "bin/hdp-ls",
      "bin/hdp-mkdir",
+     "bin/hdp-mkdirp",
      "bin/hdp-mv",
      "bin/hdp-parts_to_keys.rb",
      "bin/hdp-ps",
@@ -151,9 +152,11 @@ Gem::Specification.new do |s|
      "examples/size.rb",
      "examples/stats/avg_value_frequency.rb",
      "examples/stats/data/avg_value_frequency.tsv",
+     "examples/store/chunked_store_example.rb",
      "examples/stupidly_simple_filter.rb",
      "examples/word_count.rb",
      "lib/wukong.rb",
+     "lib/wukong/and_pig.rb",
      "lib/wukong/bad_record.rb",
      "lib/wukong/datatypes.rb",
      "lib/wukong/datatypes/enum.rb",
@@ -182,12 +185,28 @@ Gem::Specification.new do |s|
      "lib/wukong/keystore/tyrant_notes.textile",
      "lib/wukong/logger.rb",
      "lib/wukong/models/graph.rb",
+     "lib/wukong/monitor.rb",
+     "lib/wukong/monitor/chunked_store.rb",
+     "lib/wukong/monitor/periodic_logger.rb",
+     "lib/wukong/monitor/periodic_monitor.rb",
      "lib/wukong/periodic_monitor.rb",
      "lib/wukong/rdf.rb",
      "lib/wukong/schema.rb",
      "lib/wukong/script.rb",
      "lib/wukong/script/hadoop_command.rb",
      "lib/wukong/script/local_command.rb",
+     "lib/wukong/store.rb",
+     "lib/wukong/store/base.rb",
+     "lib/wukong/store/chunked_flat_file_store.rb",
+     "lib/wukong/store/conditional_store.rb",
+     "lib/wukong/store/factory.rb",
+     "lib/wukong/store/flat_file_store.rb",
+     "lib/wukong/store/key_store.rb",
+     "lib/wukong/store/null_store.rb",
+     "lib/wukong/store/read_thru_store.rb",
+     "lib/wukong/store/tokyo_tdb_key_store.rb",
+     "lib/wukong/store/tyrant_rdb_key_store.rb",
+     "lib/wukong/store/tyrant_tdb_key_store.rb",
      "lib/wukong/streamer.rb",
      "lib/wukong/streamer/accumulating_reducer.rb",
      "lib/wukong/streamer/base.rb",
@@ -226,30 +245,31 @@ Gem::Specification.new do |s|
     "spec/spec_helper.rb",
      "spec/wukong/encoding_spec.rb",
      "spec/wukong/script_spec.rb",
+     "examples/binning_percentile_estimator.rb",
+     "examples/contrib/jeans/normalize.rb",
+     "examples/contrib/jeans/sizes.rb",
+     "examples/corpus/words_to_bigrams.rb",
+     "examples/count_keys.rb",
+     "examples/count_keys_at_mapper.rb",
+     "examples/keystore/cassandra_batch_test.rb",
+     "examples/keystore/conditional_outputter_example.rb",
+     "examples/network_graph/adjacency_list.rb",
+     "examples/network_graph/breadth_first_search.rb",
+     "examples/network_graph/gen_2paths.rb",
+     "examples/network_graph/gen_multi_edge.rb",
+     "examples/network_graph/gen_symmetric_links.rb",
      "examples/pagerank/pagerank.rb",
      "examples/pagerank/pagerank_initialize.rb",
+     "examples/rank_and_bin.rb",
      "examples/sample_records.rb",
      "examples/server_logs/apache_log_parser.rb",
      "examples/server_logs/breadcrumbs.rb",
      "examples/server_logs/user_agent.rb",
-     "examples/corpus/words_to_bigrams.rb",
-     "examples/count_keys.rb",
-     "examples/rank_and_bin.rb",
-     "examples/binning_percentile_estimator.rb",
      "examples/size.rb",
-     "examples/network_graph/breadth_first_search.rb",
-     "examples/network_graph/gen_symmetric_links.rb",
-     "examples/network_graph/gen_multi_edge.rb",
-     "examples/network_graph/adjacency_list.rb",
-     "examples/network_graph/gen_2paths.rb",
-     "examples/keystore/cassandra_batch_test.rb",
-     "examples/keystore/conditional_outputter_example.rb",
      "examples/stats/avg_value_frequency.rb",
-     "examples/contrib/jeans/sizes.rb",
-     "examples/contrib/jeans/normalize.rb",
-     "examples/word_count.rb",
+     "examples/store/chunked_store_example.rb",
      "examples/stupidly_simple_filter.rb",
-     "examples/count_keys_at_mapper.rb"
+     "examples/word_count.rb"
   ]
 
   if s.respond_to? :specification_version then

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: wukong
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: false
   segments: 
   - 1
   - 4
-  - 10
-  version: 1.4.10
+  - 11
+  version: 1.4.11
 platform: ruby
 authors: 
 - Philip (flip) Kromer
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-19 00:00:00 +00:00
+date: 2010-07-30 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -136,6 +136,7 @@ files:
 - bin/hdp-kill-task
 - bin/hdp-ls
 - bin/hdp-mkdir
+- bin/hdp-mkdirp
 - bin/hdp-mv
 - bin/hdp-parts_to_keys.rb
 - bin/hdp-ps
@@ -245,9 +246,11 @@ files:
 - examples/size.rb
 - examples/stats/avg_value_frequency.rb
 - examples/stats/data/avg_value_frequency.tsv
+- examples/store/chunked_store_example.rb
 - examples/stupidly_simple_filter.rb
 - examples/word_count.rb
 - lib/wukong.rb
+- lib/wukong/and_pig.rb
 - lib/wukong/bad_record.rb
 - lib/wukong/datatypes.rb
 - lib/wukong/datatypes/enum.rb
@@ -276,12 +279,28 @@ files:
 - lib/wukong/keystore/tyrant_notes.textile
 - lib/wukong/logger.rb
 - lib/wukong/models/graph.rb
+- lib/wukong/monitor.rb
+- lib/wukong/monitor/chunked_store.rb
+- lib/wukong/monitor/periodic_logger.rb
+- lib/wukong/monitor/periodic_monitor.rb
 - lib/wukong/periodic_monitor.rb
 - lib/wukong/rdf.rb
 - lib/wukong/schema.rb
 - lib/wukong/script.rb
 - lib/wukong/script/hadoop_command.rb
 - lib/wukong/script/local_command.rb
+- lib/wukong/store.rb
+- lib/wukong/store/base.rb
+- lib/wukong/store/chunked_flat_file_store.rb
+- lib/wukong/store/conditional_store.rb
+- lib/wukong/store/factory.rb
+- lib/wukong/store/flat_file_store.rb
+- lib/wukong/store/key_store.rb
+- lib/wukong/store/null_store.rb
+- lib/wukong/store/read_thru_store.rb
+- lib/wukong/store/tokyo_tdb_key_store.rb
+- lib/wukong/store/tyrant_rdb_key_store.rb
+- lib/wukong/store/tyrant_tdb_key_store.rb
 - lib/wukong/streamer.rb
 - lib/wukong/streamer/accumulating_reducer.rb
 - lib/wukong/streamer/base.rb
@@ -348,27 +367,28 @@ test_files:
 - spec/spec_helper.rb
 - spec/wukong/encoding_spec.rb
 - spec/wukong/script_spec.rb
+- examples/binning_percentile_estimator.rb
+- examples/contrib/jeans/normalize.rb
+- examples/contrib/jeans/sizes.rb
+- examples/corpus/words_to_bigrams.rb
+- examples/count_keys.rb
+- examples/count_keys_at_mapper.rb
+- examples/keystore/cassandra_batch_test.rb
+- examples/keystore/conditional_outputter_example.rb
+- examples/network_graph/adjacency_list.rb
+- examples/network_graph/breadth_first_search.rb
+- examples/network_graph/gen_2paths.rb
+- examples/network_graph/gen_multi_edge.rb
+- examples/network_graph/gen_symmetric_links.rb
 - examples/pagerank/pagerank.rb
 - examples/pagerank/pagerank_initialize.rb
+- examples/rank_and_bin.rb
 - examples/sample_records.rb
 - examples/server_logs/apache_log_parser.rb
 - examples/server_logs/breadcrumbs.rb
 - examples/server_logs/user_agent.rb
-- examples/corpus/words_to_bigrams.rb
-- examples/count_keys.rb
-- examples/rank_and_bin.rb
-- examples/binning_percentile_estimator.rb
 - examples/size.rb
-- examples/network_graph/breadth_first_search.rb
-- examples/network_graph/gen_symmetric_links.rb
-- examples/network_graph/gen_multi_edge.rb
-- examples/network_graph/adjacency_list.rb
-- examples/network_graph/gen_2paths.rb
-- examples/keystore/cassandra_batch_test.rb
-- examples/keystore/conditional_outputter_example.rb
 - examples/stats/avg_value_frequency.rb
-- examples/contrib/jeans/sizes.rb
-- examples/contrib/jeans/normalize.rb
-- examples/word_count.rb
+- examples/store/chunked_store_example.rb
 - examples/stupidly_simple_filter.rb
-- examples/count_keys_at_mapper.rb
+- examples/word_count.rb