wukong 1.4.9 → 1.4.10

data/lib/wukong/keystore/redis_db.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+require 'rubygems' ;
+require 'redis' ;
+
+RDB = Redis.new(:host => 'localhost', :port => 6379)
+
+start_time = Time.now.utc.to_f ;
+iter=0;
+
+
+$stdin.each do |line|
+  _r, id, scat, sn, pr, fo, fr, st, fv, crat, sid, full = line.chomp.split("\t");
+  iter+=1 ;
+  break if iter > 20_000_000
+
+  if (iter % 10_000 == 0)
+    elapsed = (Time.now.utc.to_f - start_time)
+    puts "%-20s\t%7d\t%7d\t%7.2f\t%7.2f" % [sn, fo, iter, elapsed, iter.to_f/elapsed]
+  end
+
+  RDB['sn:'+sn.downcase] = id unless sn.empty?
+  RDB['sid:'+sid]        = id unless sid.empty?
+  RDB['uid:'+id]         = [sn,sid,crat,scat].join(',') unless id.empty?
+end

data/lib/wukong/keystore/tyrant_db.rb

@@ -0,0 +1,124 @@
+require 'tokyo_tyrant'
+require 'tokyo_tyrant/balancer'
+
+# -- Installing
+#    make sure tokyocabinet and tokyotyrant are installed (cehf recipe)
+#    make sure ruby-tokyotyrant is installed
+#    ldconfig
+#    mkdir -p /data/db/ttyrant /var/run/tyrant /var/log/tyrant
+#
+# -- Starting
+#    ttserver -port 12001 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/user_ids.tch      '/data/db/ttyrant/user_ids.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
+#    ttserver -port 12002 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/screen_names.tch  '/data/db/ttyrant/screen_names.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
+#    ttserver -port 12003 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/search_ids.tch    '/data/db/ttyrant/search_ids.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
+#    ttserver -port 12004 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/tweets_parsed.tch '/data/db/ttyrant/tweets_parsed.tch#bnum=800000000#opts=l#rcnum=50000#xmsiz=268435456'
+#    ttserver -port 12005 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/users_parsed.tch  '/data/db/ttyrant/users_parsed.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
+#
+# -- Monitoring
+#      tcrmgr inform -port $port -st $hostname
+#    active conns:
+#      lsof  -i | grep ttserver | wc -l
+#      netstat -a -W | grep ':120' | ruby -ne 'puts $_.split(/ +/)[3 .. 4].join("\t")' | sort | cut -d: -f1-2 | uniq -c | sort -n
+#    use db.rnum for most lightweight ping method
+#
+# -- Tuning
+#    http://korrespondence.blogspot.com/2009/09/tokyo-tyrant-tuning-parameters.html
+#    http://capttofu.livejournal.com/23381.html
+#    http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/5a46ee04006a791c#
+#    opts     "l" of large option (the size of the database can be larger than 2GB by using 64-bit bucket array.), "d" of Deflate option (each record is compressed with Deflate encoding), "b" of BZIP2 option, "t" of TCBS option
+#    bnum     number of elements of the bucket array. If it is not more than 0, the default value is specified. The default value is 131071 (128K). Suggested size of the bucket array is about from 0.5 to 4 times of the number of all records to be stored.
+#    rcnum    maximum number of records to be cached. If it is not more than 0, the record cache is disabled. It is disabled by default.
+#    xmsiz    size of the extra mapped memory. If it is not more than 0, the extra mapped memory is disabled. The default size is 67108864 (64MB).
+#    apow     size of record alignment by power of 2. If it is negative, the default value is specified. The default value is 4 standing for 2^4=16.
+#    fpow     maximum number of elements of the free block pool by power of 2. If it is negative, the default value is specified. The default value is 10 standing for 2^10=1024.
+#    dfunit   unit step number of auto defragmentation. If it is not more than 0, the auto defragmentation is disabled. It is disabled by default.
+#    mode     "w" of writer, "r" of reader,"c" of creating,"t" of truncating ,"e" of no locking,"f" of non-blocking lock
+#
+# -- Links
+#    http://1978th.net/tokyocabinet/spex-en.html
+#    http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/3bd2a93322c09eec#
+
+
+class TokyoTyrant::Balancer::Base
+  def initialize(hostnames = [], timeout = 20.0, should_retry = true)
+    @servers = hostnames.map do |hostname|
+      host, port = hostname.split(':')
+      klass.new(host, port.to_i, timeout, should_retry)
+    end
+    # yes, for some reason it's spelled 'Constistent' here
+    # DO NOT fix it because it goes deep...
+    @ring = TokyoTyrant::ConstistentHash.new(servers)
+  end
+
+  def close
+    @servers.all?{ |server| server.close rescue nil}
+  end
+
+end
+
+module TokyoDbConnection
+  class TyrantDb
+    attr_reader :dataset
+    DB_SERVERS = [
+      '10.194.101.156',
+      '10.196.73.156',
+      '10.196.75.47',
+      '10.242.217.140',
+    ].freeze unless defined?(TokyoDbConnection::TyrantDb::DB_SERVERS)
+
+    DB_PORTS = {
+      :user_ids      => 12001,
+      :screen_names  => 12002,
+      :search_ids    => 12003,
+      :tweets_parsed => 12004,
+      :users_parsed  => 12005,
+    } unless defined?(TokyoDbConnection::TyrantDb::DB_PORTS)
+
+    def initialize dataset
+      @dataset = dataset
+    end
+
+    def db
+      return @db if @db
+      port = DB_PORTS[dataset] or raise "Don't know how to reach dataset #{dataset}"
+      @db = TokyoTyrant::Balancer::DB.new(DB_SERVERS.map{|s| s+':'+port.to_s})
+      # @db = TokyoTyrant::DB.new(DB_SERVERS.first, port.to_i)
+      @db
+    end
+
+    def [](*args)      ; db[*args]        ; end
+    def size(*args)    ; db.size(*args)   ; end
+    def vanish!(*args) ; db.vanish(*args) ; end
+
+    #
+    # Insert into the cassandra database with default settings
+    #
+    def insert key, value
+      begin
+        db.putnr(key, value)
+      rescue StandardError => e ; handle_error("Insert #{[key, value].inspect}", e); end
+    end
+
+    def insert_array key, value
+      insert(key, value.join(','))
+    end
+
+    def get *args
+      begin
+        db.get(*args)
+      rescue StandardError => e ; handle_error("Fetch #{args.inspect}", e); end
+    end
+
+    def handle_error action, e
+      warn "#{action} failed: #{e} #{e.backtrace.join("\t")}" ;
+      invalidate!
+    end
+
+    def invalidate!
+      (@db && @db.close) or warn "Couldn't close #{@db.inspect}"
+      @db = nil
+      sleep 2
+    end
+  end
+end
+

data/lib/wukong/keystore/tyrant_notes.textile

@@ -0,0 +1,145 @@
+
+# -- Installing
+#    make sure tokyocabinet and tokyotyrant are installed (cehf recipe)
+#    make sure ruby-tokyotyrant is installed
+#    ldconfig
+#    mkdir -p /data/db/ttyrant /var/run/tyrant /var/log/tyrant
+#
+# -- Starting
+#    ttserver -port 12001 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/user_ids.tch      '/data/db/ttyrant/user_ids.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
+#    ttserver -port 12002 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/screen_names.tch  '/data/db/ttyrant/screen_names.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
+#    ttserver -port 12003 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/search_ids.tch    '/data/db/ttyrant/search_ids.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
+#    ttserver -port 12004 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/tweets_parsed.tch '/data/db/ttyrant/tweets_parsed.tch#bnum=800000000#opts=l#rcnum=50000#xmsiz=268435456'
+#    ttserver -port 12005 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/users_parsed.tch  '/data/db/ttyrant/users_parsed.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
+#
+# -- Monitoring
+#      tcrmgr inform -port $port -st $hostname
+#    active conns:
+#      lsof  -i | grep ttserver | wc -l
+#      netstat -a -W | grep ':120' | ruby -ne 'puts $_.split(/ +/)[3 .. 4].join("\t")' | sort | cut -d: -f1-2 | uniq -c | sort -n
+#    use db.rnum for most lightweight ping method
+#
+# -- Tuning
+#    http://korrespondence.blogspot.com/2009/09/tokyo-tyrant-tuning-parameters.html
+#    http://capttofu.livejournal.com/23381.html
+#    http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/5a46ee04006a791c#
+#    opts     "l" of large option (the size of the database can be larger than 2GB by using 64-bit bucket array.), "d" of Deflate option (each record is compressed with Deflate encoding), "b" of BZIP2 option, "t" of TCBS option
+#    bnum     number of elements of the bucket array. If it is not more than 0, the default value is specified. The default value is 131071 (128K). Suggested size of the bucket array is about from 0.5 to 4 times of the number of all records to be stored.
+#    rcnum    maximum number of records to be cached. If it is not more than 0, the record cache is disabled. It is disabled by default.
+#    xmsiz    size of the extra mapped memory. If it is not more than 0, the extra mapped memory is disabled. The default size is 67108864 (64MB).
+#    apow     size of record alignment by power of 2. If it is negative, the default value is specified. The default value is 4 standing for 2^4=16.
+#    fpow     maximum number of elements of the free block pool by power of 2. If it is negative, the default value is specified. The default value is 10 standing for 2^10=1024.
+#    dfunit   unit step number of auto defragmentation. If it is not more than 0, the auto defragmentation is disabled. It is disabled by default.
+#    mode     "w" of writer, "r" of reader,"c" of creating,"t" of truncating ,"e" of no locking,"f" of non-blocking lock
+#
+# -- Links
+#    http://1978th.net/tokyocabinet/spex-en.html
+#    http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/3bd2a93322c09eec#
+# Performance limits: http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/3bd2a93322c09eec#
+
+
+h2. Tyrant: ttserver
+
+  ttdb="test"
+  ttserver -port 12009 -thnum 96  \
+    -dmn -pid /var/run/tyrant-${ttdb}.pid
+    -ulog /mnt/tmp/ttyrant/tyrant-$[ttdb}.ulog -ulim 268435456 -uas \
+    -log  /var/log/ttyrant/tyrant-${ttdb}.log  \
+    "/data/db/ttyrant/${ttdb}.tch#bnum=200000000#opts=l#rcnum=100000#xmsiz=536870912"
+    
+ can also add host, and umask out to be read-only
+
+    * -host name : specify the host name or the address of the server. By default, every network address is bound.
+    * -port num : specify the port number. By default, it is 1978.
+    * -thnum num : specify the number of worker threads. By default, it is 8.
+    * -tout num : specify the timeout of each session in seconds. By default, no timeout is specified.
+    * -dmn : work as a daemon process.
+    * -pid path : output the process ID into the file.
+    * -kl : kill the existing process if the process ID file is detected.
+    * -log path : output log messages into the file.
+    * -ld : log debug messages also.
+    * -le : log error messages only.
+    * -ulog path : specify the update log directory.
+    * -ulim num : specify the limit size of each update log file.
+    * -uas : use asynchronous I/O for the update log.
+    * -sid num : specify the server ID.
+    * -mhost name : specify the host name of the replication master server.
+    * -mport num : specify the port number of the replication master server.
+    * -rts path : specify the replication time stamp file.
+    * -rcc : check consistency of replication.
+    * -skel name : specify the name of the skeleton database library.
+    * -mul num : specify the division number of the multiple database mechanism.
+    * -ext path : specify the script language extension file.
+    * -extpc name period : specify the function name and the calling period of a periodic command.
+    * -mask expr : specify the names of forbidden commands.
+    * -unmask expr : specify the names of allowed commands.
+
+
+h2. From "Wolfgang Gassler":http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/5a46ee04006a791c#
+
+On Sat, Dec 05, 2009 at 09:32:20PM +0100, Wolfgang Gassler wrote:
+> Hi,
+
+> did anybody look up some of the folowing parameters in the code or can
+> explain them in detail? I just have a guess what they really mean and
+> the short description at the docu homepage
+> http://korrespondence.blogspot.com/2009/09/tokyo-tyrant-tuning-parame...
+> explain them very roughly. Also the already posted blog post
+> http://korrespondence.blogspot.com/2009/09/tokyo-tyrant-tuning-parame...
+> couldn't help.
+
+this is what I gleaned from reading the source code for the hash database
+format ( tchdb.c and tchdb.h ).
+
+> xmsiz
+
+On a TC Hash database, from the beginning of the file, to the end of the bucket
+section, all of that space is mmap'd.  Setting 'xmsiz' sets the minimum amount
+of space that is mmap'd.  Since 67108864 is the default, this means, that an a
+minimum, the first 64MiB of the file will be mmap'd.
+
+If the header size, plus the bucket region is greater than 'xmsize', then xmsiz
+appers to have no affect.
+
+> apow
+
+On a TC Hash database, 'apow' determines on what byte alignment each record will
+sit.  'apow' is a power of 2.  This means that when apow is 4 ( the default for
+hash databases) all records in the database are aligned on a 16 byte boundary,
+in the database file.
+
+This means that every record will take up at a minumum 16 bytes of space, and
+all records are padded to a length that is a multiple of 16.
+
+> fpow
+
+On a TC Hash database, 'fpow' determines the maximum number of free blocks that
+can exist in the free block pool.  This is also a power-of-2 parameter so with
+the default in a Hash database of 10, this means that there can be a maximum
+of 2^10, or 1024 free blocks in the database.
+
+Free blocks come into existence when records are deleted from the database
+and their space in the db file is up for reuse.  If you never delete an
+item from the database, you will never have any free blocks.
+
+> dfunit
+
+On a TC Hash database, 'dfunit' describes how defragmentation takes place.
+Every time a free block is created a 'dfcnt' is incremented.  When 'dfcnt'
+is greater than 'dfunit' and 'dfunit' is greater than 0, defragmentation
+takes place.
+
+I don't know precisely what defragmentation does in TC. A cursory look
+at 'tchdbdefragimpl', the function implementing defagmentation for hash
+databases, it looks like it moves records around filling up free blocks
+in the hash db with real records from the end of the file and then making the
+file smaller if possible.
+
+Basically it moves records around minimizing dead space in the file.
+
+Again, defragmentation will only take place if 'dfunit' has a positive
+value and you remove records from the db creating free blocks.
+
+enjoy,
+
+-jeremy 

data/lib/wukong/periodic_monitor.rb

@@ -0,0 +1,57 @@
+Settings.define :log_interval, :default => 1000, :type => Integer, :description => 'How many iterations between log statements'
+
+#
+# Periodic monitor
+#
+#
+# This is very much a work in progress
+#
+class PeriodicMonitor
+  attr_reader   :iter, :start_time, :options
+  attr_accessor :interval
+
+  def initialize extra_options={}
+    @options      = {}
+    @options.deep_merge!( extra_options || {} )
+    @iter         = 0
+    @start_time   = now
+    @interval     = (options[:log_interval] || Settings[:log_interval]).to_i
+    @interval = 1000 unless @interval >= 1
+  end
+
+  def periodically *args, &block
+    incr!
+    if ready?
+      if block
+        block.call(iter, *args)
+      else
+        $stderr.puts progress(*args)
+      end
+    end
+  end
+
+  def incr!
+    @iter += 1
+  end
+
+  def ready?
+    iter % @interval == 0
+  end
+
+  def progress *stuff
+    [
+      "%15d" % iter,
+      "%7.1f"% elapsed_time, "sec",
+      "%7.1f"%(iter.to_f / elapsed_time), "/sec",
+      now.to_flat,
+      *stuff
+    ].flatten.join("\t")
+  end
+
+  def elapsed_time
+    now - start_time
+  end
+  def now
+    Time.now.utc
+  end
+end

data/lib/wukong/script/hadoop_command.rb

@@ -27,6 +27,8 @@ module Wukong
     Settings.define :timeout,                :jobconf => true, :description => 'mapred.task.timeout', :wukong => true
     Settings.define :reuse_jvms,             :jobconf => true, :description => 'mapred.job.reuse.jvm.num.tasks', :wukong => true
     Settings.define :respect_exit_status,    :jobconf => true, :description => 'stream.non.zero.exit.is.failure', :wukong => true
+    Settings.define :io_sort_record_percent, :jobconf => true, :description => 'io.sort.record.percent', :wukong => true
+    Settings.define :io_sort_mb,             :jobconf => true, :description => 'io.sort.mb', :wukong => true
     Settings.define :noempty,                                  :description => "don't create zero-byte reduce files (hadoop mode only)", :wukong => true
     Settings.define :job_name,               :jobconf => true, :description => 'mapred.job.name', :wukong => true
     # mapred.linerecordreader.maxlength :description => "Safeguards against corrupted data: lines longer than this (in bytes) are treated as bad records."
@@ -98,7 +100,7 @@ module Wukong
       # root of your config install.
       [
         hadoop_runner,
-        "jar #{Settings[:hadoop_home]}/contrib/streaming/hadoop-*-streaming.jar",
+        "jar #{Settings[:hadoop_home]}/contrib/streaming/hadoop-*streaming*.jar",
         hadoop_partition_args,
         hadoop_sort_args,
         hadoop_num_tasks_args,

data/lib/wukong/streamer/accumulating_reducer.rb

@@ -83,6 +83,7 @@
          super
          # don't finalize if we never saw any field at all
          finalize(){|record| emit record } unless (self.key == :__first_pass__)
+         after_stream
        end
      end
 

data/lib/wukong/streamer/cassandra_streamer.rb

@@ -0,0 +1,61 @@
+# Defines a base class for streaming data into a cassandra db connection.
+require 'cassandra' ; include Cassandra::Constants
+module Wukong
+  module Streamer
+
+    class CassandraStreamer < Wukong::Streamer::Base
+      attr_accessor :batch_count, :batch_record_count, :batch_size, :column_space, :db_seeds, :cassandra_db
+
+      def initialize *args
+        super *args
+        self.batch_count = 0
+        self.batch_record_count = 0
+        self.column_space ||= 'Twitter'
+        self.batch_size   ||= 100
+        self.db_seeds     ||= %w[10.244.191.178 10.243.19.223 10.243.17.219 10.245.70.85 10.244.206.241].map{ |s| s.to_s+':9160'}
+        self.cassandra_db ||= Cassandra.new(self.column_space, self.db_seeds)
+      end
+
+      def stream
+        while still_lines? do
+          start_batch do
+            while still_lines? && batch_not_full? do
+              line = get_line
+              record = recordize(line.chomp) or next
+              next if record.blank?
+              process(*record) do |output_record|
+                emit output_record
+              end
+              self.batch_record_count += 1
+            end
+          end
+        end
+      end
+
+      def process *args, &blk
+        Raise "Overwrite this method to insert into cassandra db"
+      end
+
+      def start_batch &blk
+        self.batch_record_count = 0
+        self.batch_count += 1
+        self.cassandra_db.batch(&blk)
+      end
+
+      def get_line
+        $stdin.gets
+      end
+
+      def still_lines?
+        !$stdin.eof?
+      end
+
+      def batch_not_full?
+        self.batch_record_count < self.batch_size
+      end
+
+    end
+  end
+
+end
+

data/lib/wukong/streamer.rb

@@ -1,16 +1,18 @@
 module Wukong
   module Streamer
-    autoload :Base,                 'wukong/streamer/base'
-    autoload :LineStreamer,         'wukong/streamer/line_streamer'
-    autoload :RecordStreamer,       'wukong/streamer/record_streamer'
-    autoload :StructStreamer,       'wukong/streamer/struct_streamer'
-    autoload :StructRecordizer,     'wukong/streamer/struct_streamer'
+    autoload :Base,                    'wukong/streamer/base'
+    autoload :LineStreamer,            'wukong/streamer/line_streamer'
+    autoload :RecordStreamer,          'wukong/streamer/record_streamer'
+    autoload :StructStreamer,          'wukong/streamer/struct_streamer'
+    autoload :StructRecordizer,        'wukong/streamer/struct_streamer'
+    # cassandra goodies
+    autoload :CassandraStreamer,       'wukong/streamer/cassandra_streamer'
     #
-    autoload :Filter,               'wukong/streamer/filter'
+    autoload :Filter,                  'wukong/streamer/filter'
     #
-    autoload :AccumulatingReducer,  'wukong/streamer/accumulating_reducer'
-    autoload :ListReducer,          'wukong/streamer/list_reducer'
-    autoload :UniqByLastReducer,    'wukong/streamer/uniq_by_last_reducer'
-    autoload :CountingReducer,      'wukong/streamer/counting_reducer'
+    autoload :AccumulatingReducer,     'wukong/streamer/accumulating_reducer'
+    autoload :ListReducer,             'wukong/streamer/list_reducer'
+    autoload :UniqByLastReducer,       'wukong/streamer/uniq_by_last_reducer'
+    autoload :CountingReducer,         'wukong/streamer/counting_reducer'
   end
 end

data/wukong.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{wukong}
-  s.version = "1.4.9"
+  s.version = "1.4.10"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer"]
-  s.date = %q{2010-06-06}
+  s.date = %q{2010-07-19}
   s.description = %q{  Treat your dataset like a:
 
       * stream of lines when it's efficient to process by lines
@@ -29,6 +29,7 @@ Gem::Specification.new do |s|
      "INSTALL.textile",
      "LICENSE.textile",
      "README.textile",
+     "TODO.textile",
      "bin/cutc",
      "bin/cuttab",
      "bin/greptrue",
@@ -38,6 +39,7 @@ Gem::Specification.new do |s|
      "bin/hdp-du",
      "bin/hdp-get",
      "bin/hdp-kill",
+     "bin/hdp-kill-task",
      "bin/hdp-ls",
      "bin/hdp-mkdir",
      "bin/hdp-mv",
@@ -62,6 +64,7 @@ Gem::Specification.new do |s|
      "bin/wu-sum",
      "docpages/INSTALL.textile",
      "docpages/LICENSE.textile",
+     "docpages/README-performance.textile",
      "docpages/README-wulign.textile",
      "docpages/UsingWukong-part1-get_ready.textile",
      "docpages/UsingWukong-part2-ThinkingBigData.textile",
@@ -118,14 +121,17 @@ Gem::Specification.new do |s|
      "docpages/usage.textile",
      "docpages/wutils.textile",
      "examples/README.txt",
+     "examples/binning_percentile_estimator.rb",
      "examples/contrib/jeans/README.markdown",
      "examples/contrib/jeans/data/normalized_sizes",
      "examples/contrib/jeans/data/orders.tsv",
      "examples/contrib/jeans/data/sizes",
      "examples/contrib/jeans/normalize.rb",
      "examples/contrib/jeans/sizes.rb",
+     "examples/corpus/words_to_bigrams.rb",
      "examples/count_keys.rb",
      "examples/count_keys_at_mapper.rb",
+     "examples/keystore/cassandra_batch_test.rb",
      "examples/keystore/conditional_outputter_example.rb",
      "examples/network_graph/adjacency_list.rb",
      "examples/network_graph/breadth_first_search.rb",
@@ -160,6 +166,7 @@ Gem::Specification.new do |s|
      "lib/wukong/extensions/class.rb",
      "lib/wukong/extensions/date_time.rb",
      "lib/wukong/extensions/emittable.rb",
+     "lib/wukong/extensions/enumerable.rb",
      "lib/wukong/extensions/hash.rb",
      "lib/wukong/extensions/hash_keys.rb",
      "lib/wukong/extensions/hash_like.rb",
@@ -170,8 +177,12 @@ Gem::Specification.new do |s|
      "lib/wukong/extensions/struct.rb",
      "lib/wukong/extensions/symbol.rb",
      "lib/wukong/keystore/cassandra_conditional_outputter.rb",
+     "lib/wukong/keystore/redis_db.rb",
+     "lib/wukong/keystore/tyrant_db.rb",
+     "lib/wukong/keystore/tyrant_notes.textile",
      "lib/wukong/logger.rb",
      "lib/wukong/models/graph.rb",
+     "lib/wukong/periodic_monitor.rb",
      "lib/wukong/rdf.rb",
      "lib/wukong/schema.rb",
      "lib/wukong/script.rb",
@@ -180,6 +191,7 @@ Gem::Specification.new do |s|
      "lib/wukong/streamer.rb",
      "lib/wukong/streamer/accumulating_reducer.rb",
      "lib/wukong/streamer/base.rb",
+     "lib/wukong/streamer/cassandra_streamer.rb",
      "lib/wukong/streamer/count_keys.rb",
      "lib/wukong/streamer/count_lines.rb",
      "lib/wukong/streamer/counting_reducer.rb",
@@ -208,51 +220,56 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://mrflip.github.com/wukong}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use it, yet handles terabyte-scale computation with ease.}
   s.test_files = [
     "spec/spec_helper.rb",
      "spec/wukong/encoding_spec.rb",
      "spec/wukong/script_spec.rb",
-     "examples/contrib/jeans/normalize.rb",
-     "examples/contrib/jeans/sizes.rb",
-     "examples/count_keys.rb",
-     "examples/count_keys_at_mapper.rb",
-     "examples/keystore/conditional_outputter_example.rb",
-     "examples/network_graph/adjacency_list.rb",
-     "examples/network_graph/breadth_first_search.rb",
-     "examples/network_graph/gen_2paths.rb",
-     "examples/network_graph/gen_multi_edge.rb",
-     "examples/network_graph/gen_symmetric_links.rb",
      "examples/pagerank/pagerank.rb",
      "examples/pagerank/pagerank_initialize.rb",
-     "examples/rank_and_bin.rb",
      "examples/sample_records.rb",
      "examples/server_logs/apache_log_parser.rb",
      "examples/server_logs/breadcrumbs.rb",
      "examples/server_logs/user_agent.rb",
+     "examples/corpus/words_to_bigrams.rb",
+     "examples/count_keys.rb",
+     "examples/rank_and_bin.rb",
+     "examples/binning_percentile_estimator.rb",
      "examples/size.rb",
+     "examples/network_graph/breadth_first_search.rb",
+     "examples/network_graph/gen_symmetric_links.rb",
+     "examples/network_graph/gen_multi_edge.rb",
+     "examples/network_graph/adjacency_list.rb",
+     "examples/network_graph/gen_2paths.rb",
+     "examples/keystore/cassandra_batch_test.rb",
+     "examples/keystore/conditional_outputter_example.rb",
      "examples/stats/avg_value_frequency.rb",
+     "examples/contrib/jeans/sizes.rb",
+     "examples/contrib/jeans/normalize.rb",
+     "examples/word_count.rb",
      "examples/stupidly_simple_filter.rb",
-     "examples/word_count.rb"
+     "examples/count_keys_at_mapper.rb"
   ]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_development_dependency(%q<yard>, [">= 0"])
       s.add_runtime_dependency(%q<addressable>, [">= 0"])
       s.add_runtime_dependency(%q<extlib>, [">= 0"])
       s.add_runtime_dependency(%q<htmlentities>, [">= 0"])
+      s.add_runtime_dependency(%q<configliere>, [">= 0"])
     else
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_dependency(%q<yard>, [">= 0"])
       s.add_dependency(%q<addressable>, [">= 0"])
       s.add_dependency(%q<extlib>, [">= 0"])
       s.add_dependency(%q<htmlentities>, [">= 0"])
+      s.add_dependency(%q<configliere>, [">= 0"])
     end
   else
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
@@ -260,6 +277,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<addressable>, [">= 0"])
     s.add_dependency(%q<extlib>, [">= 0"])
     s.add_dependency(%q<htmlentities>, [">= 0"])
+    s.add_dependency(%q<configliere>, [">= 0"])
   end
 end