wukong 1.5.4 → 2.0.0

data/examples/keystore/cassandra_batch_test.rb

@@ -1,41 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong'
-
-# An example (and test) of streaming batches of data into distributed cassandra db
-# Stream in whatever you like that has a key and value. Notice that you must
-# have already defined a column space called 'Cruft' in storage-conf.xml as well
-# as a column family called 'OhBaby'
-
-class Mapper < Wukong::Streamer::CassandraStreamer
-
-  # you must redefine the column space, batch size, and db-seeds  or they will
-  # be defaults. For testing on local machine simply seed db with 127.0.0.1:9160
-
-  def initialize *args
-    self.column_space = 'Cruft'
-    self.batch_size = 100
-    self.db_seeds = "127.0.0.1:9160"
-    super(*args)
-    @iter = 0
-  end
-
-  def process key, value, *_, &blk
-    insert_into_db(key, value)
-    yield [key, value] if (@iter %10 == 0)
-  end
-
-  # you must specify the column family, key, and value here
-  def insert_into_db key, value
-    @iter += 1
-    cassandra_db.insert(:OhBaby, key, {"value" => value}, :consistency => Cassandra::Consistency::ANY) unless key.blank?
-  end
-end
-
-#
-# Executes the script
-#
-Wukong::Script.new(
-  Mapper,
-  nil
-).run

data/examples/keystore/conditional_outputter_example.rb

@@ -1,70 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'cassandra'
-require 'wukong'
-require 'wukong/encoding'
-require 'wukong/keystore/cassandra_conditional_outputter'
-
-#
-# Usage:
-#   echo -e "bob has boobs ha ha ha" | ./examples/keystore/conditional_outputter_example.rb  --map
-#
-
-CASSANDRA_KEYSPACE = 'CorpusAnalysis'
-
-#
-# This demonstrates the CassandraConditionalOutputter module.
-#
-# CassandraConditionalOutputter uses and a cassandra key-value store to
-# track unique IDs and prevent output of any record already present in the
-# database.
-#
-# For this example, it takes an input stream, generates all letter pairs for
-# each line, and emits
-#
-#
-class LetterPairMapper <  Wukong::Streamer::LineStreamer
-  include CassandraConditionalOutputter
-
-  #
-  # A unique key for the given record. If an object with
-  # that key has been seen, it won't be re-emitted.
-  #
-  # In this example, we'll just encode the letter pair
-  #
-  def conditional_output_key record
-    record.to_s.wukong_encode(:url)
-  end
-
-  #
-  # Emit each letter pair in the line.
-  # the CassandraConditionalOutputter will swallow all duplicate lines.
-  #
-  def process line, &block
-    letter_pairs(line).each do |pair|
-      yield(pair)
-    end
-  end
-
-  # turn a string into the pairs of adjacent letters
-  #
-  # @example
-  #   letter_pairs('abracadabra')
-  #   # => ['ab', 'br',
-  def letter_pairs str, &block
-    chars = str.chars.to_a
-    chars[0..-2].zip(chars[1..-1]).map(&:join)
-  end
-
-  # Clear the entire cached keys column at the end of the run.
-  #
-  # You almost certainly don't want to do this in a real script.
-  #
-  def after_stream
-    $stderr.puts 'Clearing conditional_output_key cache...'
-    @key_cache.clear_column_family!(conditional_output_key_column)
-  end
-end
-
-# Execute the script
-Wukong::Script.new( LetterPairMapper, nil ).run

data/examples/store/chunked_store_example.rb

@@ -1,18 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong'
-# require 'wukong/store'
-
-require 'configliere'
-Configliere.use :commandline, :define, :config_file
-Settings.read('foo.yaml')
-
-# store = ChunkedFlatFileStore.new(Settings)
-
-100.times do |iter|
-  # store.save   [iter, Time.now.to_flat].join("\t")
-  $stdout.puts [iter, Time.now.to_flat].join("\t")
-  sleep 2
-end
-
-

data/lib/wukong/dfs.rb

@@ -1,81 +0,0 @@
-require 'time' # ain't it always that way
-module Wukong
-  module Dfs
-    def self.list_files dfs_path
-      Log.info{ "DFS: listing #{dfs_path}" }
-      listing = `hadoop dfs -ls #{dfs_path}`.split("\n").reject{|ls_line| ls_line =~ /Found \d+ items/i}
-      listing.map{|ls_line| HFile.new_from_ls(ls_line)}
-    end
-
-    #
-    # FIXME -- this will fail if multiple files in a listing have the
-    # same basename. Sorry.
-    #
-    def self.compare_listings src_files, dest_files, &block
-      src_files.sort.each do |src_file|
-        dest_file = dest_files.find{|df| File.basename(src_file) == df.basename }
-        case
-        when (! dest_file)                       then yield :missing, src_file, nil
-        when (! dest_file.kinda_equal(src_file)) then yield :differ,  src_file, dest_file
-        else                                          yield :same,    src_file, dest_file
-        end
-      end
-    end
-
-    HFile = TypedStruct.new(
-      [:mode_str,     String],
-      [:i_count,      String],
-      [:owner,        String],
-      [:group,        String],
-      [:size,         Integer],
-      [:date,         Bignum],
-      [:path,     String]
-      )
-    HFile.class_eval do
-      def self.new_from_ls ls_line
-        mode, ic, o, g, sz, dt, tm, path = ls_line.chomp.split(/\s+/)
-        date = Time.parse("#{dt} #{tm}").utc.to_flat
-        new mode, ic.to_i, o, g, sz.to_i, date, path
-      end
-      def dirname
-        @dirname ||= File.dirname(path)
-      end
-      def basename
-        @basename ||= File.basename(path)
-      end
-      #
-      # Two files are kinda_equal if they match in size and if
-      # the hdfs version is later than the filesystem version.
-      #
-      def kinda_equal file
-        (self.size == File.size(file)) # && (self.date >= File.mtime(file).utc.to_flat)
-      end
-      def to_s
-        to_a.join("\t")
-      end
-
-      #
-      # These will be very slow.
-      # If some kind soul will integrate JRuby callouts the bards shall
-      # celebrate your name evermore.
-      #
-
-      # rename the file on the HDFS
-      def mv new_filename
-        self.class.run_dfs_command :mv, path, new_filename
-      end
-
-      def self.mkdir dirname
-        run_dfs_command :mkdir, dirname
-      end
-      def self.mkdir_p(*args) self.mkdir *args ; end # HDFS is always -p
-
-      def self.run_dfs_command *args
-        cmd = 'hadoop dfs -'+ args.flatten.compact.join(" ")
-        Log.debug{ "DFS: Running #{cmd}" }
-        Log.info{ `#{cmd} 2>&1`.gsub(/[\r\n\t]+/, " ") }
-      end
-
-    end
-  end
-end

data/lib/wukong/keystore/cassandra_conditional_outputter.rb

@@ -1,122 +0,0 @@
-
-#
-# For a stream process that sees a significant number of duplicated heavyweight
-# objects, it may be better to deduplicate them midflight (rather than, say,
-# using a reducer to effectively `cat | sort | uniq` the data).
-#
-# This uses a cassandra key-value store to track unique IDs and prevent output
-# of any record already present in the database.  (Why cassandra? Because we use
-# it in production.  Might be nice to rewrite this example against redis or
-# TokyoTyrant or something less demanding.)
-#
-# Things you have to do:
-#
-# * Override the conditional_output_key method to distinguish identical records
-# * Define a constant CASSANDRA_KEYSPACE giving the Cassandra keyspace you're working in
-# * (Optionally) override conditional_output_key_column
-#
-# * In your cassandra storage-conf.xml, add a column family to your keyspace:
-#
-#     <Keyspace Name="CorpusAnalysis">
-#         <KeysCachedFraction>0.01</KeysCachedFraction>
-#
-#         <!-- Added for CassandraConditionalOutputter -->
-#         <ColumnFamily CompareWith="UTF8Type" Name="LetterPairMapperKeys" />
-#
-#         <ReplicaPlacementStrategy>org.apache.cassandra.locator.RackUnawareStrategy</ReplicaPlacementStrategy>
-#         <ReplicationFactor>1</ReplicationFactor>
-#         <EndPointSnitch>org.apache.cassandra.locator.EndPointSnitch</EndPointSnitch>
-#     </Keyspace>
-#
-#  In this example, the CASSANDRA_KEYSPACE is 'CorpusAnalysis' and the
-#  conditional_output_key_column is 'LetterPairMapperKeys'
-#
-# @example
-#    Given
-#      tweet  123456789   20100102030405     @frank: I'm having a bacon sandwich
-#      tweet      24601   20100104136526     @jerry, I'm having your baby
-#      tweet    8675309   20100102030405     I find pastrami to be the most sensual of the salted, cured meats.
-#      tweet      24601   20100104136526     @jerry, I'm having your baby
-#      tweet       1137   20100119234532     These pretzels are making me thirsty
-#      ....
-#    will emit:
-#      tweet  123456789   20100102030405     @frank: I'm having a bacon sandwich
-#      tweet      24601   20100104136526     @jerry, I'm having your baby
-#      tweet    8675309   20100102030405     I find pastrami to be the most sensual of the salted, cured meats.
-#      tweet      24601   20100104136526     @jerry, I'm having your baby
-#      tweet       1137   20100119234532     These pretzels are making me thirsty
-#      ....
-#
-module CassandraConditionalOutputter
-
-  #
-  # A unique key for the given record. If an object with
-  # that key has been seen, it won't be re-emitted.
-  #
-  # You will almost certainly want to override this method in your subclass.  Be
-  # sure that the key is a string, and is encoded properly (Cassandra likes to
-  # strip whitespace from keys, for instance).
-  #
-  def conditional_output_key record
-    record.to_s
-  end
-
-  #
-  # Checks each record against the key cache
-  # Swallows records already there,
-  #
-  #
-  def emit record, &block
-    key = conditional_output_key(record)
-    if should_emit?(record)
-      set_key(key, {'t' => record.timestamp})
-      super record
-    end
-  end
-
-  # Default. Emit record if its key is not already contained
-  # in the key-value store. Overwrite this as necessary
-  def should_emit? record
-    key = conditional_output_key(record)
-    !has_key?(key)
-  end
-
-  # Check for presence of key in the cache
-  def has_key? key
-    not key_cache.get(conditional_output_key_column, key).blank?
-  end
-
-  # register key in the key_cache
-  def set_key key, data={'t' => '0'}
-    key_cache.insert(conditional_output_key_column, key, data)
-  end
-
-  # nuke key from the key_cache
-  def remove_key key
-    key_cache.remove(conditional_output_key_column, key)
-  end
-
-  #
-  # Key cache implementation in Cassandra
-  #
-
-  # The cache
-  def key_cache
-    @key_cache ||= Cassandra.new(CASSANDRA_KEYSPACE)
-  end
-
-  # The column to use for the key cache. By default, the class name plus 'Keys',
-  # but feel free to override.
-  #
-  # @example
-  #
-  #    class FooMapper < Wukong::Streamer::RecordStreamer
-  #      include ConditionalOutputter
-  #    end
-  #    FooMapper.new.conditional_output_key_column
-  #    # => 'FooMapperKeys'
-  #
-  def conditional_output_key_column
-    self.class.to_s+'Keys'
-  end
-end

data/lib/wukong/keystore/redis_db.rb

@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems' ;
-require 'redis' ;
-
-RDB = Redis.new(:host => 'localhost', :port => 6379)
-
-start_time = Time.now.utc.to_f ;
-iter=0;
-
-
-$stdin.each do |line|
-  _r, id, scat, sn, pr, fo, fr, st, fv, crat, sid, full = line.chomp.split("\t");
-  iter+=1 ;
-  break if iter > 20_000_000
-
-  if (iter % 10_000 == 0)
-    elapsed = (Time.now.utc.to_f - start_time)
-    puts "%-20s\t%7d\t%7d\t%7.2f\t%7.2f" % [sn, fo, iter, elapsed, iter.to_f/elapsed]
-  end
-
-  RDB['sn:'+sn.downcase] = id unless sn.empty?
-  RDB['sid:'+sid]        = id unless sid.empty?
-  RDB['uid:'+id]         = [sn,sid,crat,scat].join(',') unless id.empty?
-end

data/lib/wukong/keystore/tyrant_db.rb

@@ -1,137 +0,0 @@
-require 'tokyo_tyrant'
-require 'tokyo_tyrant/balancer'
-
-# -- Installing
-#    make sure tokyocabinet and tokyotyrant are installed (cehf recipe)
-#    make sure ruby-tokyotyrant is installed
-#    ldconfig
-#    mkdir -p /data/db/ttyrant /var/run/tyrant /var/log/tyrant
-#
-# -- Starting
-#    ttserver -port 12001 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/user_ids.tch      '/data/db/ttyrant/user_ids.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
-#    ttserver -port 12002 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/screen_names.tch  '/data/db/ttyrant/screen_names.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
-#    ttserver -port 12003 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/search_ids.tch    '/data/db/ttyrant/search_ids.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
-#    ttserver -port 12004 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/tweets_parsed.tch '/data/db/ttyrant/tweets_parsed.tch#bnum=800000000#opts=l#rcnum=50000#xmsiz=268435456'
-#    ttserver -port 12005 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/users_parsed.tch  '/data/db/ttyrant/users_parsed.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
-#
-# -- Monitoring
-#      tcrmgr inform -port $port -st $hostname
-#    active conns:
-#      lsof  -i | grep ttserver | wc -l
-#      netstat -a -W | grep ':120' | ruby -ne 'puts $_.split(/ +/)[3 .. 4].join("\t")' | sort | cut -d: -f1-2 | uniq -c | sort -n
-#    use db.rnum for most lightweight ping method
-#
-# -- Tuning
-#    http://korrespondence.blogspot.com/2009/09/tokyo-tyrant-tuning-parameters.html
-#    http://capttofu.livejournal.com/23381.html
-#    http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/5a46ee04006a791c#
-#    opts     "l" of large option (the size of the database can be larger than 2GB by using 64-bit bucket array.), "d" of Deflate option (each record is compressed with Deflate encoding), "b" of BZIP2 option, "t" of TCBS option
-#    bnum     number of elements of the bucket array. If it is not more than 0, the default value is specified. The default value is 131071 (128K). Suggested size of the bucket array is about from 0.5 to 4 times of the number of all records to be stored.
-#    rcnum    maximum number of records to be cached. If it is not more than 0, the record cache is disabled. It is disabled by default.
-#    xmsiz    size of the extra mapped memory. If it is not more than 0, the extra mapped memory is disabled. The default size is 67108864 (64MB).
-#    apow     size of record alignment by power of 2. If it is negative, the default value is specified. The default value is 4 standing for 2^4=16.
-#    fpow     maximum number of elements of the free block pool by power of 2. If it is negative, the default value is specified. The default value is 10 standing for 2^10=1024.
-#    dfunit   unit step number of auto defragmentation. If it is not more than 0, the auto defragmentation is disabled. It is disabled by default.
-#    mode     "w" of writer, "r" of reader,"c" of creating,"t" of truncating ,"e" of no locking,"f" of non-blocking lock
-#
-# -- Links
-#    http://1978th.net/tokyocabinet/spex-en.html
-#    http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/3bd2a93322c09eec#
-
-
-class TokyoTyrant::Balancer::Base
-  def initialize(hostnames = [], timeout = 20.0, should_retry = true)
-    @servers = hostnames.map do |hostname|
-      host, port = hostname.split(':')
-      klass.new(host, port.to_i, timeout, should_retry)
-    end
-    # yes, for some reason it's spelled 'Constistent' here
-    # DO NOT fix it because it goes deep...
-    @ring = TokyoTyrant::ConstistentHash.new(servers)
-  end
-
-  def close
-    @servers.all?{ |server| server.close rescue nil}
-  end
-end
-
-module TokyoDbConnection
-  class TyrantDb
-    attr_reader :dataset
-    DB_SERVERS = [
-      '10.194.101.156',
-      '10.196.73.156',
-      '10.196.75.47',
-      '10.242.217.140',
-    ].freeze unless defined?(TokyoDbConnection::TyrantDb::DB_SERVERS)
-
-    DB_PORTS = {
-      :tw_screen_names  => 12002,
-      :tw_search_ids    => 12003,
-      #
-      :tw_user_info     => 14000,
-      :tw_wordbag       => 14101,
-      :tw_influence     => 14102,
-      :tw_trstrank      => 14103,
-      :tw_conversation  => 14104,
-      #
-      :tw_screen_names2 => 12004,
-      :tw_search_ids2   => 12005,
-      #
-      :tw_user_info2    => 14200,
-      :tw_wordbag2      => 14201,
-      :tw_influence2    => 14202,
-      :tw_trstrank2     => 14203,
-      :tw_conversation2 => 14204,
-      :tw_strong_links2 => 14205,
-      :tw_word_stats2   => 14210,
-      #
-      :ip_geo_census    => 14400,
-    } unless defined?(TokyoDbConnection::TyrantDb::DB_PORTS)
-
-    def initialize dataset
-      @dataset = dataset
-    end
-
-    def db
-      return @db if @db
-      port = DB_PORTS[dataset] or raise "Don't know how to reach dataset #{dataset}"
-      @db = TokyoTyrant::Balancer::DB.new(DB_SERVERS.map{|s| s+':'+port.to_s})
-    end
-
-    def [](*args)      ; db[*args]        ; end
-    def size(*args)    ; db.size(*args)   ; end
-    def vanish!(*args) ; db.vanish(*args) ; end
-
-    #
-    # Insert into the cassandra database with default settings
-    #
-    def insert key, value
-      begin
-        db.putnr(key, value)
-      rescue StandardError => e ; handle_error("Insert #{[key, value].inspect}", e); end
-    end
-
-    def insert_array key, value
-      insert(key, value.join(','))
-    end
-
-    def get *args
-      begin
-        db.get(*args)
-      rescue StandardError => e ; handle_error("Fetch #{args.inspect}", e); end
-    end
-
-    def handle_error action, e
-      Log.warn "#{action} failed: #{e} #{e.backtrace.join("\t")}" ;
-      invalidate!
-    end
-
-    def invalidate!
-      (@db && @db.close) or warn "Couldn't close #{@db.inspect}"
-      @db = nil
-      sleep 2
-    end
-  end
-end
-

data/lib/wukong/keystore/tyrant_notes.textile

@@ -1,145 +0,0 @@
-
-# -- Installing
-#    make sure tokyocabinet and tokyotyrant are installed (cehf recipe)
-#    make sure ruby-tokyotyrant is installed
-#    ldconfig
-#    mkdir -p /data/db/ttyrant /var/run/tyrant /var/log/tyrant
-#
-# -- Starting
-#    ttserver -port 12001 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/user_ids.tch      '/data/db/ttyrant/user_ids.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
-#    ttserver -port 12002 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/screen_names.tch  '/data/db/ttyrant/screen_names.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
-#    ttserver -port 12003 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/search_ids.tch    '/data/db/ttyrant/search_ids.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
-#    ttserver -port 12004 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/tweets_parsed.tch '/data/db/ttyrant/tweets_parsed.tch#bnum=800000000#opts=l#rcnum=50000#xmsiz=268435456'
-#    ttserver -port 12005 -thnum 96 -tout 3 -pid /var/run/tyrant/screen_names.pid -kl -log /var/log/tyrant/users_parsed.tch  '/data/db/ttyrant/users_parsed.tch#bnum=100000000#opts=l#rcnum=50000#xmsiz=268435456'
-#
-# -- Monitoring
-#      tcrmgr inform -port $port -st $hostname
-#    active conns:
-#      lsof  -i | grep ttserver | wc -l
-#      netstat -a -W | grep ':120' | ruby -ne 'puts $_.split(/ +/)[3 .. 4].join("\t")' | sort | cut -d: -f1-2 | uniq -c | sort -n
-#    use db.rnum for most lightweight ping method
-#
-# -- Tuning
-#    http://korrespondence.blogspot.com/2009/09/tokyo-tyrant-tuning-parameters.html
-#    http://capttofu.livejournal.com/23381.html
-#    http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/5a46ee04006a791c#
-#    opts     "l" of large option (the size of the database can be larger than 2GB by using 64-bit bucket array.), "d" of Deflate option (each record is compressed with Deflate encoding), "b" of BZIP2 option, "t" of TCBS option
-#    bnum     number of elements of the bucket array. If it is not more than 0, the default value is specified. The default value is 131071 (128K). Suggested size of the bucket array is about from 0.5 to 4 times of the number of all records to be stored.
-#    rcnum    maximum number of records to be cached. If it is not more than 0, the record cache is disabled. It is disabled by default.
-#    xmsiz    size of the extra mapped memory. If it is not more than 0, the extra mapped memory is disabled. The default size is 67108864 (64MB).
-#    apow     size of record alignment by power of 2. If it is negative, the default value is specified. The default value is 4 standing for 2^4=16.
-#    fpow     maximum number of elements of the free block pool by power of 2. If it is negative, the default value is specified. The default value is 10 standing for 2^10=1024.
-#    dfunit   unit step number of auto defragmentation. If it is not more than 0, the auto defragmentation is disabled. It is disabled by default.
-#    mode     "w" of writer, "r" of reader,"c" of creating,"t" of truncating ,"e" of no locking,"f" of non-blocking lock
-#
-# -- Links
-#    http://1978th.net/tokyocabinet/spex-en.html
-#    http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/3bd2a93322c09eec#
-# Performance limits: http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/3bd2a93322c09eec#
-
-
-h2. Tyrant: ttserver
-
-  ttdb="test"
-  ttserver -port 12009 -thnum 96  \
-    -dmn -pid /var/run/tyrant-${ttdb}.pid
-    -ulog /mnt/tmp/ttyrant/tyrant-$[ttdb}.ulog -ulim 268435456 -uas \
-    -log  /var/log/ttyrant/tyrant-${ttdb}.log  \
-    "/data/db/ttyrant/${ttdb}.tch#bnum=200000000#opts=l#rcnum=100000#xmsiz=536870912"
-    
- can also add host, and umask out to be read-only
-
-    * -host name : specify the host name or the address of the server. By default, every network address is bound.
-    * -port num : specify the port number. By default, it is 1978.
-    * -thnum num : specify the number of worker threads. By default, it is 8.
-    * -tout num : specify the timeout of each session in seconds. By default, no timeout is specified.
-    * -dmn : work as a daemon process.
-    * -pid path : output the process ID into the file.
-    * -kl : kill the existing process if the process ID file is detected.
-    * -log path : output log messages into the file.
-    * -ld : log debug messages also.
-    * -le : log error messages only.
-    * -ulog path : specify the update log directory.
-    * -ulim num : specify the limit size of each update log file.
-    * -uas : use asynchronous I/O for the update log.
-    * -sid num : specify the server ID.
-    * -mhost name : specify the host name of the replication master server.
-    * -mport num : specify the port number of the replication master server.
-    * -rts path : specify the replication time stamp file.
-    * -rcc : check consistency of replication.
-    * -skel name : specify the name of the skeleton database library.
-    * -mul num : specify the division number of the multiple database mechanism.
-    * -ext path : specify the script language extension file.
-    * -extpc name period : specify the function name and the calling period of a periodic command.
-    * -mask expr : specify the names of forbidden commands.
-    * -unmask expr : specify the names of allowed commands.
-
-
-h2. From "Wolfgang Gassler":http://groups.google.com/group/tokyocabinet-users/browse_thread/thread/5a46ee04006a791c#
-
-On Sat, Dec 05, 2009 at 09:32:20PM +0100, Wolfgang Gassler wrote:
-> Hi,
-
-> did anybody look up some of the folowing parameters in the code or can
-> explain them in detail? I just have a guess what they really mean and
-> the short description at the docu homepage
-> http://korrespondence.blogspot.com/2009/09/tokyo-tyrant-tuning-parame...
-> explain them very roughly. Also the already posted blog post
-> http://korrespondence.blogspot.com/2009/09/tokyo-tyrant-tuning-parame...
-> couldn't help.
-
-this is what I gleaned from reading the source code for the hash database
-format ( tchdb.c and tchdb.h ).
-
-> xmsiz
-
-On a TC Hash database, from the beginning of the file, to the end of the bucket
-section, all of that space is mmap'd.  Setting 'xmsiz' sets the minimum amount
-of space that is mmap'd.  Since 67108864 is the default, this means, that an a
-minimum, the first 64MiB of the file will be mmap'd.
-
-If the header size, plus the bucket region is greater than 'xmsize', then xmsiz
-appers to have no affect.
-
-> apow
-
-On a TC Hash database, 'apow' determines on what byte alignment each record will
-sit.  'apow' is a power of 2.  This means that when apow is 4 ( the default for
-hash databases) all records in the database are aligned on a 16 byte boundary,
-in the database file.
-
-This means that every record will take up at a minumum 16 bytes of space, and
-all records are padded to a length that is a multiple of 16.
-
-> fpow
-
-On a TC Hash database, 'fpow' determines the maximum number of free blocks that
-can exist in the free block pool.  This is also a power-of-2 parameter so with
-the default in a Hash database of 10, this means that there can be a maximum
-of 2^10, or 1024 free blocks in the database.
-
-Free blocks come into existence when records are deleted from the database
-and their space in the db file is up for reuse.  If you never delete an
-item from the database, you will never have any free blocks.
-
-> dfunit
-
-On a TC Hash database, 'dfunit' describes how defragmentation takes place.
-Every time a free block is created a 'dfcnt' is incremented.  When 'dfcnt'
-is greater than 'dfunit' and 'dfunit' is greater than 0, defragmentation
-takes place.
-
-I don't know precisely what defragmentation does in TC. A cursory look
-at 'tchdbdefragimpl', the function implementing defagmentation for hash
-databases, it looks like it moves records around filling up free blocks
-in the hash db with real records from the end of the file and then making the
-file smaller if possible.
-
-Basically it moves records around minimizing dead space in the file.
-
-Again, defragmentation will only take place if 'dfunit' has a positive
-value and you remove records from the db creating free blocks.
-
-enjoy,
-
--jeremy 

data/lib/wukong/models/graph.rb

@@ -1,25 +0,0 @@
-
-module Wukong
-  module Models
-    Edge = TypedStruct.new(
-      [:src,              Integer],
-      [:dest,             Integer]
-      )
-
-    MultiEdge = TypedStruct.new(
-      [:src,              Integer],
-      [:dest,             Integer],
-      [:a_follows_b,      Integer],
-      [:b_follows_a,      Integer],
-      [:a_replies_b,      Integer],
-      [:b_replies_a,      Integer],
-      [:a_atsigns_b,      Integer],
-      [:b_atsigns_a,      Integer],
-      [:a_retweets_b,     Integer],
-      [:b_retweets_a,     Integer],
-      [:a_favorites_b,    Integer],
-      [:b_favorites_a,    Integer]
-      )
-
-  end
-end