higgs 0.1.0

data/misc/dbm_bench/dbm_multi_thread.rb

@@ -0,0 +1,199 @@
+#!/usr/local/bin/ruby
+
+# for ident(1)
+CVS_ID = '$Id: dbm_multi_thread.rb 559 2007-09-25 15:20:20Z toki $'
+
+$: << File.join(File.dirname($0), '..', '..', 'lib')
+
+require 'benchmark'
+require 'higgs/dbm'
+require 'higgs/lock'
+require 'higgs/thread'
+require 'thread'
+
+loop_count = (ARGV.shift || '100').to_i
+transaction_count = (ARGV.shift || '100').to_i
+thread_count = (ARGV.shift || '10').to_i
+puts "#{$0}: LOOP:#{loop_count}, TRANSACTION:#{transaction_count}, THREAD:#{thread_count}"
+puts ''
+
+name = File.join(File.dirname($0), 'foo')
+conf_path = File.join(File.dirname($0), '.strc')
+
+options = {}
+options.update(Higgs::Storage.load_conf(conf_path)) if (File.exist? conf_path)
+
+[ Higgs::GiantLockManager.new,
+  Higgs::FineGrainLockManager.new
+].each do |lock_manager|
+  puts lock_manager.class
+  options[:lock_manager] = lock_manager
+  Higgs::DBM.open(name, options) {|dbm|
+    dbm.transaction{|tx|
+      tx[:foo] = 'a'
+      thread_count.times do |i|
+        tx[i] = i.to_s
+      end
+    }
+
+    Benchmark.bm(16) do |x|
+      x.report('   read') {
+        (thread_count * transaction_count).times do
+          dbm.transaction(true) {|tx|
+            loop_count.times do
+              tx[:foo]
+            end
+          }
+        end
+      }
+
+      barrier = Higgs::Barrier.new(thread_count + 1)
+      th_grp = ThreadGroup.new
+      thread_count.times do
+        th_grp.add Thread.new{
+          barrier.wait
+          transaction_count.times do
+            dbm.transaction(true) {|tx|
+              loop_count.times do
+                tx[:foo]
+              end
+            }
+          end
+        }
+      end
+
+      x.report('MT read') {
+        barrier.wait
+        for th in th_grp.list
+          th.join
+        end
+      }
+
+      x.report('   write') {
+        (thread_count * transaction_count).times do
+          dbm.transaction{|tx|
+            loop_count.times do
+              tx[:foo] = 'a'
+            end
+          }
+        end
+      }
+
+      barrier = Higgs::Barrier.new(thread_count + 1)
+      th_grp = ThreadGroup.new
+      thread_count.times do
+        th_grp.add Thread.new{
+          barrier.wait
+          transaction_count.times do
+            dbm.transaction{|tx|
+              loop_count.times do
+                tx[:foo] = 'a'
+              end
+            }
+          end
+        }
+      end
+
+      x.report('MT write') {
+        barrier.wait
+        for th in th_grp.list
+          th.join
+        end
+      }
+
+      x.report('   sparse write') {
+        thread_count.times do |i|
+          key = i
+          value = i.to_s
+          transaction_count.times do
+            dbm.transaction{|tx|
+              loop_count.times do
+                tx[key] = value
+              end
+            }
+          end
+        end
+      }
+
+      barrier = Higgs::Barrier.new(thread_count + 1)
+      th_grp = ThreadGroup.new
+      thread_count.times do |i|
+        key = i
+        value = i.to_s
+        th_grp.add Thread.new{
+          barrier.wait
+          transaction_count.times do
+            dbm.transaction{|tx|
+              loop_count.times do
+                tx[key] = value
+              end
+            }
+          end
+        }
+      end
+
+      x.report('MT sparse write') {
+        barrier.wait
+        for th in th_grp.list
+          th.join
+        end
+      }
+
+      x.report('   read/write') {
+        ((thread_count - 1) * transaction_count).times do
+          dbm.transaction(true) {|tx|
+            loop_count.times do
+              tx[:foo]
+            end
+          }
+        end
+        transaction_count.times do
+          dbm.transaction{|tx|
+            loop_count.times do
+              tx[:foo] = 'a'
+            end
+          }
+        end
+      }
+
+      barrier = Higgs::Barrier.new(thread_count + 1)
+      th_grp = ThreadGroup.new
+      (thread_count - 1).times do
+        th_grp.add Thread.new{
+          barrier.wait
+          transaction_count.times do
+            dbm.transaction(true) {|tx|
+              loop_count.times do
+                tx[:foo]
+              end
+            }
+          end
+        }
+      end
+      th_grp.add Thread.new{
+        barrier.wait
+        transaction_count.times do
+          dbm.transaction{|tx|
+            loop_count.times do
+              tx[:foo] = 'a'
+            end
+          }
+        end
+      }
+
+      x.report('MT read/write') {
+        barrier.wait
+        for th in th_grp.list
+          th.join
+        end
+      }
+    end
+
+    puts ''
+  }
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/misc/dbm_bench/dbm_rnd_delete.rb

@@ -0,0 +1,43 @@
+#!/usr/local/bin/ruby
+
+# for ident(1)
+CVS_ID = '$Id: dbm_rnd_delete.rb 559 2007-09-25 15:20:20Z toki $'
+
+$: << File.join(File.dirname($0), '..', '..', 'lib')
+
+require 'benchmark'
+require 'higgs/dbm'
+
+loop_count = (ARGV.shift || '100').to_i
+data_count = (ARGV.shift || '10').to_i
+puts "#{$0}: LOOP:#{loop_count}, DATA:#{data_count}"
+
+name = File.join(File.dirname($0), 'foo')
+conf_path = File.join(File.dirname($0), '.strc')
+
+options = {}
+options.update(Higgs::Storage.load_conf(conf_path)) if (File.exist? conf_path)
+
+Higgs::DBM.open(name, options) {|dbm|
+  srand(0)
+  key_list = dbm.transaction{|tx| tx.keys }
+
+  Benchmark.bm do |x|
+    x.report('dbm rnd delete:') {
+      loop_count.times do
+        dbm.transaction{|tx|
+          data_count.times do
+            k = key_list[rand(key_list.length)]
+            tx.delete(k)
+          end
+        }
+      end
+    }
+  end
+  print "\n"
+}
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/misc/dbm_bench/dbm_rnd_read.rb

@@ -0,0 +1,44 @@
+#!/usr/local/bin/ruby
+
+# for ident(1)
+CVS_ID = '$Id: dbm_rnd_read.rb 559 2007-09-25 15:20:20Z toki $'
+
+ $: << File.join(File.dirname($0), '..', '..', 'lib')
+
+require 'benchmark'
+require 'higgs/dbm'
+
+loop_count = (ARGV.shift || '100').to_i
+data_count = (ARGV.shift || '10').to_i
+puts "#{$0}: LOOP:#{loop_count}, DATA:#{data_count}"
+
+name = File.join(File.dirname($0), 'foo')
+conf_path = File.join(File.dirname($0), '.strc')
+
+options = {}
+options.update(Higgs::Storage.load_conf(conf_path)) if (File.exist? conf_path)
+options[:read_only] = true
+
+Higgs::DBM.open(name, options) {|dbm|
+  srand(0)
+  key_list = dbm.transaction{|tx| tx.keys }
+
+  Benchmark.bm do |x|
+    x.report('dbm rnd read:') {
+      loop_count.times do
+        dbm.transaction{|tx|
+          data_count.times do
+            k = key_list[rand(key_list.length)]
+            tx[k]
+          end
+        }
+      end
+    }
+  end
+  print "\n"
+}
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/misc/dbm_bench/dbm_rnd_update.rb

@@ -0,0 +1,44 @@
+#!/usr/local/bin/ruby
+
+# for ident(1)
+CVS_ID = '$Id: dbm_rnd_update.rb 559 2007-09-25 15:20:20Z toki $'
+
+$: << File.join(File.dirname($0), '..', '..', 'lib')
+
+require 'benchmark'
+require 'higgs/dbm'
+
+loop_count = (ARGV.shift || '100').to_i
+data_count = (ARGV.shift || '10').to_i
+max_dat_len = (ARGV.shift || '32768').to_i
+puts "#{$0}: LOOP:#{loop_count}, DATA:#{data_count}, MAX_DAT_LEN:#{max_dat_len}"
+
+name = File.join(File.dirname($0), 'foo')
+conf_path = File.join(File.dirname($0), '.strc')
+
+options = {}
+options.update(Higgs::Storage.load_conf(conf_path)) if (File.exist? conf_path)
+
+Higgs::DBM.open(name, options) {|dbm|
+  srand(0)
+  key_list = dbm.transaction{|tx| tx.keys }
+
+  Benchmark.bm do |x|
+    x.report('dbm rnd update:') {
+      loop_count.times do
+        dbm.transaction{|tx|
+          data_count.times do
+            k = key_list[rand(key_list.length)]
+            tx[k] = 0xFF.chr * (rand(max_dat_len))
+          end
+        }
+      end
+    }
+  end
+  print "\n"
+}
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/misc/dbm_bench/dbm_seq_read.rb

@@ -0,0 +1,45 @@
+#!/usr/local/bin/ruby
+
+# for ident(1)
+CVS_ID = '$Id: dbm_seq_read.rb 559 2007-09-25 15:20:20Z toki $'
+
+ $: << File.join(File.dirname($0), '..', '..', 'lib')
+
+require 'benchmark'
+require 'higgs/dbm'
+
+loop_count = (ARGV.shift || '100').to_i
+data_count = (ARGV.shift || '10').to_i
+puts "#{$0}: LOOP:#{loop_count}, DATA:#{data_count}"
+
+name = File.join(File.dirname($0), 'foo')
+conf_path = File.join(File.dirname($0), '.strc')
+
+options = {}
+options.update(Higgs::Storage.load_conf(conf_path)) if (File.exist? conf_path)
+options[:read_only] = true
+
+Higgs::DBM.open(name, options) {|dbm|
+  key_list = dbm.transaction{|tx|
+    tx.keys.map{|k| k.to_i }.sort.map{|i| i.to_s }
+  }
+
+  Benchmark.bm do |x|
+    x.report('dbm seq read:') {
+      loop_count.times do |i|
+        dbm.transaction{|tx|
+          data_count.times do |j|
+            k = key_list[(i + j) % key_list.length]
+            tx[k]
+          end
+        }
+      end
+    }
+  end
+  print "\n"
+}
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/misc/dbm_bench/dbm_seq_write.rb

@@ -0,0 +1,44 @@
+#!/usr/local/bin/ruby
+
+# for ident(1)
+CVS_ID = '$Id: dbm_seq_write.rb 559 2007-09-25 15:20:20Z toki $'
+
+$: << File.join(File.dirname($0), '..', '..', 'lib')
+
+require 'benchmark'
+require 'higgs/dbm'
+
+loop_count = (ARGV.shift || '100').to_i
+data_count = (ARGV.shift || '10').to_i
+max_dat_len = (ARGV.shift || '32768').to_i
+puts "#{$0}: LOOP:#{loop_count}, DATA:#{data_count}, MAX_DAT_LEN:#{max_dat_len}"
+
+name = File.join(File.dirname($0), 'foo')
+conf_path = File.join(File.dirname($0), '.strc')
+
+options = {}
+options.update(Higgs::Storage.load_conf(conf_path)) if (File.exist? conf_path)
+
+Higgs::DBM.open(name, options) {|dbm|
+  srand(0)
+
+  Benchmark.bm do |x|
+    x.report('dbm seq write:') {
+      loop_count.times do |i|
+        dbm.transaction{|tx|
+          data_count.times do |j|
+            k = (i * data_count + j).to_s
+            d = 0xFF.chr * (rand(max_dat_len))
+            tx[k] = d
+          end
+        }
+      end
+    }
+  end
+  print "\n"
+}
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/misc/dbm_bench/st_verify.rb

@@ -0,0 +1,28 @@
+#!/usr/local/bin/ruby
+
+# for ident(1)
+CVS_ID = '$Id: st_verify.rb 559 2007-09-25 15:20:20Z toki $'
+
+$: << File.join(File.dirname($0), '..', '..', 'lib')
+
+require 'benchmark'
+require 'higgs/storage'
+
+puts $0
+
+st = Higgs::Storage.new('foo', :read_only => true)
+begin
+  Benchmark.bm do |x|
+    x.report('st verify') {
+      st.verify
+    }
+  end
+  print "\n"
+ensure
+  st.shutdown
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/misc/io_bench/cksum_bench.rb

@@ -0,0 +1,48 @@
+#!/usr/local/bin/ruby
+
+# for dient(1)
+CVS_ID = '$Id: cksum_bench.rb 559 2007-09-25 15:20:20Z toki $'
+
+require 'benchmark'
+require 'digest/md5'
+require 'digest/rmd160'
+require 'digest/sha1'
+require 'digest/sha2'
+
+count = (ARGV.shift || '10000').to_i
+size = (ARGV.shift || '65536').to_i
+puts "#{$0}: COUNT:#{count}, SIZE:#{size}"
+
+data = 0xFF.chr * size
+
+task_list = [ 8, 16, 32, 64, 128 ].map{|n|
+  [ "String\#sum(#{n}):",
+    proc{
+      count.times do
+        data.sum(n)
+      end
+    }
+  ]
+} + [
+  Digest::MD5, Digest::RMD160,
+  Digest::SHA1, Digest::SHA256, Digest::SHA384, Digest::SHA512
+].map{|d|
+  [ "#{d}:",
+    proc{
+      count.times do
+        d.digest(data)
+      end
+    }
+  ]
+}
+
+Benchmark.bm(task_list.map{|n, t| n.length }.max) do |x|
+  task_list.each do |name, task|
+    x.report(name) { task.call }
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/misc/io_bench/jlog_bench.rb

@@ -0,0 +1,71 @@
+#!/usr/local/bin/ruby
+
+# for dient(1)
+CVS_ID = '$Id: jlog_bench.rb 559 2007-09-25 15:20:20Z toki $'
+
+$: << File.join(File.dirname($0), '..', '..', 'lib')
+
+require 'benchmark'
+require 'fileutils'
+require 'higgs/jlog'
+
+count = (ARGV.shift || '10000').to_i
+log_size = (ARGV.shift || '16384').to_i
+data_file = ARGV.shift || 'jlog_test.dat'
+
+puts "#{$0}: COUNT:#{count}, LOG:#{log_size}"
+
+class JLogWriteTask
+  def initialize(path, count, log_size, fsync, cksum_type)
+    @path = path
+    @count = count
+    @log_dat = 0xFF.chr * log_size
+    @fsync = fsync
+    @cksum_type = cksum_type
+  end
+
+  def open
+    log = Higgs::JournalLogger.open(@path, @fsync, @cksum_type)
+    begin
+      yield(log)
+    ensure
+      log.close
+    end
+  end
+  private :open
+
+  def call
+    open{|log|
+      @count.times do
+        log.write(@log_dat)
+      end
+    }
+  end
+end
+
+task_list = []
+cksum_list = [ :SUM16, :MD5, :RMD160, :SHA1, :SHA256, :SHA384, :SHA512 ]
+for cksum_type in cksum_list
+  task_list << [
+    "jlog write [#{cksum_type}]:",
+    JLogWriteTask.new(data_file, count, log_size, false, cksum_type)
+  ]
+end
+for cksum_type in cksum_list
+  task_list << [
+    "jlog write (fsync) [#{cksum_type}]:",
+    JLogWriteTask.new(data_file, count, log_size, true, cksum_type)
+  ]
+end
+
+Benchmark.bm(task_list.map{|n, t| n.length }.max) do |x|
+  FileUtils.rm_f(data_file)
+  task_list.each do |name, task|
+    x.report(name) { task.call }
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/misc/io_bench/write_bench.rb

@@ -0,0 +1,128 @@
+#!/usr/local/bin/ruby
+
+# for dient(1)
+CVS_ID = '$Id: write_bench.rb 559 2007-09-25 15:20:20Z toki $'
+
+require 'benchmark'
+
+segment_count = (ARGV.shift || '10000').to_i
+segment_size = (ARGV.shift || '16384').to_i
+chunk_count = (ARGV.shift || '10').to_i
+data_file = ARGV.shift || 'io_test.dat'
+
+puts "#{$0}: COUNT:#{segment_count}, SEGMENT:#{segment_size}, CHUNK:#{chunk_count}"
+
+class WriteTask
+  def initialize(path, count, size, chunk, fsync)
+    @path = path
+    @count = count
+    @chunk = chunk
+    @data = 0xFF.chr * size
+    @fsync = fsync
+    srand(0)                    # reset for rand
+  end
+
+  def open
+    begin
+      w = File.open(@path, File::WRONLY | File::CREAT | File::EXCL)
+    rescue Errno::EEXIST
+      w = File.open(@path, File::WRONLY)
+    end
+
+    begin
+      w.binmode
+      yield(w)
+    ensure
+      w.close
+    end
+  end
+  private :open
+
+  def io_sync(io)
+    if (@fsync) then
+      io.fsync
+    else
+      io.sync
+    end
+  end
+  private :io_sync
+end
+
+class SequentialWriteTask < WriteTask
+  def work
+    open{|w|
+      @count.times do |i|
+        w.write(@data)
+        if (i % @chunk == 0) then
+          io_sync(w)
+        end
+      end
+      io_sync(w)
+    }
+  end
+end
+
+class RandomWriteTask < WriteTask
+  def work
+    open{|w|
+      @count.times do |i|
+        pos = rand(@count)
+        w.seek(@data.size * pos)
+        w.write(@data)
+        if (i % @chunk == 0) then
+          io_sync(w)
+        end
+      end
+      io_sync(w)
+    }
+  end
+end
+
+class SortedRandomWriteTask < WriteTask
+  def work
+    open{|w|
+      write_list = []
+      @count.times do |i|
+        pos = rand(@count)
+        write_list << pos
+        if (i % @chunk == 0) then
+          write_list.sort!
+          for pos in write_list
+            w.seek(@data.size * pos)
+            w.write(@data)
+          end
+          write_list.clear
+          io_sync(w)
+        end
+      end
+
+      write_list.sort!
+      for pos in write_list
+        w.seek(@data.size * pos)
+        w.write(@data)
+      end
+      write_list.clear
+      io_sync(w)
+    }
+  end
+end
+
+task_list = [
+  [ 'seq write:',         SequentialWriteTask.new(data_file, segment_count, segment_size, chunk_count, false) ],
+  [ 'seq write (fsync):', SequentialWriteTask.new(data_file, segment_count, segment_size, chunk_count, true) ],
+  [ 'rnd write:',         RandomWriteTask.new(data_file, segment_count, segment_size, chunk_count, false) ],
+  [ 'rnd write (fsync):', RandomWriteTask.new(data_file, segment_count, segment_size, chunk_count, true) ],
+  [ 'sorted rnd write:',  SortedRandomWriteTask.new(data_file, segment_count, segment_size, chunk_count, false) ],
+  [ 'sorted rnd write (fsync):', SortedRandomWriteTask.new(data_file, segment_count, segment_size, chunk_count, true) ]
+]
+
+Benchmark.bm(task_list.map{|n,t| n.length }.max) do |x|
+  task_list.each do |name, work|
+    x.report(name) { work.work }
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End: