higgs 0.1.0

data/test/test_jlog.rb

@@ -0,0 +1,180 @@
+#!/usr/local/bin/ruby
+
+require 'fileutils'
+require 'higgs/block'
+require 'higgs/jlog'
+require 'test/unit'
+
+module Higgs::Test
+  class JournalLoggerTest < Test::Unit::TestCase
+    include Higgs
+
+    # for ident(1)
+    CVS_ID = '$Id: test_jlog.rb 559 2007-09-25 15:20:20Z toki $'
+
+    def setup
+      @path = 't.jlog'
+    end
+
+    def teardown
+      FileUtils.rm_f(@path)
+    end
+
+    def test_jlog
+      log = JournalLogger.open(@path)
+      assert_equal(false, log.sync?)
+      assert_equal(0, log.size)
+      log.close
+
+      JournalLogger.each_log(@path) do |log|
+        assert_fail('not to reach')
+      end
+
+      log = JournalLogger.open(@path, false)
+      assert_equal(false, log.sync?)
+      log.write('foo')
+      assert_equal(1024, log.size)
+      log.close
+
+      count = 0
+      expected_values = [ 'foo' ]
+      JournalLogger.each_log(@path) do |log|
+        nth = "loop: #{count}"
+        assert(! expected_values.empty?, nth)
+        assert_equal(expected_values.shift, log, nth)
+        count += 1
+      end
+      assert(expected_values.empty?)
+
+      log = JournalLogger.open(@path, true)
+      assert_equal(true, log.sync?)
+      log.write(:bar)
+      assert_equal(2048, log.size)
+      log.close
+
+      count = 0
+      expected_values = [ 'foo', :bar ]
+      JournalLogger.each_log(@path) do |log|
+        nth = "loop: #{count}"
+        assert(! expected_values.empty?, nth)
+        assert_equal(expected_values.shift, log, nth)
+        count += 1
+      end
+      assert(expected_values.empty?)
+
+      log = JournalLogger.open(@path, false)
+      assert_equal(false, log.sync?)
+      log.write(777)
+      assert_equal(3072, log.size)
+      log.close
+
+      count = 0
+      expected_values = [ 'foo', :bar, 777 ]
+      JournalLogger.each_log(@path) do |log|
+        nth = "loop: #{count}"
+        assert(! expected_values.empty?, nth)
+        assert_equal(expected_values.shift, log, nth)
+        count += 1
+      end
+      assert(expected_values.empty?)
+
+      log = JournalLogger.open(@path, true)
+      assert_equal(true, log.sync?)
+      log.write('X' * Block::BLOCK_SIZE)
+      log.write('Y' * (Block::BLOCK_SIZE + 1))
+      log.close
+
+      count = 0
+      expected_values = [
+        'foo', :bar, 777,
+        'X' * Block::BLOCK_SIZE,
+        'Y' * (Block::BLOCK_SIZE + 1)
+      ]
+      JournalLogger.each_log(@path) do |log|
+        nth = "loop: #{count}"
+        assert(! expected_values.empty?, nth)
+        assert_equal(expected_values.shift, log, nth)
+        count += 1
+      end
+      assert(expected_values.empty?)
+    end
+
+    def test_eof_mark
+      File.open(@path, 'w') {|w|
+        w.binmode
+        w << 'x' * Block::BLOCK_SIZE
+        JournalLogger.eof_mark(w)
+      }
+      assert_equal(true, (JournalLogger.has_eof_mark? @path))
+    end
+
+    def test_eof_mark_no_file
+      assert_equal(false, (File.exist? @path))
+      assert_equal(true, (JournalLogger.has_eof_mark? @path))
+    end
+
+    def test_eof_mark_empty_file
+      FileUtils.touch(@path)
+      assert_equal(false, (JournalLogger.has_eof_mark? @path))
+    end
+
+    def test_eof_mark_broken_file
+      File.open(@path, 'w') {|w|
+        w.binmode
+        w << 'x' * Block::BLOCK_SIZE
+        JournalLogger.eof_mark(w)
+      }
+      File.truncate(@path, File.stat(@path).size - 1)
+      assert_equal(false, (JournalLogger.has_eof_mark? @path))
+    end
+
+    def test_eof_mark_no_eof_mark
+      File.open(@path, 'w') {|w|
+        w.binmode
+        jlog = JournalLogger.new(w)
+        jlog.write(:no_EOF)
+        jlog.close(false)
+      }
+      assert_equal(false, (JournalLogger.has_eof_mark? @path))
+    end
+
+    def test_open_error
+      log = JournalLogger.open(@path)
+      log.write(:foo)
+      log.close(false)
+
+      assert_raise(RuntimeError) {
+        JournalLogger.open(@path)
+      }
+    end
+
+    def test_scan_log_error
+      log = JournalLogger.open(@path)
+      log.write(:foo)
+      log.write(:bar)
+      log.write(:baz)
+      log.close(false)
+      File.truncate(@path, File.stat(@path).size - 1)
+
+      File.open(@path) {|r|
+        r.binmode
+        count = 0
+        expected_values = [ :foo, :bar ]
+        assert_raise(Block::BrokenError) {
+          JournalLogger.scan_log(r) do |log|
+            nth = "loop: #{count}"
+            assert_equal(expected_values.shift, log, nth)
+            count += 1
+          end
+        }
+        assert(expected_values.empty?)
+        assert_equal(2048, r.tell)
+      }
+    end
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/test/test_lock.rb

@@ -0,0 +1,320 @@
+#!/usr/local/bin/ruby
+
+require 'higgs/lock'
+require 'higgs/thread'
+require 'test/unit'
+require 'thwait'
+
+module Higgs::Test
+  module LockManagerTest
+    include Higgs
+
+    # for ident(1)
+    CVS_ID = '$Id: test_lock.rb 559 2007-09-25 15:20:20Z toki $'
+
+    WORK_COUNT = 100
+    THREAD_COUNT = 10
+
+    def test_read_transaction_single_thread
+      v = "foo"
+      WORK_COUNT.times do
+        @lock_manager.transaction(true) {|lock_handler|
+          lock_handler.lock(:foo)
+          assert_equal("foo", v)
+        }
+      end
+    end
+
+    def test_write_transaction_single_thread
+      count = 0
+      WORK_COUNT.times do
+        @lock_manager.transaction{|lock_handler|
+          lock_handler.lock(:foo)
+          count += 1
+        }
+      end
+      assert_equal(WORK_COUNT, count)
+    end
+
+    def test_read_transaction_multithread
+      v = "foo"
+      th_grp = ThreadGroup.new
+      barrier = Barrier.new(THREAD_COUNT + 1)
+
+      THREAD_COUNT.times{|i|
+        th_grp.add Thread.new{
+          @lock_manager.transaction(true) {|lock_handler|
+            lock_handler.lock(:foo)
+            barrier.wait
+            WORK_COUNT.times do |j|
+              assert_equal("foo", v, "read transaction: #{i}.#{j}")
+            end
+          }
+        }
+      }
+
+      barrier.wait
+      for t in th_grp.list
+        t.join
+      end
+    end
+
+    def test_write_transaction_multithread
+      count = 0
+      th_grp = ThreadGroup.new
+      barrier = Barrier.new(THREAD_COUNT + 1)
+
+      THREAD_COUNT.times do
+        th_grp.add Thread.new{
+          barrier.wait
+          WORK_COUNT.times do
+            @lock_manager.transaction{|lock_handler|
+              lock_handler.lock(:foo)
+              count += 1
+            }
+          end
+        }
+      end
+
+      barrier.wait
+      for t in th_grp.list
+        t.join
+      end
+      assert_equal(THREAD_COUNT * WORK_COUNT, count)
+    end
+
+    def test_read_write_transaction_multithread
+      count = 0
+      th_grp = ThreadGroup.new
+      barrier = Barrier.new(THREAD_COUNT * 2 + 1)
+
+      THREAD_COUNT.times{|i|
+        th_grp.add Thread.new{
+          @lock_manager.transaction(true) {|lock_handler|
+            lock_handler.lock(:foo)
+            barrier.wait
+            WORK_COUNT.times do |j|
+              assert_equal(0, count, "read transaction: #{i}.#{j}")
+            end
+          }
+        }
+      }
+
+      THREAD_COUNT.times do
+        th_grp.add Thread.new{
+          barrier.wait
+          WORK_COUNT.times do
+            @lock_manager.transaction{|lock_handler|
+              lock_handler.lock(:foo)
+              count += 1
+            }
+          end
+        }
+      end
+
+      barrier.wait
+      for t in th_grp.list
+        t.join
+      end
+      assert_equal(THREAD_COUNT * WORK_COUNT, count)
+    end
+  end
+
+  class GiantLockManagerTest < Test::Unit::TestCase
+    include Higgs
+    include LockManagerTest
+
+    # for ident(1)
+    CVS_ID = '$Id: test_lock.rb 559 2007-09-25 15:20:20Z toki $'
+
+    def setup
+      @lock_manager = GiantLockManager.new
+    end
+
+    def test_write_read_transaction_multithread
+      count = 0
+      th_grp = ThreadGroup.new
+      barrier = Barrier.new(THREAD_COUNT + 2)
+
+      th_grp.add Thread.new{
+        @lock_manager.transaction{|lock_handler|
+          lock_handler.lock(:foo)
+          barrier.wait
+          THREAD_COUNT.times do
+            WORK_COUNT.times do
+              count += 1
+            end
+          end
+        }
+      }
+
+      THREAD_COUNT.times{|i|
+        th_grp.add Thread.new{
+          barrier.wait          # different point from FineGrainLockManager
+          @lock_manager.transaction(true) {|lock_handler|
+            lock_handler.lock(:foo)
+            assert_equal(THREAD_COUNT * WORK_COUNT, count, "read transaction: #{i}")
+          }
+        }
+      }
+
+      barrier.wait
+      for t in th_grp.list
+        t.join
+      end
+    end
+  end
+
+  class FineGrainLockManagerTest < Test::Unit::TestCase
+    include Higgs
+    include LockManagerTest
+
+    # for ident(1)
+    CVS_ID = '$Id: test_lock.rb 559 2007-09-25 15:20:20Z toki $'
+
+    def setup
+      @lock_manager = FineGrainLockManager.new
+    end
+
+    def test_write_read_transaction_multithread
+      count = 0
+      th_grp = ThreadGroup.new
+      barrier = Barrier.new(THREAD_COUNT + 2)
+
+      th_grp.add Thread.new{
+        @lock_manager.transaction{|lock_handler|
+          lock_handler.lock(:foo)
+          barrier.wait
+          THREAD_COUNT.times do
+            WORK_COUNT.times do
+              count += 1
+            end
+          end
+        }
+      }
+
+      THREAD_COUNT.times{|i|
+        th_grp.add Thread.new{
+          @lock_manager.transaction(true) {|lock_handler|
+            barrier.wait   # different point from GiantLockManager
+            lock_handler.lock(:foo)
+            assert_equal(THREAD_COUNT * WORK_COUNT, count, "read transaction: #{i}")
+          }
+        }
+      }
+
+      barrier.wait
+      for t in th_grp.list
+        t.join
+      end
+    end
+  end
+
+  class GiantLockManagerNoDeadLockTest < Test::Unit::TestCase
+    include Higgs
+
+    # for ident(1)
+    CVS_ID = '$Id: test_lock.rb 559 2007-09-25 15:20:20Z toki $'
+
+    WORK_COUNT = 1000
+
+    def setup
+      @lock_manager = GiantLockManager.new
+    end
+
+    def test_transaction_no_dead_lock
+      barrier = Barrier.new(3)
+
+      t1 = Thread.new{
+        barrier.wait
+        WORK_COUNT.times do
+          @lock_manager.transaction{|lock_handler|
+            lock_handler.lock(:foo)
+            lock_handler.lock(:bar)
+          }
+        end
+      }
+
+      t2 = Thread.new{
+        barrier.wait
+        WORK_COUNT.times do
+          @lock_manager.transaction{|lock_handler|
+            lock_handler.lock(:bar)
+            lock_handler.lock(:foo)
+          }
+        end
+      }
+
+      barrier.wait
+      t1.join
+      t2.join
+    end
+  end
+
+  class FineGrainLockManagerDeadLockTest < Test::Unit::TestCase
+    include Higgs
+
+    # for ident(1)
+    CVS_ID = '$Id: test_lock.rb 559 2007-09-25 15:20:20Z toki $'
+
+    def setup
+      @lock_manager = FineGrainLockManager.new(:spin_lock_count   => 10,
+                                               :try_lock_limit    => 0.1,
+                                               :try_lock_interval => 0.001)
+    end
+
+    def test_transaction_dead_lock
+      barrier = Barrier.new(3)
+
+      m1 = Mutex.new
+      end_of_t1 = false
+
+      m2 = Mutex.new
+      end_of_t2 = false
+
+      t1 = Thread.new{
+        barrier.wait
+        begin
+          until (m2.synchronize{ end_of_t2 })
+            @lock_manager.transaction{|lock_handler|
+              lock_handler.lock(:foo)
+              sleep(0.03)
+              lock_handler.lock(:bar)
+            }
+          end
+        ensure
+          m1.synchronize{ end_of_t1 = true }
+        end
+      }
+
+      t2 = Thread.new{
+        barrier.wait
+        begin
+          until (m1.synchronize{ end_of_t1 })
+            @lock_manager.transaction{|lock_handler|
+              lock_handler.lock(:bar)
+              sleep(0.07)
+              lock_handler.lock(:foo)
+            }
+          end
+        ensure
+          m2.synchronize{ end_of_t2 = true }
+        end
+      }
+
+      barrier.wait
+      assert_raise(LockManager::TryLockTimeoutError) {
+        t1.join
+        t2.join
+      }
+
+      # join threads without exception
+      ThreadsWait.all_waits(t1, t2)
+    end
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/test/test_online_backup.rb

@@ -0,0 +1,169 @@
+#!/usr/local/bin/ruby
+
+require 'drb'
+require 'fileutils'
+require 'higgs/index'
+require 'higgs/storage'
+require 'logger'
+require 'test/unit'
+
+module Higgs::Test
+  module OnlineBackupParams
+    STORAGE_ITEMS = (ENV['STORAGE_ITEMS'] || '100').to_i
+    COMMIT_ITEMS = (ENV['COMMIT_ITEMS'] || '10').to_i
+    MAX_ITEM_BYTES = (ENV['MAX_ITEM_BYTES'] || '16384').to_i
+    LEAST_COMMITS_PER_ROTATION = (ENV['LEAST_COMMITS_PER_ROTATION'] || '8').to_i
+    UPTIME_SECONDS = (ENV['UPTIME_SECONDS'] || '3').to_i
+    ITEM_CHARS = ('A'..'Z').to_a + ('a'..'z').to_a
+
+    if ($DEBUG) then
+      puts 'online backup test parameters...'
+      for name in constants
+        puts "#{name} = #{const_get(name)}"
+      end
+      puts ''
+    end
+  end
+
+  class OnlineBackupTest < Test::Unit::TestCase
+    include Higgs
+    include OnlineBackupParams
+
+    # for ident(1)
+    CVS_ID = '$Id: test_online_backup.rb 559 2007-09-25 15:20:20Z toki $'
+
+    def setup
+      srand(0)
+
+      @backup_dir = 'onbk_test_backup'
+      @backup_name = File.join(@backup_dir, 'foo')
+      FileUtils.rm_rf(@backup_dir) # for debug
+      FileUtils.mkdir_p(@backup_dir)
+
+      @restore_dir = 'onbk_test_restore'
+      @restore_name = File.join(@restore_dir, 'foo')
+      FileUtils.rm_rf(@restore_dir) # for debug
+      FileUtils.mkdir_p(@restore_dir)
+
+      @jlog_rotate_service_uri = 'druby://localhost:14142'
+
+      @start_latch = File.join(@backup_dir, '.start')
+      @stop_latch = File.join(@backup_dir, '.stop')
+      @stopped_latch = File.join(@backup_dir, '.stopped')
+      @end_latch = File.join(@backup_dir, '.end')
+    end
+
+    def teardown
+      FileUtils.rm_rf(@backup_dir) unless $DEBUG
+      FileUtils.rm_rf(@restore_dir) unless $DEBUG
+    end
+
+    def run_backup_storage
+      # step 0: storage starts with jlog_rotate_service_uri option
+      # (jlog_rotate_service_uri option is disabled by default)
+      st = Storage.new(@backup_name,
+                       :jlog_rotate_max => 0,
+                       :jlog_rotate_size => COMMIT_ITEMS * MAX_ITEM_BYTES * LEAST_COMMITS_PER_ROTATION,
+                       :jlog_rotate_service_uri => @jlog_rotate_service_uri,
+                       :logger => proc{|path|
+                         logger = Logger.new(path, 1)
+                         logger.level = Logger::DEBUG
+                         logger
+                       })
+
+      begin
+        FileUtils.touch(@start_latch)
+        until (File.exist? @stop_latch)
+          write_list = []
+          ope = [ :write, :system_properties, :custom_properties, :delete ][rand(3)]
+          key = rand(STORAGE_ITEMS)
+          case (ope)
+          when :write
+            value = rand(256).chr * rand(MAX_ITEM_BYTES)
+            write_list << [ ope, key, value ]
+          when :system_properties
+            next unless (st.key? key)
+            write_list << [ ope, key, { 'string_only' => [ true, false ][rand(2)] } ]
+          when :custom_properties
+            next unless (st.key? key)
+            value = ITEM_CHARS[rand(ITEM_CHARS.size)] * rand(MAX_ITEM_BYTES)
+            write_list << [ ope, key, { 'foo' => value } ]
+          when :delete
+            next unless (st.key? key)
+            write_list << [ ope, key ]
+          else
+            raise "unknown operation: #{ope}"
+          end
+          st.write_and_commit(write_list)
+        end
+        FileUtils.touch(@stopped_latch)
+        until (File.exist? @end_latch)
+          # spin lock
+        end
+        st.verify
+      ensure
+        st.shutdown
+      end
+    end
+    private :run_backup_storage
+
+    def test_online_backup
+      pid = fork{ run_backup_storage }
+      begin
+        until (File.exist? @start_latch)
+          # spin lock
+        end
+        jlog_rotate_service = DRbObject.new_with_uri(@jlog_rotate_service_uri)
+        sleep(UPTIME_SECONDS)
+
+        # step 1: backup index
+        jlog_rotate_service.call("#{@restore_name}.idx")
+
+        # step 2: backup data
+        FileUtils.cp("#{@backup_name}.tar", "#{@restore_name}.tar")
+
+        # transactions are stopped for comparison between original
+        # files and restored files. on real operation, transactions
+        # are not stopped.
+        FileUtils.touch(@stop_latch)
+        until (File.exist? @stopped_latch)
+          # spin lock
+        end
+
+        # step 3: rotate journal log
+        jlog_rotate_service.call(true)
+
+        # step 4: backup old journal logs
+        for path in Storage.rotate_entries("#{@backup_name}.jlog")
+          FileUtils.cp(path, File.join(@restore_dir, File.basename(path)))
+          FileUtils.rm(path)
+        end
+
+        # step 4: recover from backup
+        Storage.recover(@restore_name)
+
+        # recovered files are same as original files.
+        assert(FileUtils.cmp("#{@backup_name}.tar", "#{@restore_name}.tar"), 'tar')
+        assert_equal(Index.new.load("#{@backup_name}.idx").to_h,
+                     Index.new.load("#{@restore_name}.idx").to_h, 'idx')
+      ensure
+        FileUtils.touch(@stop_latch)
+        FileUtils.touch(@end_latch)
+        Process.waitpid(pid)
+      end
+      assert_equal(0, $?.exitstatus)
+
+      st = Storage.new(@restore_name, :read_only => true)
+      begin
+        st.verify
+      ensure
+        st.shutdown
+      end
+    end
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End: