higgs 0.1.2 → 0.1.3

data/test/test_services.rb

@@ -0,0 +1,125 @@
+#!/usr/local/bin/ruby
+
+require 'drb'
+require 'fileutils'
+require 'higgs/storage'
+require 'test/unit'
+
+module Higgs::Test
+  class RemoteServicesTest < Test::Unit::TestCase
+    include Higgs
+
+    # for ident(1)
+    CVS_ID = '$Id: test_services.rb 676 2007-11-08 16:09:51Z toki $'
+
+    STORAGE_DIR = 'remote_storage'
+    REMOTE_SERVICES_URI = 'druby://localhost:31415'
+
+    class << self
+      include Higgs
+
+      def start_services
+	return if @pid
+
+	storage_name = File.join(STORAGE_DIR, 'foo')
+	FileUtils.rm_rf(STORAGE_DIR) unless $DEBUG
+	FileUtils.mkdir_p(STORAGE_DIR)
+
+	start_latch = File.join(STORAGE_DIR, '.start')
+	stop_latch = File.join(STORAGE_DIR, '.stop')
+	@pid = fork{
+	  begin
+	    require 'higgs/services'
+	    require 'logger'
+
+	    st = Storage.new(storage_name,
+			     :jlog_rotate_max => 0,
+			     :logger => proc{|path|
+			       logger = Logger.new(path, 1)
+			       logger.level = Logger::DEBUG
+			       logger
+			     })
+
+	    sv = RemoteServices.new(:remote_services_uri => REMOTE_SERVICES_URI,
+				    :storage => st)
+
+	    FileUtils.touch(start_latch)
+	    until (File.exist? stop_latch)
+	      sleep(0.001)	# spin lock
+	    end
+	  ensure
+	    st.shutdown if st
+	    sv.shutdown if sv
+	    FileUtils.rm_rf(STORAGE_DIR) unless $DEBUG
+	  end
+	}
+
+	until (File.exist? start_latch)
+	  # spin lock
+	end
+
+	at_exit{
+	  FileUtils.touch(stop_latch)
+	  Process.waitpid(@pid)
+	}
+
+	DRb.start_service
+
+	nil
+      end
+
+      attr_reader :pid
+    end
+
+    def setup
+      RemoteServicesTest.start_services
+      @services = DRbObject.new_with_uri(REMOTE_SERVICES_URI)
+      @localhost_check_tmpfile = File.join(STORAGE_DIR, ".localhost_check.#{RemoteServicesTest.pid}")
+    end
+
+    def test_alive_service_v1
+      alive_service = @services[:alive_service_v1] or flunk
+      assert_equal(true, alive_service.call)
+    end
+
+    def test_localhost_check_service_v1
+      Dir.chdir('/') {
+	localhost_check_service = @services[:localhost_check_service_v1] or flunk
+	localhost_check_service.call{|localhost_check|
+	  localhost_check.call
+	}
+      }
+    end
+
+    def test_localhost_check_service_v1_exists_tmpfile
+      localhost_check_service = @services[:localhost_check_service_v1] or flunk
+      FileUtils.touch(@localhost_check_tmpfile)
+      begin
+	localhost_check_service.call{|localhost_check|
+	  localhost_check.call
+	}
+      ensure
+	FileUtils.rm(@localhost_check_tmpfile, :force => true)
+      end
+    end
+
+    def test_localhost_check_service_v1_RuntimeError_not_found_tmpfile
+      localhost_check_service = @services[:localhost_check_service_v1] or flunk
+      localhost_check_service.call{|localhost_check|
+	FileUtils.rm(@localhost_check_tmpfile, :force => true)
+	assert_raise(RuntimeError) { localhost_check.call }
+      }
+    end
+
+    def test_localhost_check_service_v1_RuntimeError_mismatch_message
+      localhost_check_service = @services[:localhost_check_service_v1] or flunk
+      localhost_check_service.call{|localhost_check|
+	File.open(@localhost_check_tmpfile, 'w') {|f|
+	  f.binmode
+	  f << 'bar'
+	}
+	assert_raise(RuntimeError) { localhost_check.call }
+      }
+    end
+  end
+end

data/test/test_storage.rb

@@ -1,5 +1,6 @@
 #!/usr/local/bin/ruby
 
+require 'digest'
 require 'fileutils'
 require 'higgs/storage'
 require 'logger'
@@ -10,7 +11,7 @@ module Higgs::Test
     include Higgs
 
     # for ident(1)
-    CVS_ID = '$Id: test_storage.rb 627 2007-10-12 07:10:13Z toki $'
+    CVS_ID = '$Id: test_storage.rb 676 2007-11-08 16:09:51Z toki $'
 
     def setup
       srand(0)                  # preset for rand
@@ -36,12 +37,11 @@ module Higgs::Test
     include StorageTestCase
 
     # for ident(1)
-    CVS_ID = '$Id: test_storage.rb 627 2007-10-12 07:10:13Z toki $'
+    CVS_ID = '$Id: test_storage.rb 676 2007-11-08 16:09:51Z toki $'
 
     def new_storage
       Storage.new(@name, :logger => @logger)
     end
-    private :new_storage
 
     def test_raw_write_and_commit
       write_list = [
@@ -141,6 +141,15 @@ module Higgs::Test
       }
     end
 
+    def test_write_and_commit_standby_NotWritableError
+      @st.shutdown
+      @st = nil
+      @st = Storage.new(@name, :read_only => :standby, :logger => @logger)
+      assert_raise(Storage::NotWritableError) {
+        @st.write_and_commit([ [ :write, 'foo', "Hello world.\n" ] ])
+      }
+    end
+
     def test_write_and_commit_IndexError_not_exist_properties
       assert_raise(IndexError) {
         @st.write_and_commit([ [ :system_properties, 'foo', {} ] ])
@@ -224,6 +233,36 @@ module Higgs::Test
       assert_raise(IndexError) { @st.string_only('foo') }
     end
 
+    def test_change_number_and_unique_data_id
+      assert_equal(nil, @st.data_change_number(:foo))
+      assert_equal(nil, @st.properties_change_number(:foo))
+      assert_equal(nil, @st.unique_data_id(:foo))
+
+      @st.write_and_commit([ [ :write, :foo, 'apple' ] ])
+
+      assert_equal(1, @st.data_change_number(:foo))
+      assert_equal(1, @st.properties_change_number(:foo))
+      assert_equal("foo\t1", @st.unique_data_id(:foo))
+
+      @st.write_and_commit([ [ :custom_properties, :foo, { 'bar' => 'banana' } ] ])
+
+      assert_equal(1, @st.data_change_number(:foo))
+      assert_equal(2, @st.properties_change_number(:foo))
+      assert_equal("foo\t1", @st.unique_data_id(:foo))
+
+      @st.write_and_commit([ [ :write, :foo, 'orange' ] ])
+
+      assert_equal(3, @st.data_change_number(:foo))
+      assert_equal(3, @st.properties_change_number(:foo))
+      assert_equal("foo\t3", @st.unique_data_id(:foo))
+
+      @st.write_and_commit([ [ :system_properties, :foo, { 'string_only' => true } ] ])
+
+      assert_equal(3, @st.data_change_number(:foo))
+      assert_equal(4, @st.properties_change_number(:foo))
+      assert_equal("foo\t3", @st.unique_data_id(:foo))
+    end
+
     def test_key
       assert_equal(false, (@st.key? 'foo'))
       @st.write_and_commit([ [ :write, 'foo', "Hello world.\n" ] ])
@@ -242,6 +281,16 @@ module Higgs::Test
       assert_equal(false, (@st.key? 'bar'))
     end
 
+    def test_key_standby
+      @st.write_and_commit([ [ :write, 'foo', "Hello world.\n" ] ])
+      @st.shutdown
+      @st = nil
+      @st = Storage.new(@name, :read_only => :standby, :logger => @logger)
+
+      assert_equal(true, (@st.key? 'foo'))
+      assert_equal(false, (@st.key? 'bar'))
+    end
+
     def test_each_key
       @st.each_key do |key|
         flunk('not to reach')
@@ -283,6 +332,22 @@ module Higgs::Test
       assert(expected_keys.empty?)
     end
 
+    def test_each_key_standby
+      @st.write_and_commit([ [ :write, 'foo', 'one' ],
+                             [ :write, 'bar', 'two' ],
+                             [ :write, 'baz', 'three' ]
+                           ])
+      @st.shutdown
+      @st = nil
+      @st = Storage.new(@name, :read_only => :standby, :logger => @logger)
+
+      expected_keys = %w[ foo bar baz ]
+      @st.each_key do |key|
+        assert(expected_keys.delete(key), "each_key do |#{key}|")
+      end
+      assert(expected_keys.empty?)
+    end
+
     def test_verify
       @st.write_and_commit([ [ :write, 'foo', "Hello world.\n" ] ])
       @st.verify
@@ -353,7 +418,11 @@ module Higgs::Test
     end
 
     def test_shutdown
+      assert_equal(false, @st.shutdown?)
+      assert_equal(true, @st.alive?)
       @st.shutdown
+      assert_equal(true, @st.shutdown?)
+      assert_equal(false, @st.alive?)
       assert_raise(Storage::ShutdownException) { @st.shutdown }
       assert_raise(Storage::ShutdownException) { @st.fetch('foo') }
       assert_raise(Storage::ShutdownException) { @st.fetch_properties('foo') }
@@ -366,20 +435,54 @@ module Higgs::Test
       assert_raise(Storage::ShutdownException) { @st.write_and_commit([]) }
       assert_raise(Storage::ShutdownException) { @st.verify }
     end
+
+    def test_switch_to_write_RuntimeError_not_standby_mode
+      assert_equal(false, @st.read_only)
+      assert_raise(RuntimeError) {
+        @st.switch_to_write
+      }
+    end
+
+    def test_switch_to_write_RuntimeError_not_standby_mode_on_read_only_mode
+      @st.shutdown
+      @st = nil
+      @st = Storage.new(@name, :read_only => true, :logger => @logger)
+
+      assert_equal(true, @st.read_only)
+      assert_raise(RuntimeError) {
+        @st.switch_to_write
+      }
+    end
+  end
+
+  class StorageSwitchToWriteTest < StorageTest
+    def new_storage
+      st = Storage.new(@name, :logger => @logger, :read_only => :standby)
+      assert_equal(:standby, st.read_only)
+      st.switch_to_write
+      assert_equal(false, st.read_only)
+      st
+    end
+
+    for name in instance_methods(true)
+      case (name)
+      when /^test_.*read_only/, /^test_.*standby/
+        undef_method name
+      end
+    end
   end
 
   class StorageRecoveryTest < Test::Unit::TestCase
     include StorageTestCase
 
     # for ident(1)
-    CVS_ID = '$Id: test_storage.rb 627 2007-10-12 07:10:13Z toki $'
+    CVS_ID = '$Id: test_storage.rb 676 2007-11-08 16:09:51Z toki $'
 
     def new_storage
       Storage.new(@name,
                   :jlog_rotate_max => 0, # unlimited rotation
                   :logger => @logger)
     end
-    private :new_storage
 
     def write_data(loop_count=100, data_count=10, data_max_size=1024*5)
       loop_count.times do
@@ -403,6 +506,10 @@ module Higgs::Test
     private :write_data
 
     def test_manual_recovery
+      other_name = File.join(@test_dir, 'bar')
+      FileUtils.cp("#{@name}.tar", "#{other_name}.tar", :preserve => true)
+      FileUtils.cp("#{@name}.idx", "#{other_name}.idx", :preserve => true)
+
       write_data
 
       3.times do
@@ -414,16 +521,41 @@ module Higgs::Test
 
       @st.shutdown
 
-      other_name = File.join(@test_dir, 'bar')
-      for name in Storage.rotate_entries("#{@name}.jlog")
+      for name in Storage.rotated_entries("#{@name}.jlog")
         name =~ /\.jlog.*$/ or raise 'mismatch'
         FileUtils.cp(name, other_name + $&, :preserve => true)
       end
       Storage.recover(other_name)
 
-      assert(FileUtils.cmp("#{@name}.tar", "#{other_name}.tar"), 'data')
+      assert(FileUtils.cmp("#{@name}.tar", "#{other_name}.tar"), 'DATA should be same.')
       assert(Index.new.load("#{@name}.idx").to_h ==
-             Index.new.load("#{other_name}.idx").to_h, 'index')
+             Index.new.load("#{other_name}.idx").to_h, 'INDEX should be same.')
+    end
+
+    def test_recovery_PanicError_unexpected_storage_id
+      other_name = File.join(@test_dir, 'bar')
+      st2 = Storage.new(other_name)
+      st2.shutdown
+
+      write_data
+
+      3.times do
+        @st.rotate_journal_log(false)
+      end
+      3.times do
+        @st.rotate_journal_log(true)
+      end
+
+      @st.shutdown
+
+      for name in Storage.rotated_entries("#{@name}.jlog")
+        name =~ /\.jlog.*$/ or raise 'mismatch'
+        FileUtils.cp(name, other_name + $&, :preserve => true)
+      end
+
+      assert_raise(Storage::PanicError) {
+        Storage.recover(other_name)
+      }
     end
 
     def test_auto_recovery
@@ -446,13 +578,63 @@ module Higgs::Test
       st2 = Storage.new(other_name, :logger => @logger)
       st2.shutdown
 
-      assert(FileUtils.cmp("#{@name}.tar", "#{other_name}.tar"), 'data')
+      assert(FileUtils.cmp("#{@name}.tar", "#{other_name}.tar"), 'DATA should be same.')
       assert(Index.new.load("#{@name}.idx").to_h ==
-             Index.new.load("#{other_name}.idx").to_h)
-      assert(FileUtils.cmp("#{@name}.jlog", "#{other_name}.jlog"), 'index')
+             Index.new.load("#{other_name}.idx").to_h, 'INDEX should be same.')
+      assert(FileUtils.cmp("#{@name}.jlog", "#{other_name}.jlog"), 'JOURNAL LOG should be same.')
     end
 
-    def test_lost_journal_log_error
+    def test_auto_recovery_on_standby_mode
+      write_data
+      @st.rotate_journal_log(true)
+
+      other_name = File.join(@test_dir, 'bar')
+      FileUtils.cp("#{@name}.tar", "#{other_name}.tar", :preserve => true)
+      FileUtils.cp("#{@name}.idx", "#{other_name}.idx", :preserve => true)
+
+      # write_data(10 * 10 * 256) < jlog_rotate_size(256 * 1024)
+      write_data(10, 10, 256)
+
+      # not closed journal log for other storage.
+      FileUtils.cp("#{@name}.jlog", "#{other_name}.jlog", :preserve => true)
+
+      @st.shutdown
+
+      # auto-recovery for other storage
+      st2 = Storage.new(other_name, :logger => @logger, :read_only => :standby)
+      st2.shutdown
+
+      assert(FileUtils.cmp("#{@name}.tar", "#{other_name}.tar"), 'DATA should be same.')
+      assert(Index.new.load("#{@name}.idx").to_h ==
+             Index.new.load("#{other_name}.idx").to_h, 'INDEX should be same.')
+      assert(FileUtils.cmp("#{@name}.jlog", "#{other_name}.jlog"), 'JOURNAL LOG should be same.')
+    end
+
+    def test_auto_recovery_NotWritableError
+      write_data
+      @st.rotate_journal_log(true)
+
+      other_name = File.join(@test_dir, 'bar')
+      FileUtils.cp("#{@name}.tar", "#{other_name}.tar", :preserve => true)
+      FileUtils.cp("#{@name}.idx", "#{other_name}.idx", :preserve => true)
+
+      # write_data(10 * 10 * 256) < jlog_rotate_size(256 * 1024)
+      write_data(10, 10, 256)
+
+      # not closed journal log for other storage.
+      FileUtils.cp("#{@name}.jlog", "#{other_name}.jlog", :preserve => true)
+
+      @st.shutdown
+
+      # auto-recovery for other storage
+      assert_raise(Higgs::Storage::NotWritableError) {
+        st2 = Storage.new(other_name, :logger => @logger, :read_only => true)
+        st2.shutdown
+        flunk('not to reach')
+      }
+    end
+
+    def test_recovery_PanicError_lost_journal_log_error
       write_data
       @st.rotate_journal_log(true)
 
@@ -470,13 +652,153 @@ module Higgs::Test
         st2 = Storage.new(other_name, :logger => @logger)
       }
     end
+
+    def test_apply_journal_log
+      other_name = File.join(@test_dir, 'bar')
+      FileUtils.cp("#{@name}.tar", "#{other_name}.tar", :preserve => true)
+      FileUtils.cp("#{@name}.idx", "#{other_name}.idx", :preserve => true)
+
+      write_data
+      @st.rotate_journal_log(true)
+      @st.shutdown
+
+      st2 = Storage.new(other_name, :jlog_rotate_size => 1024 * 8)
+      begin
+        for path in Storage.rotated_entries("#{@name}.jlog")
+          st2.apply_journal_log(path)
+        end
+      ensure
+        st2.shutdown
+      end
+
+      assert(FileUtils.cmp("#{@name}.tar", "#{other_name}.tar"), 'DATA should be same.')
+      assert(Index.new.load("#{@name}.idx").to_h ==
+             Index.new.load("#{other_name}.idx").to_h, 'INDEX should be same.')
+    end
+
+    def test_apply_journal_log_on_standby_mode
+      other_name = File.join(@test_dir, 'bar')
+      FileUtils.cp("#{@name}.tar", "#{other_name}.tar", :preserve => true)
+      FileUtils.cp("#{@name}.idx", "#{other_name}.idx", :preserve => true)
+
+      write_data
+      @st.rotate_journal_log(true)
+      @st.shutdown
+
+      st2 = Storage.new(other_name, :jlog_rotate_size => 1024 * 8, :read_only => :standby)
+      begin
+        for path in Storage.rotated_entries("#{@name}.jlog")
+          st2.apply_journal_log(path)
+        end
+      ensure
+        st2.shutdown
+      end
+
+      assert(FileUtils.cmp("#{@name}.tar", "#{other_name}.tar"), 'DATA should be same.')
+      assert(Index.new.load("#{@name}.idx").to_h ==
+             Index.new.load("#{other_name}.idx").to_h, 'INDEX should be same.')
+    end
+
+    def test_apply_journal_log_PanicError_unexpected_storage_id
+      write_data
+      @st.rotate_journal_log(true)
+      @st.shutdown
+
+      other_name = File.join(@test_dir, 'bar')
+      st2 = Storage.new(other_name, :jlog_rotate_size => 1024 * 8)
+      assert_equal(true, st2.alive?, 'alive storage')
+      assert_equal(false, st2.shutdown?)
+
+      begin
+        for path in Storage.rotated_entries("#{@name}.jlog")
+          assert_raise(Storage::PanicError) {
+            st2.apply_journal_log(path)
+          }
+        end
+        assert_equal(false, st2.alive?, 'panic storage')
+        assert_equal(false, st2.shutdown?)
+      ensure
+        st2.shutdown
+      end
+
+      assert_equal(false, st2.alive?, 'shutdown storage')
+      assert_equal(true, st2.shutdown?)
+    end
+
+    def test_apply_journal_log_PanicError_lost_journal_log
+      other_name = File.join(@test_dir, 'bar')
+      FileUtils.cp("#{@name}.tar", "#{other_name}.tar", :preserve => true)
+      FileUtils.cp("#{@name}.idx", "#{other_name}.idx", :preserve => true)
+
+      write_data
+      write_data
+      write_data
+      @st.rotate_journal_log(true)
+      @st.shutdown
+
+      st2 = Storage.new(other_name, :jlog_rotate_size => 1024 * 8)
+      assert_equal(true, st2.alive?, 'alive storage')
+      assert_equal(false, st2.shutdown?)
+
+      begin
+        entries = Storage.rotated_entries("#{@name}.jlog")
+        entries.shift           # skip first journal log
+        for path in entries
+          assert_raise(Storage::PanicError) {
+            st2.apply_journal_log(path)
+          }
+        end
+        assert_equal(false, st2.alive?, 'panic storage')
+        assert_equal(false, st2.shutdown?)
+      ensure
+        st2.shutdown
+      end
+
+      assert_equal(false, st2.alive?, 'shutdown storage')
+      assert_equal(true, st2.shutdown?)
+    end
+
+    def test_apply_journal_log_from_online_backup
+      write_data
+      @st.rotate_journal_log(true)
+
+      other_name = File.join(@test_dir, 'bar')
+      FileUtils.cp("#{@name}.tar", "#{other_name}.tar", :preserve => true)
+      FileUtils.cp("#{@name}.idx", "#{other_name}.idx", :preserve => true)
+
+      other_name = File.join(@test_dir, 'bar')
+      for name in Storage.rotated_entries("#{@name}.jlog")
+        name =~ /\.jlog.*$/ or raise 'mismatch'
+        FileUtils.cp(name, other_name + $&, :preserve => true)
+        FileUtils.rm(name)
+      end
+      Storage.recover(other_name)
+
+      write_data
+      @st.rotate_journal_log(true)
+      @st.shutdown
+
+      other_name = File.join(@test_dir, 'bar')
+      st2 = Storage.new(other_name, :jlog_rotate_size => 1024 * 8)
+      begin
+        for path in Storage.rotated_entries("#{@name}.jlog")
+          st2.apply_journal_log(path)
+        end
+      ensure
+        st2.shutdown
+      end
+
+      assert(FileUtils.cmp("#{@name}.tar", "#{other_name}.tar"), 'DATA should be same.')
+      assert(Index.new.load("#{@name}.idx").to_h ==
+             Index.new.load("#{other_name}.idx").to_h, 'INDEX should be same.')
+    end
   end
 
-  class ReadOnlyStorageFirstOpenTest < Test::Unit::TestCase
+  class ReadOnlyStorageFirstOpenErrorTest < Test::Unit::TestCase
     include Higgs
 
     # for ident(1)
-    CVS_ID = '$Id: test_storage.rb 627 2007-10-12 07:10:13Z toki $'
+    CVS_ID = '$Id: test_storage.rb 676 2007-11-08 16:09:51Z toki $'
 
     def setup
       @test_dir = 'st_test'
@@ -494,7 +816,7 @@ module Higgs::Test
       FileUtils.rm_rf(@test_dir) unless $DEBUG
     end
 
-    def test_read_only_first_open
+    def test_read_only_first_open_error
       assert_raise(Errno::ENOENT) {
         Storage.new(@name, :read_only => true, :logger => @logger)
       }

data/test/test_storage_conf.rb

@@ -7,7 +7,7 @@ require 'test/unit'
 module Higgs::Test
   class StorageConfTest < Test::Unit::TestCase
     # for ident(1)
-    CVS_ID = '$Id: test_storage_conf.rb 559 2007-09-25 15:20:20Z toki $'
+    CVS_ID = '$Id: test_storage_conf.rb 636 2007-10-14 09:43:23Z toki $'
 
     include Higgs
 
@@ -89,14 +89,6 @@ module Higgs::Test
       assert_equal(0, options[:jlog_rotate_max])
     end
 
-    def test_jlog_rotate_service_uri
-      File.open(@conf_path, 'w') {|w|
-	w << "jlog_rotate_service_uri: druby://localhost:14142\n"
-      }
-      options = Storage.load_conf(@conf_path)
-      assert_equal('druby://localhost:14142', options[:jlog_rotate_service_uri])
-    end
-
     def test_properties_cache_limit_size
       File.open(@conf_path, 'w') {|w|
 	w << "properties_cache_limit_size: 256\n"

data/test/test_storage_init_opts.rb

@@ -6,7 +6,7 @@ require 'test/unit'
 module Higgs::Test
   class StorageInitOptionsTest < Test::Unit::TestCase
     # for ident(1)
-    CVS_ID = '$Id: test_storage_init_opts.rb 590 2007-09-30 06:13:54Z toki $'
+    CVS_ID = '$Id: test_storage_init_opts.rb 636 2007-10-14 09:43:23Z toki $'
 
     include Higgs::Storage::InitOptions
 
@@ -25,8 +25,6 @@ module Higgs::Test
       assert_equal(1024 * 256, self.jlog_rotate_size)
       assert_equal(1, @jlog_rotate_max)
       assert_equal(1, self.jlog_rotate_max)
-      assert_equal(nil, @jlog_rotate_service_uri)
-      assert_equal(nil, self.jlog_rotate_service_uri)
       assert_instance_of(Proc, @Logger)
     end
 
@@ -101,12 +99,6 @@ module Higgs::Test
       assert_equal(100, self.jlog_rotate_max)
     end
 
-    def test_jlog_rotate_service_uri
-      init_options(:jlog_rotate_service_uri => 'druby://localhost:14142')
-      assert_equal('druby://localhost:14142', @jlog_rotate_service_uri)
-      assert_equal('druby://localhost:14142', self.jlog_rotate_service_uri)
-    end
-
     def test_logger
       init_options(:logger => proc{|path| :dummy_logger })
       assert_equal(:dummy_logger, @Logger.call('foo'))