higgs 0.1.0 → 0.1.1

data/lib/higgs/utils/bman.rb

@@ -1,8 +1,8 @@
 # = backup manager
 #
 # Author:: $Author: toki $
-# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
-# Revision:: $Revision: 559 $
+# Date:: $Date: 2007-10-08 22:10:08 +0900 (Mon, 08 Oct 2007) $
+# Revision:: $Revision: 621 $
 #
 # == license
 #   :include:LICENSE
@@ -10,6 +10,7 @@
 
 require 'drb'
 require 'fileutils'
+require 'higgs/flock'
 require 'higgs/storage'
 
 module Higgs
@@ -31,22 +32,24 @@ module Higgs
     # simple online-backup is like this...
     #
     #   % higgs_backup -v -f foo -t backup_dir -u druby://localhost:17320
-    #   2007-09-23 03:00:08.925 [23706]: **** START BACKUP SCENARIO ****
-    #   2007-09-23 03:00:08.936 [23706]: start index backup.
-    #   2007-09-23 03:00:09.331 [23706]: completed index backup.
-    #   2007-09-23 03:00:09.333 [23706]: start data backup.
-    #   2007-09-23 03:09:16.663 [23706]: completed data backup.
-    #   2007-09-23 03:09:16.692 [23706]: start journal log rotation.
-    #   2007-09-23 03:09:17.153 [23706]: completed journal log rotation.
-    #   2007-09-23 03:09:17.154 [23706]: start journal logs backup.
-    #   2007-09-23 03:09:17.205 [23706]: completed journal logs backup.
-    #   2007-09-23 03:09:17.206 [23706]: start backup storage recovery.
-    #   2007-09-23 03:09:17.798 [23706]: completed backup storage recovery.
-    #   2007-09-23 03:09:17.799 [23706]: start backup storage verify.
-    #   2007-09-23 03:25:44.122 [23706]: completed backup storage verify.
-    #   2007-09-23 03:25:44.140 [23706]: start journal logs clean.
-    #   2007-09-23 03:25:44.541 [23706]: completed journal logs clean.
-    #   2007-09-23 03:25:44.542 [23706]: **** COMPLETED BACKUP SCENARIO ****
+    #   2007-10-03 00:32:58.117 [7558]: **** START BACKUP SCENARIO ****
+    #   2007-10-03 00:32:58.118 [7558]: start index backup.
+    #   2007-10-03 00:32:58.550 [7558]: completed index backup.
+    #   2007-10-03 00:32:58.551 [7558]: start data backup.
+    #   2007-10-03 00:42:00.637 [7558]: completed data backup.
+    #   2007-10-03 00:42:00.665 [7558]: start journal log rotation.
+    #   2007-10-03 00:42:00.907 [7558]: completed journal log rotation.
+    #   2007-10-03 00:42:00.909 [7558]: start journal logs backup.
+    #   2007-10-03 00:42:00.958 [7558]: completed journal logs backup.
+    #   2007-10-03 00:42:00.959 [7558]: start backup storage recovery.
+    #   2007-10-03 00:42:01.550 [7558]: completed backup storage recovery.
+    #   2007-10-03 00:42:01.552 [7558]: start backup storage verify.
+    #   2007-10-03 00:58:56.885 [7558]: completed backup storage verify.
+    #   2007-10-03 00:58:56.904 [7558]: start journal logs clean of from-storage.
+    #   2007-10-03 00:58:56.954 [7558]: completed jounal logs clean of from-storage.
+    #   2007-10-03 00:58:56.955 [7558]: start journal logs clean of to-storage.
+    #   2007-10-03 00:58:56.977 [7558]: completed jounal logs clean of to-storage.
+    #   2007-10-03 00:58:56.978 [7558]: **** COMPLETED BACKUP SCENARIO ****
     #
     # online-backup scenario includes these processes.
     #
@@ -56,32 +59,48 @@ module Higgs
     # 4. journal logs backup. see Higgs::Utils::BackupManager#backup_jlog.
     # 5. backup storage recovery. see Higgs::Utils::BackupManager#recover.
     # 6. backup storage verify. see Higgs::Utils::BackupManager#verify.
-    # 7. journal logs clean. see Higgs::Utils::BackupManager#clean_jlog.
+    # 7. journal logs clean of from-storage. see Higgs::Utils::BackupManager#clean_jlog_from.
+    # 8. journal logs clean of to-storage. see Higgs::Utils::BackupManager#clean_jlog_to.
     #
     # == restore from online-backup
-    # === 0. situation
-    # storage name is `foo' and backup directory is `backup_dir'.
     #
-    # === 1. recovery from last online-backup
-    # run these commands.
-    #   % cp -p backup_dir/foo.idx foo.idx
-    #   % cp -p backup_dir/foo.tar foo.tar
-    #   % higgs_backup -t . -n foo --command recover
+    # simple restore is like this...
     #
-    # === 2. apply last journal log
-    # if system is aborted then last journal log is broken.
-    # Higgs::Storage applies last jounal log to a readable point at
-    # the read-write open.
+    #   % higgs_backup -f images -t ~/misc/photon/dat/1 -v restore
+    #   2007-10-08 20:56:07.048 [18133]: **** START RESTORE SCENARIO ****
+    #   2007-10-08 20:56:07.066 [18133]: start storage files restore.
+    #   2007-10-08 21:09:49.614 [18133]: completed storage files restore.
+    #   2007-10-08 21:09:49.614 [18133]: start restored storage recovery.
+    #   2007-10-08 21:09:51.090 [18133]: completed restored storage recovery.
+    #   2007-10-08 21:09:51.093 [18133]: start restored storage verify.
+    #   2007-10-08 21:13:26.521 [18133]: completed restored storage verify.
+    #   2007-10-08 21:13:26.521 [18133]: **** COMPLETED RESTORE SCENARIO ****
     #
-    # <em>WARNING.</em> Higgs::Storage is normal shutdown and last
-    # journal log is not broken. last journal log is not applied and
-    # storage data is old version. <em>this situation is inconsistent.</em>
+    # restore scenario includes these processes.
+    #
+    # 1. storage files restore. see Higgs::Utils::BackupManager#restore_files.
+    # 2. restored storage recovery. see Higgs::Utils::BackupManager#restore_recover.
+    # 3. restored storage verify. see Higgs::Utils::BackupManager#restore_verify.
     #
     # == command-line options
     #
     #   % higgs_backup --help
-    #   Usage: higgs_backup [options]
-    #           --command=BACKUP_COMMAND
+    #   Usage: higgs_backup [OPTIONs] [COMMANDs]
+    #   COMMANDs:
+    #       online_backup
+    #       index
+    #       data
+    #       rotate
+    #       jlog
+    #       recover
+    #       verify
+    #       clean_from
+    #       clean_to
+    #       restore
+    #       restore_files
+    #       restore_recover
+    #       restore_verify
+    #   OPTIONs:
     #       -f, --from=BACKUP_TARGET_STORAGE
     #       -t, --to-dir=DIR_TO_BACKUP
     #       -n, --to-name=NAME_TO_BACKUP
@@ -90,9 +109,9 @@ module Higgs
     #       -v, --verbose, --[no-]verbose
     #           --verbose-level=LEVEL
     #
-    # === option: <tt>--command=BACKUP_COMMAND</tt>
+    # === COMMANDs
     # select a process of online-backup.
-    # <tt>BACKUP_COMMAND</tt>s are these.
+    # COMMANDs for online-backup are these.
     #
     # <tt>online_backup</tt>:: default. run online-backup scenario.
     #                          see Higgs::Utils::BackupManager#online_backup.
@@ -102,33 +121,44 @@ module Higgs
     # <tt>jlog</tt>:: journal logs backup. see Higgs::Utils::BackupManager#backup_jlog.
     # <tt>recover</tt>:: backup storage recovery. see Higgs::Utils::BackupManager#recover.
     # <tt>verify</tt>:: backup storage verify. see Higgs::Utils::BackupManager#verify.
-    # <tt>clean</tt>:: journal logs clean. see Higgs::Utils::BackupManager#clean_jlog.
+    # <tt>clean_from</tt>:: journal logs clean. see Higgs::Utils::BackupManager#clean_jlog_from.
+    # <tt>clean_to</tt>:: journal logs clean. see Higgs::Utils::BackupManager#clean_jlog_to.
+    #
+    # COMMANDs for restore are these.
+    #
+    # <tt>restore</tt>:: run restore scenario. see Higgs::Utils::BackupManager#restore.
+    # <tt>restore_files</tt>:: storage files restore.
+    #                          see Higgs::Utils::BackupManager#restore_files.
+    # <tt>restore_recover</tt>:: restored storage recovery.
+    #                            see Higgs::Utils::BackupManager#restore_recover.
+    # <tt>restore_verify</tt>:: restored storage verify.
+    #                           see Higgs::Utils::BackupManager#restore_verify.
     #
-    # === option: <tt>--from=BACKUP_TARGET_STORAGE</tt>
+    # === OPTION: <tt>--from=BACKUP_TARGET_STORAGE</tt>
     # <tt>BACKUP_TARGET_STORAGE</tt> is the name of backup target storage.
     #
-    # === option: <tt>--to-dir=DIR_TO_BACKUP</tt>
+    # === OPTION: <tt>--to-dir=DIR_TO_BACKUP</tt>
     # backuped storage is copied to the directory of <tt>DIR_TO_BACKUP</tt>.
     #
-    # === option: <tt>--to-name=NAME_TO_BACKUP</tt>
-    # <tt>NAME_to_BACKUP</tt> is the name of backuped storage.
+    # === OPTION: <tt>--to-name=NAME_TO_BACKUP</tt>
+    # <tt>NAME_TO_BACKUP</tt> is the name of backuped storage.
     # if this option is omitted then <tt>NAME_TO_BACKUP</tt> is the same
     # as <tt>BACKUP_TARGET_STORAGE</tt>.
     #
-    # === option: <tt>--jlog-rotate-service-uri=URI</tt>
+    # === OPTION: <tt>--jlog-rotate-service-uri=URI</tt>
     # access point journal log rotation remote service.
     # <tt>URI</tt> is the same as <tt>:jlog_rotate_service_uri</tt>
     # when Higgs::Storage is opened.
     #
-    # === option: <tt>--verbose</tt>
+    # === OPTION: <tt>--verbose</tt>
     # verbose level up.
     #
-    # === option: <tt>--verbose-level=LEVEL</tt>
+    # === OPTION: <tt>--verbose-level=LEVEL</tt>
     # set verbose level to <tt>LEVEL</tt>.
     #
     class BackupManager
       # for ident(1)
-      CVS_ID = '$Id: bman.rb 559 2007-09-25 15:20:20Z toki $'
+      CVS_ID = '$Id: bman.rb 621 2007-10-08 13:10:08Z toki $'
 
       def initialize(options={})
         @from = options[:from]
@@ -201,7 +231,7 @@ module Higgs
         unless (@to) then
           raise 'required to_storage'
         end
-        for path in Storage.rotate_entries(@from + '.jlog')
+        for path in Storage.rotate_entries("#{@from}.jlog")
           path =~ /\.jlog\.\d+$/ or raise "mismatch jlog name: #{path}"
           ext = $&
           FileUtils.cp(path, "#{@to}#{ext}", :preserve => true, :verbose => @verbose >= 2)
@@ -235,8 +265,8 @@ module Higgs
         nil
       end
 
-      def clean_jlog
-        @out << log('start journal logs clean.') if (@verbose >= 1)
+      def clean_jlog_from
+        @out << log('start journal logs clean of from-storage.') if (@verbose >= 1)
 
         unless (@from) then
           raise 'required from_storage'
@@ -254,11 +284,22 @@ module Higgs
           end
         end
 
+        @out << log('completed jounal logs clean of from-storage.') if (@verbose >= 1)
+        nil
+      end
+
+      def clean_jlog_to
+        @out << log('start journal logs clean of to-storage.') if (@verbose >= 1)
+
+        unless (@to) then
+          raise 'required to_storage'
+        end
+
         for to_jlog in Storage.rotate_entries("#{@to}.jlog")
           FileUtils.rm(to_jlog, :verbose => @verbose >= 2)
         end
 
-        @out << log('completed journal logs clean.') if (@verbose >= 1)
+        @out << log('completed jounal logs clean of to-storage.') if (@verbose >= 1)
         nil
       end
 
@@ -271,10 +312,67 @@ module Higgs
         backup_jlog
         recover
         verify
-        clean_jlog
+        clean_jlog_from
+        clean_jlog_to
         @out << log('**** COMPLETED BACKUP SCENARIO ****') if (@verbose >= 1)
         nil
       end
+
+      def restore_files
+        @out << log('start storage files restore.') if (@verbose >= 1)
+        unless (@from) then
+          raise 'required from_storage'
+        end
+        unless (@to) then
+          raise 'required to_storage'
+        end
+        FileLock.new("#{@from}.lock").synchronize{
+          FileUtils.cp("#{@to}.idx", "#{@from}.idx", :preserve => true, :verbose => @verbose >= 2)
+          FileUtils.cp("#{@to}.tar", "#{@from}.tar", :preserve => true, :verbose => @verbose >= 2)
+          for path in Storage.rotate_entries("#{@to}.jlog")
+            path =~ /\.jlog\.\d+$/ or raise "mismatch jlog name: #{path}"
+            ext = $&
+            FileUtils.cp(path, "#{@from}#{ext}", :preserve => true, :verbose => @verbose >= 2)
+          end
+        }
+        @out << log('completed storage files restore.') if (@verbose >= 1)
+        nil
+      end
+
+      def restore_recover
+        @out << log('start restored storage recovery.') if (@verbose >= 1)
+        unless (@from) then
+          raise 'required from_storage'
+        end
+        Storage.recover(@from, @out, @verbose - 1)
+        @out << log('completed restored storage recovery.') if (@verbose >= 1)
+        nil
+      end
+
+      def restore_verify
+        @out << log('start restored storage verify.') if (@verbose >= 1)
+        unless (@from) then
+          raise 'required from_storage'
+        end
+        st = Storage.new(@from)   # read-write open for recovery
+        begin
+          st.verify(@out, @verbose - 1)
+        ensure
+          st.shutdown
+        end
+        @out << log('completed restored storage verify.') if (@verbose >= 1)
+        nil
+      end
+
+      # run restore scenario
+      def restore
+        @out << log('**** START RESTORE SCENARIO ****') if (@verbose >= 1)
+        restore_files
+        restore_recover
+        restore_verify
+        @out << log('**** COMPLETED RESTORE SCENARIO ****') if (@verbose >= 1)
+        nil
+      end
     end
   end
 end

data/lib/higgs/version.rb

@@ -1,8 +1,8 @@
 # = version
 #
 # Author:: $Author: toki $
-# Date:: $Date: 2007-09-29 18:06:25 +0900 (Sat, 29 Sep 2007) $
-# Revision:: $Revision: 578 $
+# Date:: $Date: 2007-10-08 22:16:13 +0900 (Mon, 08 Oct 2007) $
+# Revision:: $Revision: 622 $
 #
 # == license
 #   :include:LICENSE
@@ -10,9 +10,9 @@
 
 module Higgs
   # for ident(1)
-  CVS_ID = '$Id: version.rb 578 2007-09-29 09:06:25Z toki $'
+  CVS_ID = '$Id: version.rb 622 2007-10-08 13:16:13Z toki $'
 
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end
 
 # Local Variables:

data/mkrdoc.rb

@@ -1,8 +1,11 @@
 #!/usr/local/bin/ruby
 
-CVS_ID = '$Id: mkrdoc.rb 564 2007-09-27 16:10:25Z toki $'			# for ident(1)
+# for ident(1)
+CVS_ID = '$Id: mkrdoc.rb 604 2007-10-02 15:28:03Z toki $'
 
+require 'fileutils'
 require 'yaml'
 
 rdoc_opts = YAML.load(IO.read('rdoc.yml'))
-system 'rdoc', *(rdoc_opts['CommonOptions'] + rdoc_opts['CommandLineOptions']).flatten
+FileUtils.rm_rf(rdoc_opts['CommandLineOptions'].assoc('-o')[1], :verbose => true)
+system('rdoc', *(rdoc_opts['CommonOptions'] + rdoc_opts['CommandLineOptions']).flatten)

data/test/test_block.rb

@@ -10,7 +10,7 @@ module Higgs::Test
     include Higgs::Block
 
     # for ident(1)
-    CVS_ID = '$Id: test_block.rb 559 2007-09-25 15:20:20Z toki $'
+    CVS_ID = '$Id: test_block.rb 591 2007-09-30 06:20:15Z toki $'
 
     def setup
       @io = File.open('block.test_io', 'w+')
@@ -154,6 +154,64 @@ module Higgs::Test
         block_read(@io, 'FOO')
       }
     end
+
+    def test_head_write_BrokenError_short_write
+      def @io.write(*args)
+        super
+        0
+      end
+
+      body = 'foo'
+      body_cksum_bin = Digest::SHA512.digest(body)
+      assert_raise(BrokenError) {
+        head_write(@io, 'FOO', body.length, 'SHA512', body_cksum_bin)
+      }
+    end
+
+    def test_block_write_body_BrokenError_short_write
+      def @io.write(*args)
+        @count = 0 unless @count
+        @count += 1
+        r = super(*args)
+        case (@count)
+        when 1
+          return r
+        else
+          return 0
+        end
+      end
+
+      body = 'foo'
+      assert_raise(BrokenError) {
+        block_write(@io, 'FOO', body)
+      }
+    end
+
+    def test_block_write_padding_BrokenError_short_write
+      def @io.write(*args)
+        @count = 0 unless @count
+        @count += 1
+        r = super(*args)
+        case (@count)
+        when 1, 2
+          return r
+        else
+          return 0
+        end
+      end
+
+      body = 'foo'
+      assert_raise(BrokenError) {
+        block_write(@io, 'FOO', body)
+      }
+    end
+
+    def test_block_write_ArgumentError_unknown_body_hash_type
+      body = 'foo'
+      assert_raise(ArgumentError) {
+        block_write(@io, 'FOO', body, :UNKNWON)
+      }
+    end
   end
 end
 

data/test/test_cache.rb

@@ -10,7 +10,7 @@ module Higgs::Test
     include Higgs
 
     # for ident(1)
-    CVS_ID = '$Id: test_cache.rb 559 2007-09-25 15:20:20Z toki $'
+    CVS_ID = '$Id: test_cache.rb 588 2007-09-30 05:39:54Z toki $'
 
     CACHE_LIMIT_SIZE = 10
 
@@ -84,7 +84,7 @@ module Higgs::Test
     include Higgs
 
     # for ident(1)
-    CVS_ID = '$Id: test_cache.rb 559 2007-09-25 15:20:20Z toki $'
+    CVS_ID = '$Id: test_cache.rb 588 2007-09-30 05:39:54Z toki $'
 
     def setup
       @calc_calls = 0
@@ -192,13 +192,27 @@ module Higgs::Test
         assert_equal(2, @calc_calls, "#{n}th")
       end
     end
+
+    def test_store
+      @cache[WORK_COUNT] = 0
+      assert_equal(0, @cache[WORK_COUNT])
+      assert_equal(0, @calc_calls, 'no call')
+    end
+
+    def test_store_overwrite
+      assert(@cache[WORK_COUNT] != 0)
+      assert_equal(1, @calc_calls)
+      @cache[WORK_COUNT] = 0
+      assert_equal(0, @cache[WORK_COUNT])
+      assert_equal(1, @calc_calls)
+    end
   end
 
   class SharedWorkCacheNoWorkBlockTest < Test::Unit::TestCase
     include Higgs
 
     # for ident(1)
-    CVS_ID = '$Id: test_cache.rb 559 2007-09-25 15:20:20Z toki $'
+    CVS_ID = '$Id: test_cache.rb 588 2007-09-30 05:39:54Z toki $'
 
     def test_no_work_block
       assert_raise(RuntimeError) {

data/test/test_dbm.rb

@@ -0,0 +1,163 @@
+#!/usr/local/bin/ruby
+
+require 'fileutils'
+require 'higgs/dbm'
+require 'logger'
+require 'test/unit'
+
+module Higgs::Test
+  class DBMTest < Test::Unit::TestCase
+    include Higgs
+
+    # for ident(1)
+    CVS_ID = '$Id: test_dbm.rb 597 2007-10-01 15:21:47Z toki $'
+
+    def setup
+      @test_dir = 'dbm_test'
+      FileUtils.rm_rf(@test_dir) # for debug
+      FileUtils.mkdir_p(@test_dir)
+      @name = File.join(@test_dir, 'foo')
+      @db = DBM.new(@name,
+                    :logger => proc{|path|
+                      logger = Logger.new(path, 1)
+                      logger.level = Logger::DEBUG
+                      logger
+                    })
+    end
+
+    def teardown
+      @db.shutdown unless @db.shutdown?
+      FileUtils.rm_rf(@test_dir) unless $DEBUG
+    end
+
+    def test_fetch_and_store
+      @db.transaction{|tx|
+        assert_equal(nil, tx['foo'])
+        assert_equal(nil, tx['bar'])
+
+        assert_equal(false, (tx.key? 'foo'))
+        assert_equal(false, (tx.key? 'bar'))
+
+        tx['foo'] = 'HALO'
+        tx['bar'] = ''
+
+        assert_equal('HALO', tx['foo'])
+        assert_equal('', tx['bar'])
+
+        assert_equal(true, (tx.key? 'foo'))
+        assert_equal(true, (tx.key? 'bar'))
+      }
+
+      @db.transaction{|tx|
+        assert_equal('HALO', tx['foo'])
+        assert_equal('', tx['bar'])
+
+        assert_equal(true, (tx.key? 'foo'))
+        assert_equal(true, (tx.key? 'bar'))
+      }
+    end
+
+    def test_store_TypeError_cant_convert_into_String
+      @db.transaction{|tx|
+        tx['foo'] = :foo
+        assert_raise(TypeError) { tx.commit }
+        tx.rollback
+      }
+    end
+
+    def test_update_no_use
+      @db.transaction{|tx|
+        tx['foo'] = ''
+      }
+
+      @db.transaction{|tx|
+        tx.update('foo') {|s|
+          assert_equal(true, s.frozen?)
+          assert_raise(TypeError) {
+            s << 'foo' << ',' << 'bar' << ',' << 'baz'
+          }
+        }
+
+        assert_equal('', tx['foo'], 'not updated')
+      }
+    end
+
+    def test_rollback
+      @db.transaction{|tx|
+        tx['foo'] = 'apple'
+      }
+
+      @db.transaction{|tx|
+        assert_equal('apple', tx['foo'])
+        tx['foo'] = 'banana'
+        assert_equal('banana', tx['foo'])
+        tx.rollback
+        assert_equal('apple', tx['foo'])
+      }
+
+      @db.transaction{|tx|
+        assert_equal('apple', tx['foo'])
+      }
+    end
+
+    def test_each_key
+      @db.transaction{|tx|
+        tx['foo'] = 'HALO'
+        tx['bar'] = ''
+
+        expected_keys = %w[ foo bar ]
+        tx.each_key do |key|
+          assert((expected_keys.include? key), key)
+          expected_keys.delete(key)
+        end
+        assert_equal([], expected_keys)
+      }
+
+      @db.transaction{|tx|
+        expected_keys = %w[ foo bar ]
+        tx.each_key do |key|
+          assert((expected_keys.include? key), key)
+          expected_keys.delete(key)
+        end
+        assert_equal([], expected_keys)
+      }
+    end
+  end
+
+  class DBMOpenTest < Test::Unit::TestCase
+    include Higgs
+
+    # for ident(1)
+    CVS_ID = '$Id: test_dbm.rb 597 2007-10-01 15:21:47Z toki $'
+
+    def setup
+      @test_dir = 'dbm_test'
+      FileUtils.rm_rf(@test_dir) # for debug
+      FileUtils.mkdir_p(@test_dir)
+      @name = File.join(@test_dir, 'foo')
+    end
+
+    def teardown
+      FileUtils.rm_rf(@test_dir) unless $DEBUG
+    end
+
+    def test_open
+      DBM.open(@name) {|db|
+        db.transaction{|tx|
+          tx['foo'] = 'apple'
+        }
+      }
+
+      DBM.open(@name, :read_only => true) {|db|
+        db.transaction{|tx|
+          assert_equal('apple', tx['foo'])
+        }
+      }
+    end
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End: