higgs 0.1.2 → 0.1.3

data/ChangeLog

@@ -1,3 +1,106 @@
+2007-11-11  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/version.rb: version 0.1.3.
+
+2007-11-10  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* bin/higgs_apply_jlog, bin/higgs_ping: some utilities.
+
+2007-11-09  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb (Higgs::Storage class): probe for alived
+	storage.
+
+2007-11-04  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb (Higgs::Storage class): standby mode check.
+
+	* lib/higgs/tman.rb (Higgs::TransactionManager class),
+	lib/higgs/services.rb (Higgs::RemoteServices class),
+	lib/higgs/sman.rb (Higgs::StorageManager class):
+	some methods for replication.
+
+2007-10-31  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/thread.rb (Higgs::SharedWork class),
+	lib/higgs/cache.rb (Higgs::SharedWorkCache class): ArgumentError
+	for no block argument.
+
+2007-10-28  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/utils/bman.rb (Higgs::Utils::BackupManager class):
+	checked eof-mark of journal log to backup.
+
+2007-10-25  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb (Higgs::Storage class): fixed a bug, buffer
+	flush of write I/O handle is not done.
+
+2007-10-24  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb (Higgs::Storage class): renamed from
+	`rotate_entries' to `rotated_entries'.
+
+2007-10-23  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/lock.rb: exclusive block of lock manager.
+
+2007-10-21  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb (Higgs::Storage class): storage id
+	consistency check on applying journal log.
+
+	* lib/higgs/index.rb (Higgs::Index class),
+	lib/higgs/storage.rb (Higgs::Storage class): storage id.
+
+2007-10-16  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/utils/bman.rb (Higgs::Utils::BackupManager class):
+	localhost check for runtime.
+
+	* lib/higgs/storage.rb (Higgs::Storage class),
+	lib/higgs/services.rb (Higgs::RemoteServices class): provided
+	localhost check service.
+
+2007-10-14  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb, lib/higgs/tman.rb: obsolete
+	Higgs::Storage::Export module, obsolete
+	Higgs::TransactionManager::Export module.
+
+	* lib/higgs/sman.rb (Higgs::StorageManager class): base class of
+	Higgs::Store and Higgs::DBM. this class integrates Higgs::Storage,
+	Higgs::TransactionManager, and Higgs::RemoteServices and is those
+	front end.
+
+	* bin/higgs_backup: obsolete `--jlog-rotate-service-uri=URI'
+	option, new `--remote-services-uri=URI' option.
+
+	* lib/higgs/utils/bman.rb: obsolete `:jlog_rotate_service_uri'
+	option, new `:remote_services_uri' option.
+
+	* lib/higgs/storage.rb (Higgs::Storage class): obsolete
+	`:jlog_rotate_service_uri' option. this remote service is
+	integrated to Higgs::RemoteServices.
+
+	* lib/higgs/services.rb (Higgs::RemoteServices class): class to
+	integrate remote services.
+
+	* lib/higgs/storage.rb (Higgs::Storage class): enabled to apply
+	journal log to online storage.
+
+2007-10-13  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb (Higgs::Storage class): write EOA to
+	storage at each applying journal log.
+
+	* lib/higgs/storage.rb (Higgs::Storage class), lib/higgs/tman.rb
+	(Higgs::TransactionManager class, Higgs::TransactionContext
+	class): unique data id for secondary cache. index structure
+	changed a little.
+
+	* lib/higgs/tman.rb: fixed a bug of wrong key of secondary cache.
+
 2007-10-12  TOKI Yoshinori  <toki@freedom.ne.jp>
 
 	* lib/higgs/version.rb: version 0.1.2.

data/Rakefile

@@ -1,5 +1,5 @@
 # for ident(1)
-CVS_ID = '$Id: Rakefile 586 2007-09-30 03:03:12Z toki $'
+CVS_ID = '$Id: Rakefile 678 2007-11-09 16:08:34Z toki $'
 
 require 'lib/higgs/version'
 require 'rake/gempackagetask'
@@ -46,7 +46,14 @@ spec = Gem::Specification.new{|s|
   s.summary = 'pure ruby transactional storage compatible with unix TAR format'
   s.author = 'TOKI Yoshinori'
   s.email = 'toki@freedom.ne.jp'
-  s.executables  = %w[ higgs_backup higgs_dump_index higgs_dump_jlog higgs_verify ]
+  s.executables  = %w[
+    higgs_apply_jlog
+    higgs_backup
+    higgs_dump_index
+    higgs_dump_jlog
+    higgs_ping
+    higgs_verify
+  ]
   s.files = Dir['**/{Rakefile,.strc,*.{rb,yml}}']
   s.files += %w[ ChangeLog LICENSE README ]
   s.test_files = %w[ test/run.rb ]

data/bin/higgs_apply_jlog

@@ -0,0 +1,35 @@
+#!/usr/local/bin/ruby
+
+require 'drb'
+require 'optparse'
+
+# for ident(1)
+CVS_ID = '$Id: higgs_apply_jlog 680 2007-11-11 02:04:47Z toki $'
+
+opt_not_delete = false
+opt_switch_to_write = false
+
+opts = OptionParser.new
+opts.banner = "Usage: #{opts.program_name} [OPTIONs] URI_FOR_REMOTE_SERVICES\n"
+opts.banner << "OPTIONs:\n"
+opts.on('-n', '--not-delete') {|value|
+  opt_not_delete = value ? true : false
+}
+opts.on('--switch-to-write') {|value|
+  opt_switch_to_write = true
+}
+opts.parse!
+
+if (ARGV.empty?) then
+  puts opts
+  exit 1
+end
+
+uri = ARGV.shift
+services = DRbObject.new_with_uri(uri)
+jlog_apply_service = services[:jlog_apply_service_v1] or raise 'not provided remote service: jlog_apply_service_v1'
+jlog_apply_service.call(opt_not_delete)
+if (opt_switch_to_write) then
+  switch_to_write_service = services[:switch_to_write_service_v1] or raise 'not provided remote service: switch_to_write_service_v1'
+  switch_to_write_service.call
+end

data/bin/higgs_backup

@@ -1,10 +1,11 @@
 #!/usr/local/bin/ruby
 
+require 'drb'
 require 'higgs/utils/bman'
 require 'optparse'
 
 # for ident(1)
-CVS_ID = '$Id: higgs_backup 619 2007-10-08 12:42:36Z toki $'
+CVS_ID = '$Id: higgs_backup 641 2007-10-18 16:04:02Z toki $'
 
 STDOUT.sync = true
 
@@ -42,8 +43,8 @@ opts.on('-t', '--to-dir=DIR_TO_BACKUP', String) {|value|
 opts.on('-n', '--to-name=NAME_TO_BACKUP', String) {|value|
   options[:to_name] = value
 }
-opts.on('-u', '--jlog-rotate-service-uri=URI', String) {|value|
-  options[:jlog_rotate_service_uri] = value
+opts.on('-U', '--remote-services-uri=URI', String) {|value|
+  options[:remote_services_uri] = value
 }
 opts.on('-v', '--verbose', '--[no-]verbose') {|value|
   if (value) then
@@ -71,6 +72,7 @@ unless (ARGV.empty?) then
   commands = ARGV
 end
 
+DRb.start_service
 bman = Higgs::Utils::BackupManager.new(options)
 for cmd in commands
   unless (bman.public_methods(false).include? cmd) then

data/bin/higgs_dump_jlog

@@ -6,7 +6,7 @@ require 'pp'
 require 'yaml'
 
 # for ident(1)
-CVS_ID = '$Id: higgs_dump_jlog 462 2007-07-15 09:25:07Z toki $'
+CVS_ID = '$Id: higgs_dump_jlog 677 2007-11-08 16:22:02Z toki $'
 
 opts = OptionParser.new
 opts.banner = "Usage: #{opts.program_name} [options] [JOURNAL_LOG_PATHs]"
@@ -25,16 +25,32 @@ opts.on_tail('-h', '--help') {
 }
 opts.parse!
 
-Higgs::JournalLogger.scan_log(ARGF) {|log|
-  case (output_format)
-  when :pretty
-    pp log
-  when :yaml
-    puts log.to_yaml
+def scan_args
+  if (ARGV.empty?) then
+    yield(STDIN)
   else
-    raise "unknown output format: #{output_format}"
+    for filename in ARGV
+      File.open(filename, 'r') {|r|
+        r.binmode
+        yield(r)
+      }
+    end
   end
-}
+  nil
+end
+
+scan_args do |r|
+  Higgs::JournalLogger.scan_log(r) do |log|
+    case (output_format)
+    when :pretty
+      pp log
+    when :yaml
+      puts log.to_yaml
+    else
+      raise "unknown output format: #{output_format}"
+    end
+  end
+end
 
 # Local Variables:
 # mode: Ruby

data/bin/higgs_ping

@@ -0,0 +1,46 @@
+#!/usr/local/bin/ruby
+
+require 'drb'
+require 'optparse'
+
+# for ident(1)
+CVS_ID = '$Id: higgs_ping 679 2007-11-11 01:41:18Z toki $'
+
+opt_verbose = 0
+
+opts = OptionParser.new
+opts.banner = "Usage: #{opts.program_name} [OPTIONs] URI_FOR_REMOTE_SERVICES\n"
+opts.banner << "OPTIONs:\n"
+opts.on('-v', '--[no-]verbose') {|value|
+  if (value) then
+    opt_verbose += 1
+  else
+    opt_verbose -= 1
+  end
+}
+opts.on('--verbose-level=LEVEL', Integer) {|value|
+  opt_verbose = value
+}
+opts.parse!
+
+if (ARGV.empty?) then
+  puts opts
+  exit 1
+end
+
+uri = ARGV.shift
+begin
+  puts "connect to #{uri}" if (opt_verbose >= 2)
+  services = DRbObject.new_with_uri(uri)
+  alive_service = services[:alive_service_v1] or raise 'not provided remote service: alive_service_v1'
+  if (alive_service.call) then
+    puts "alive" if (opt_verbose >= 1)
+    exit 0
+  end
+rescue DRb::DRbError
+  # fall through
+  puts "error: #{$!}" if (opt_verbose >= 2)
+end
+
+puts "dead" if (opt_verbose >= 1)
+exit 255

data/lib/higgs/block.rb

@@ -1,13 +1,14 @@
 # = block read/write
 #
 # Author:: $Author: toki $
-# Date:: $Date: 2007-09-30 15:20:15 +0900 (Sun, 30 Sep 2007) $
-# Revision:: $Revision: 591 $
+# Date:: $Date: 2007-10-24 01:04:47 +0900 (Wed, 24 Oct 2007) $
+# Revision:: $Revision: 654 $
 #
 # == license
 #   :include:LICENSE
 #
 
+require 'digest'
 require 'higgs/exceptions'
 
 module Higgs
@@ -27,7 +28,7 @@ module Higgs
   #
   module Block
     # for ident(1)
-    CVS_ID = '$Id: block.rb 591 2007-09-30 06:20:15Z toki $'
+    CVS_ID = '$Id: block.rb 654 2007-10-23 16:04:47Z toki $'
 
     include Exceptions
 
@@ -42,26 +43,17 @@ module Higgs
     HEAD_CKSUM_FMT = 'v'
 
     BODY_HASH = {}
-    [ [ :SUM16,  proc{|s| s.sum(16).to_s           },  nil            ],
-      [ :MD5,    proc{|s| Digest::MD5.digest(s)    }, 'digest/md5'    ],
-      [ :RMD160, proc{|s| Digest::RMD160.digest(s) }, 'digest/rmd160' ],
-      [ :SHA1,   proc{|s| Digest::SHA1.digest(s)   }, 'digest/sha1'   ],
-      [ :SHA256, proc{|s| Digest::SHA256.digest(s) }, 'digest/sha2'   ],
-      [ :SHA384, proc{|s| Digest::SHA384.digest(s) }, 'digest/sha2'   ],
-      [ :SHA512, proc{|s| Digest::SHA512.digest(s) }, 'digest/sha2'   ]
-    ].each do |hash_symbol, hash_proc, hash_lib|
-      if (hash_lib) then
-        begin
-          require(hash_lib)
-        rescue LoadError
-          next
-        end
-      end
-      BODY_HASH[hash_symbol] = hash_proc
-    end
-
     BODY_HASH_BIN = {}
-    BODY_HASH.each do |hash_symbol, hash_proc|
+
+    [ [ :SUM16,  proc{|s| s.sum(16).to_s           } ],
+      [ :MD5,    proc{|s| Digest::MD5.digest(s)    } ],
+      [ :RMD160, proc{|s| Digest::RMD160.digest(s) } ],
+      [ :SHA1,   proc{|s| Digest::SHA1.digest(s)   } ],
+      [ :SHA256, proc{|s| Digest::SHA256.digest(s) } ],
+      [ :SHA384, proc{|s| Digest::SHA384.digest(s) } ],
+      [ :SHA512, proc{|s| Digest::SHA512.digest(s) } ]
+    ].each do |hash_symbol, hash_proc|
+      BODY_HASH[hash_symbol] = hash_proc
       BODY_HASH_BIN[hash_symbol.to_s] = hash_proc
     end
 

data/lib/higgs/cache.rb

@@ -1,8 +1,8 @@
 # = cache utilities
 #
 # Author:: $Author: toki $
-# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
-# Revision:: $Revision: 559 $
+# Date:: $Date: 2007-10-31 00:36:09 +0900 (Wed, 31 Oct 2007) $
+# Revision:: $Revision: 659 $
 #
 # == license
 #   :include:LICENSE
@@ -16,7 +16,7 @@ module Higgs
   # = cache by Least Recently Used strategy
   class LRUCache
     # for ident(1)
-    CVS_ID = '$Id: cache.rb 559 2007-09-25 15:20:20Z toki $'
+    CVS_ID = '$Id: cache.rb 659 2007-10-30 15:36:09Z toki $'
 
     extend Forwardable
 
@@ -73,11 +73,11 @@ module Higgs
   # = multi-thread safe cache
   class SharedWorkCache
     # for ident(1)
-    CVS_ID = '$Id: cache.rb 559 2007-09-25 15:20:20Z toki $'
+    CVS_ID = '$Id: cache.rb 659 2007-10-30 15:36:09Z toki $'
 
     def initialize(cache={}, &work)
       unless (work) then
-        raise 'required work block'
+        raise ArgumentError, 'required work block'
       end
       @work = work
       @lock = Mutex.new

data/lib/higgs/dbm.rb

@@ -1,15 +1,14 @@
 # = storage like dbm
 #
 # Author:: $Author: toki $
-# Date:: $Date: 2007-10-02 01:05:07 +0900 (Tue, 02 Oct 2007) $
-# Revision:: $Revision: 600 $
+# Date:: $Date: 2007-10-24 00:46:00 +0900 (Wed, 24 Oct 2007) $
+# Revision:: $Revision: 653 $
 #
 # == license
 #   :include:LICENSE
 #
 
-require 'higgs/storage'
-require 'higgs/tman'
+require 'higgs/sman'
 
 module Higgs
   # = storage like dbm
@@ -53,34 +52,19 @@ module Higgs
   #   custom_property[number]: 0
   #   %
   #
-  class DBM
+  class DBM < StorageManager
     # for ident(1)
-    CVS_ID = '$Id: dbm.rb 600 2007-10-01 16:05:07Z toki $'
-
-    include Storage::Export
-    include TransactionManager::Export
+    CVS_ID = '$Id: dbm.rb 653 2007-10-23 15:46:00Z toki $'
 
     DECODE = proc{|r| r }
     ENCODE = proc{|w| w }
 
-    # <tt>name</tt> is a storage name and see Higgs::Storage.new for
-    # detail. see Higgs::Storage::InitOptions and
-    # Higgs::TransactionManager::InitOptions for <tt>options</tt>.
+    # see Higgs::StorageManager.new for <tt>name</tt> and <tt>options</tt>.
     def initialize(name, options={})
-      @storage = Storage.new(name, options)
+      options = options.dup
       options[:decode] = DECODE
       options[:encode] = ENCODE
-      @tman = TransactionManager.new(@storage, options)
-    end
-
-    def self.open(*args)
-      dbm = new(*args)
-      begin
-        r = yield(dbm)
-      ensure
-        dbm.shutdown
-      end
-      r
+      super(name, options)
     end
   end
 end

data/lib/higgs/index.rb

@@ -1,8 +1,8 @@
 # = storage index
 #
 # Author:: $Author: toki $
-# Date:: $Date: 2007-09-30 14:54:30 +0900 (Sun, 30 Sep 2007) $
-# Revision:: $Revision: 589 $
+# Date:: $Date: 2007-10-21 22:40:57 +0900 (Sun, 21 Oct 2007) $
+# Revision:: $Revision: 649 $
 #
 # == license
 #   :include:LICENSE
@@ -15,14 +15,14 @@ module Higgs
   # = storage index
   class Index
     # for ident(1)
-    CVS_ID = '$Id: index.rb 589 2007-09-30 05:54:30Z toki $'
+    CVS_ID = '$Id: index.rb 649 2007-10-21 13:40:57Z toki $'
 
     extend Forwardable
     include Block
 
     MAGIC_SYMBOL = 'HIGGS_INDEX'
     MAJOR_VERSION = 0
-    MINOR_VERSION = 1
+    MINOR_VERSION = 2
 
     def initialize
       @change_number = 0
@@ -30,10 +30,12 @@ module Higgs
       @free_lists = {}
       @index = {}
       @identities = {}
+      @storage_id = nil
     end
 
     attr_reader :change_number
     attr_accessor :eoa
+    attr_accessor :storage_id
 
     def succ!
       @change_number = @change_number.succ
@@ -104,7 +106,8 @@ module Higgs
         :eoa => @eoa,
         :free_lists => @free_lists,
         :index => @index,
-        :identities => @identities
+        :identities => @identities,
+        :storage_id => @storage_id
       }
     end
 
@@ -130,6 +133,21 @@ module Higgs
     end
     private :migration_0_0_to_0_1
 
+    def migration_0_1_to_0_2(index_data)
+      if ((index_data[:version] <=> [ 0, 1 ]) > 0) then
+        return
+      end
+      if ((index_data[:version] <=> [ 0, 1 ]) < 0) then
+        raise "unexpected index format version: #{index_data[:version].join('.')}"
+      end
+
+      index_data[:storage_id] = @storage_id
+      index_data[:version] = [ 0, 2 ]
+
+      index_data
+    end
+    private :migration_0_1_to_0_2
+
     def save(path)
       tmp_path = "#{path}.tmp.#{$$}"
       File.open(tmp_path, File::WRONLY | File::CREAT | File::TRUNC, 0660) {|f|
@@ -147,6 +165,7 @@ module Higgs
         f.binmode
         index_data = Marshal.load(block_read(f, MAGIC_SYMBOL))
         migration_0_0_to_0_1(index_data)
+        migration_0_1_to_0_2(index_data)
         if (index_data[:version] != [ MAJOR_VERSION, MINOR_VERSION ]) then
           raise "unsupported version: #{index_data[:version].join('.')}"
         end
@@ -155,6 +174,7 @@ module Higgs
         @free_lists = index_data[:free_lists]
         @index = index_data[:index]
         @identities = index_data[:identities]
+        @storage_id = index_data[:storage_id]
       }
       self
     end

data/lib/higgs/jlog.rb

@@ -1,8 +1,8 @@
 # = journal log writer
 #
 # Author:: $Author: toki $
-# Date:: $Date: 2007-10-03 00:02:48 +0900 (Wed, 03 Oct 2007) $
-# Revision:: $Revision: 603 $
+# Date:: $Date: 2007-10-28 23:35:37 +0900 (Sun, 28 Oct 2007) $
+# Revision:: $Revision: 657 $
 #
 # == license
 #   :include:LICENSE
@@ -14,7 +14,7 @@ module Higgs
   # = journal log writer
   class JournalLogger
     # for ident(1)
-    CVS_ID = '$Id: jlog.rb 603 2007-10-02 15:02:48Z toki $'
+    CVS_ID = '$Id: jlog.rb 657 2007-10-28 14:35:37Z toki $'
 
     include Block
 
@@ -71,10 +71,6 @@ module Higgs
 
     class << self
       def has_eof_mark?(path)
-        unless (File.exist? path) then
-          return true
-        end
-
         File.open(path, 'r') {|f|
           f.binmode
           fsiz = f.stat.size
@@ -98,7 +94,7 @@ module Higgs
       end
 
       def need_for_recovery?(path)
-        ! (has_eof_mark? path)
+        (File.exist? path) && ! (has_eof_mark? path)
       end
 
       def open(path, *args)

data/lib/higgs/lock.rb

@@ -1,8 +1,8 @@
 # = multi-thread lock manager
 #
 # Author:: $Author: toki $
-# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
-# Revision:: $Revision: 559 $
+# Date:: $Date: 2007-10-23 00:23:50 +0900 (Tue, 23 Oct 2007) $
+# Revision:: $Revision: 652 $
 #
 # == license
 #   :include:LICENSE
@@ -18,7 +18,7 @@ module Higgs
   # = multi-thread lock manager
   class LockManager
     # for ident(1)
-    CVS_ID = '$Id: lock.rb 559 2007-09-25 15:20:20Z toki $'
+    CVS_ID = '$Id: lock.rb 652 2007-10-22 15:23:50Z toki $'
 
     include Exceptions
 
@@ -84,7 +84,7 @@ module Higgs
 
   class GiantLockManager < LockManager
     # for ident(1)
-    CVS_ID = '$Id: lock.rb 559 2007-09-25 15:20:20Z toki $'
+    CVS_ID = '$Id: lock.rb 652 2007-10-22 15:23:50Z toki $'
 
     def initialize(*args)
       super
@@ -115,14 +115,23 @@ module Higgs
       }
       r
     end
+
+    def exclusive
+      r = nil
+      @rw_lock.write_lock.synchronize{
+        r = yield
+      }
+      r
+    end
   end
 
   class FineGrainLockManager < LockManager
     # for ident(1)
-    CVS_ID = '$Id: lock.rb 559 2007-09-25 15:20:20Z toki $'
+    CVS_ID = '$Id: lock.rb 652 2007-10-22 15:23:50Z toki $'
 
     def initialize(*args)
       super
+      @tx_lock = ReadWriteLock.new
       @cache = SharedWorkCache.new{|key| ReadWriteLock.new }
     end
 
@@ -171,13 +180,26 @@ module Higgs
       else
         lock_handler = ReadWriteLockHandler.new(self, @cache)
       end
-      begin
-        r = yield(lock_handler)
-      ensure
-        for lock in lock_handler.lock_list
-          lock.unlock
+
+      r = nil
+      @tx_lock.read_lock.synchronize{
+        begin
+          r = yield(lock_handler)
+        ensure
+          for lock in lock_handler.lock_list
+            lock.unlock
+          end
         end
-      end
+      }
+
+      r
+    end
+
+    def exclusive
+      r = nil
+      @tx_lock.write_lock.synchronize{
+        r = yield
+      }
       r
     end
   end