higgs 0.1.0

data/lib/higgs/block.rb

@@ -0,0 +1,192 @@
+# = block read/write
+#
+# Author:: $Author: toki $
+# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
+# Revision:: $Revision: 559 $
+#
+# == license
+#   :include:LICENSE
+#
+
+require 'higgs/exceptions'
+
+module Higgs
+  # block header format
+  # 
+  #   0..15  : Z16  : magic symbol
+  #  16..19  : V    : body length
+  #  20..31  : x12  : (reserved)
+  #  32..33  : v    : format version
+  #  34..35  : x2   : (reserved)
+  #  36..37  : v    : head cksum
+  #  38..39  : x2   : (reserved)
+  #  40..55  : Z16  : body hash type
+  #  56..57  : v    : body hash length
+  #  58..71  : x14  : (reserved)
+  #  73..511 : a440 : body hash binary
+  #
+  module Block
+    # for ident(1)
+    CVS_ID = '$Id: block.rb 559 2007-09-25 15:20:20Z toki $'
+
+    include Exceptions
+
+    class BrokenError < HiggsError
+    end
+
+    BLOCK_SIZE = 512
+
+    FMT_VERSION = 0x00_00
+    HEAD_CKSUM_BITS = 16
+    HEAD_CKSUM_POS = 36..37
+    HEAD_CKSUM_FMT = 'v'
+
+    BODY_HASH = {}
+    [ [ :SUM16,  proc{|s| s.sum(16).to_s           },  nil            ],
+      [ :MD5,    proc{|s| Digest::MD5.digest(s)    }, 'digest/md5'    ],
+      [ :RMD160, proc{|s| Digest::RMD160.digest(s) }, 'digest/rmd160' ],
+      [ :SHA1,   proc{|s| Digest::SHA1.digest(s)   }, 'digest/sha1'   ],
+      [ :SHA256, proc{|s| Digest::SHA256.digest(s) }, 'digest/sha2'   ],
+      [ :SHA384, proc{|s| Digest::SHA384.digest(s) }, 'digest/sha2'   ],
+      [ :SHA512, proc{|s| Digest::SHA512.digest(s) }, 'digest/sha2'   ]
+    ].each do |hash_symbol, hash_proc, hash_lib|
+      if (hash_lib) then
+        begin
+          require(hash_lib)
+        rescue LoadError
+          next
+        end
+      end
+      BODY_HASH[hash_symbol] = hash_proc
+    end
+
+    BODY_HASH_BIN = {}
+    BODY_HASH.each do |hash_symbol, hash_proc|
+      BODY_HASH_BIN[hash_symbol.to_s] = hash_proc
+    end
+
+    HEAD_FMT = [
+      'Z16',                    # magic symbol
+      'V',                      # body length
+      'x12',                    # (reserved)
+      'v',                      # format version
+      'x2',                     # (reserved)
+      HEAD_CKSUM_FMT,           # head cksum
+      'x2',                     # (reserved)
+      'Z16',                    # body hash type
+      'v',                      # body hash length
+      'x14',                    # (reserved)
+      'a440'                    # body hash binary
+    ].join('')
+
+    def padding_size(bytes)
+      r = bytes % BLOCK_SIZE
+      (r > 0) ? BLOCK_SIZE - r : 0
+    end
+    module_function :padding_size
+
+    def head_read(io, magic_symbol)
+      head_block = io.read(BLOCK_SIZE) or return
+      if (head_block.size != BLOCK_SIZE) then
+        raise BrokenError, 'short read'
+      end
+
+      _magic_symbol, body_len, fmt_version, head_cksum,
+        body_hash_type, body_hash_len, body_hash_bucket = head_block.unpack(HEAD_FMT)
+
+      head_block[HEAD_CKSUM_POS] = "\000\000"
+      if (head_block.sum(HEAD_CKSUM_BITS) != head_cksum) then
+        raise BrokenError, 'broken head block'
+      end
+
+      if (_magic_symbol != magic_symbol) then
+        raise BrokenError, "unknown magic symbol: #{_magic_symbol}"
+      end
+
+      if (fmt_version != FMT_VERSION) then
+        raise BrokenError, format('unknown format version: 0x%04F', fmt_version)
+      end
+
+      body_hash_bin = body_hash_bucket[0, body_hash_len]
+
+      return body_len, body_hash_type, body_hash_bin
+    end
+    module_function :head_read
+
+    def head_write(io, magic_symbol, body_len, body_hash_type, body_hash_bin)
+      head_block = [
+        magic_symbol,
+        body_len,
+        FMT_VERSION,
+        0,
+        body_hash_type,
+        body_hash_bin.length,
+        body_hash_bin
+      ].pack(HEAD_FMT)
+
+      head_cksum = head_block.sum(HEAD_CKSUM_BITS)
+      head_block[HEAD_CKSUM_POS] = [ head_cksum ].pack(HEAD_CKSUM_FMT)
+
+      bytes = io.write(head_block)
+      if (bytes != head_block.size) then
+        raise BrokenError, 'short write'
+      end
+      bytes
+    end
+    module_function :head_write
+
+    def block_read(io, magic_symbol)
+      body_len, body_hash_type, body_hash_bin = head_read(io, magic_symbol)
+      unless (body_len) then
+        return
+      end
+
+      body = io.read(body_len) or raise BrokenError, 'unexpected EOF'
+      if (body.length != body_len) then
+        raise BrokenError, 'short read'
+      end
+
+      unless (hash_proc = BODY_HASH_BIN[body_hash_type]) then
+        raise BrokenError, "unknown body hash type: #{body_hash_type}"
+      end
+
+      if (hash_proc.call(body) != body_hash_bin) then
+        raise BrokenError, 'body hash error'
+      end
+
+      padding_size = padding_size(body_len)
+      padding = io.read(padding_size) or raise BrokenError, 'unexpected EOF'
+      if (padding.size != padding_size) then
+        raise BrokenError, 'short read'
+      end
+
+      body
+    end
+    module_function :block_read
+
+    def block_write(io, magic_symbol, body, body_hash_type=:MD5)
+      hash_proc = BODY_HASH[body_hash_type.to_sym] or "unknown body hash type: #{body_hash_type}"
+      body_hash = hash_proc.call(body)
+      head_write(io, magic_symbol, body.length, body_hash_type.to_s, body_hash)
+
+      bytes = io.write(body)
+      if (bytes != body.size) then
+        raise BrokenError, 'short write'
+      end
+
+      padding_size = padding_size(body.length)
+      bytes = io.write("\0" * padding_size)
+      if (bytes != padding_size) then
+        raise BrokenError, 'short write'
+      end
+
+      body.size + padding_size
+    end
+    module_function :block_write
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/lib/higgs/cache.rb

@@ -0,0 +1,117 @@
+# = cache utilities
+#
+# Author:: $Author: toki $
+# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
+# Revision:: $Revision: 559 $
+#
+# == license
+#   :include:LICENSE
+#
+
+require 'forwardable'
+require 'higgs/thread'
+require 'thread'
+
+module Higgs
+  # = cache by Least Recently Used strategy
+  class LRUCache
+    # for ident(1)
+    CVS_ID = '$Id: cache.rb 559 2007-09-25 15:20:20Z toki $'
+
+    extend Forwardable
+
+    def initialize(limit_size=1000)
+      @limit_size = limit_size
+      @count = 0
+      @cache = {}
+    end
+
+    attr_reader :limit_size
+    def_delegator :@cache, :keys
+    def_delegator :@cache, :key?
+    alias has_key? key?
+    alias include? key?
+
+    def [](key)
+      if (cached_pair = @cache[key]) then
+        cached_pair[1] = @count
+        @count = @count.succ
+        return cached_pair[0]
+      end
+      nil
+    end
+
+    def []=(key, value)
+      if (cached_pair = @cache[key]) then
+        cached_pair[1] = @count
+      else
+        @cache[key] = [ value, @count ]
+      end
+      @count = @count.succ
+      if (@cache.size > @limit_size) then
+        purge_old_cache
+      end
+      value
+    end
+
+    def purge_old_cache
+      c_list = @cache.map{|key, (value, c)| c }
+      c_list.sort!
+      threshold = c_list[c_list.size / 2]
+      @cache.delete_if{|key, (value, c)| c < threshold }
+    end
+    private :purge_old_cache
+
+    def delete(key)
+      if (cached_pair = @cache.delete(key)) then
+        return cached_pair[0]
+      end
+      nil
+    end
+  end
+
+  # = multi-thread safe cache
+  class SharedWorkCache
+    # for ident(1)
+    CVS_ID = '$Id: cache.rb 559 2007-09-25 15:20:20Z toki $'
+
+    def initialize(cache={}, &work)
+      unless (work) then
+        raise 'required work block'
+      end
+      @work = work
+      @lock = Mutex.new
+      @cache = cache
+    end
+
+    def fetch_work(key)
+      @lock.synchronize{
+        if (@cache.key? key) then
+          @cache[key]
+        else
+          @cache[key] = SharedWork.new{ @work.call(key) } 
+        end
+      }
+    end
+    private :fetch_work
+
+    def [](key)
+      fetch_work(key).result
+    end
+
+    def []=(key, value)
+      work = fetch_work(key)
+      work.result = value
+    end
+
+    def delete(key)
+      r = @lock.synchronize{ @cache.delete(key) }
+      r ? true : false
+    end
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/lib/higgs/dbm.rb

@@ -0,0 +1,55 @@
+# = storage like dbm
+#
+# Author:: $Author: toki $
+# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
+# Revision:: $Revision: 559 $
+#
+# == license
+#   :include:LICENSE
+#
+
+require 'higgs/storage'
+require 'higgs/tman'
+
+module Higgs
+  # = storage like dbm
+  # == sample script
+  # sample/dbmtest.rb
+  #   :include: sample/dbmtest.rb
+  #
+  class DBM
+    # for ident(1)
+    CVS_ID = '$Id: dbm.rb 559 2007-09-25 15:20:20Z toki $'
+
+    include Storage::Export
+    include TransactionManager::Export
+
+    DECODE = proc{|r| r }
+    ENCODE = proc{|w| w }
+
+    # <tt>name</tt> is a storage name and see Higgs::Storage.new for
+    # detail. see Higgs::Storage::InitOptions and
+    # Higgs::TransactionManager::InitOptions for <tt>options</tt>.
+    def initialize(name, options={})
+      @storage = Storage.new(name, options)
+      options[:decode] = DECODE
+      options[:encode] = ENCODE
+      @tman = TransactionManager.new(@storage, options)
+    end
+
+    def self.open(*args)
+      dbm = new(*args)
+      begin
+        r = yield(dbm)
+      ensure
+        dbm.shutdown
+      end
+      r
+    end
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/lib/higgs/exceptions.rb

@@ -0,0 +1,31 @@
+# = exceptions
+#
+# Author:: $Author: toki $
+# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
+# Revision:: $Revision: 559 $
+#
+# == license
+#   :include:LICENSE
+#
+
+module Higgs
+  module Exceptions
+    # for ident(1)
+    CVS_ID = '$Id: exceptions.rb 559 2007-09-25 15:20:20Z toki $'
+
+    class HiggsError < StandardError
+    end
+
+    class HiggsException < Exception
+    end
+
+    class ShutdownException < HiggsException
+    end
+  end
+  include Exceptions
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/lib/higgs/flock.rb

@@ -0,0 +1,77 @@
+# = file lock
+#
+# Author:: $Author: toki $
+# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
+# Revision:: $Revision: 559 $
+#
+# == license
+#   :include:LICENSE
+#
+
+module Higgs
+  class FileLock
+    # for ident(1)
+    CVS_ID = '$Id: flock.rb 559 2007-09-25 15:20:20Z toki $'
+
+    def initialize(path, read_only=false)
+      @path = path
+      @read_only = read_only
+      rdwr_mode = (@read_only) ? File::RDONLY : File::RDWR
+      begin
+        @f = File.open(path, rdwr_mode | File::CREAT | File::EXCL, 0660)
+      rescue Errno::EEXIST
+        @f = File.open(path, rdwr_mode, 0660)
+      end
+    end
+
+    attr_reader :read_only
+
+    def read_lock
+      @f.flock(File::LOCK_SH)
+      nil
+    end
+
+    def write_lock
+      if (@read_only) then
+        raise 'read only'
+      end
+      @f.flock(File::LOCK_EX)
+      nil
+    end
+
+    def unlock
+      @f.flock(File::LOCK_UN)
+      nil
+    end
+
+    def close
+      unlock
+      @f.close
+      nil
+    end
+
+    def synchronize(mode=:EX)
+      case (mode)
+      when :SH
+        read_lock
+      when :EX
+        write_lock
+      else
+        raise ArgumentError, "unknown lock mode: #{mode}"
+      end
+
+      begin
+        r = yield
+      ensure
+        unlock
+      end
+
+      r
+    end
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/lib/higgs/index.rb

@@ -0,0 +1,164 @@
+# = storage index
+#
+# Author:: $Author: toki $
+# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
+# Revision:: $Revision: 559 $
+#
+# == license
+#   :include:LICENSE
+#
+
+require 'forwardable'
+require 'higgs/block'
+
+module Higgs
+  # = storage index
+  class Index
+    # for ident(1)
+    CVS_ID = '$Id: index.rb 559 2007-09-25 15:20:20Z toki $'
+
+    extend Forwardable
+    include Block
+
+    MAGIC_SYMBOL = 'HIGGS_INDEX'
+    MAJOR_VERSION = 0
+    MINOR_VERSION = 1
+
+    def initialize
+      @change_number = 0
+      @eoa = 0
+      @free_lists = {}
+      @index = {}
+      @identities = {}
+    end
+
+    attr_reader :change_number
+    attr_accessor :eoa
+
+    def succ!
+      @change_number = @change_number.succ
+      self
+    end
+
+    def free_fetch(size)
+      @free_lists[size].shift if (@free_lists.key? size)
+    end
+
+    def free_fetch_at(pos, size)
+      @free_lists[size].delete(pos) or raise "not found free region: (#{pos},#{size})"
+    end
+
+    def free_store(pos, size)
+      @free_lists[size] = [] unless (@free_lists.key? size)
+      @free_lists[size] << pos
+      nil
+    end
+
+    def_delegator :@index, :key?
+    def_delegator :@index, :keys
+
+    def identity(key)
+      i = @index[key] and i[0]
+    end
+
+    def [](key)
+      i = @index[key] and i[1]
+    end
+
+    def self.create_id(key, identities)
+      id = key.to_s
+      if (identities.key? id) then
+        id += '.a'
+        id.succ! while (identities.key? id)
+      end
+      id
+    end
+
+    def []=(key, value)
+      delete(key)
+      id = Index.create_id(key, @identities)
+      @identities[id] = key
+      @index[key] = [ id, value ]
+      value
+    end
+
+    def delete(key)
+      id, value = @index.delete(key)
+      @identities.delete(id) if id
+      value
+    end
+
+    def each_key
+      @index.each_key do |key|
+        yield(key)
+      end
+      self
+    end
+
+    def to_h
+      { :version => [ MAJOR_VERSION, MINOR_VERSION ],
+        :change_number => @change_number,
+        :eoa => @eoa,
+        :free_lists => @free_lists,
+        :index => @index,
+        :identities => @identities
+      }
+    end
+
+    def migration_0_0_to_0_1(index_data)
+      if ((index_data[:version] <=> [ 0, 0 ]) > 0) then
+        return
+      end
+      if ((index_data[:version] <=> [ 0, 0 ]) < 0) then
+        raise "unexpected index format version: #{index_data[:version].join('.')}"
+      end
+
+      index = index_data[:index]
+      identities = index_data[:identities] = {}
+      for key in index.keys
+        id = Index.create_id(key, identities)
+        identities[id] = key
+        value = index[key]
+        index[key] = [ id, value ]
+      end
+      index_data[:version] = [ 0, 1 ]
+
+      index_data
+    end
+    private :migration_0_0_to_0_1
+
+    def save(path)
+      tmp_path = "#{path}.tmp.#{$$}"
+      File.open(tmp_path, File::WRONLY | File::CREAT | File::TRUNC, 0660) {|f|
+        f.binmode
+        index_data = self.to_h
+        block_write(f, MAGIC_SYMBOL, Marshal.dump(index_data))
+        f.fsync
+      }
+      File.rename(tmp_path, path)
+      self
+    end
+
+    def load(path)
+      File.open(path, 'r') {|f|
+        f.binmode
+        index_data = Marshal.load(block_read(f, MAGIC_SYMBOL))
+        migration_0_0_to_0_1(index_data)
+        if ((index_data[:version] <=> [ MAJOR_VERSION, MINOR_VERSION ]) > 0) then
+          raise "unsupported version: #{index_data[:version].join('.')}"
+        end
+        @change_number = index_data[:change_number]
+        @eoa = index_data[:eoa]
+        @free_lists = index_data[:free_lists]
+        @index = index_data[:index]
+        @identities = index_data[:identities]
+      }
+      self
+    end
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End: