higgs 0.1.0

data/lib/higgs/store.rb

@@ -0,0 +1,228 @@
+# = storage like pstore
+#
+# Author:: $Author: toki $
+# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
+# Revision:: $Revision: 559 $
+#
+# == license
+#   :include:LICENSE
+#
+
+require 'higgs/storage'
+require 'higgs/tman'
+
+module Higgs
+  # = storage like pstore
+  # == tutorial
+  #
+  # === 1. open a new store
+  #
+  #   % irb
+  #   irb(main):001:0> require 'higgs'
+  #   => true
+  #   irb(main):002:0> st = Higgs::Store.new('foo')
+  #   => #<Higgs::Store:0xb7cac7d8
+  #
+  # === 2. write an object
+  #
+  #   irb(main):003:0> st.transaction{|tx|
+  #   irb(main):004:1*   tx[:foo] = Object.new
+  #   irb(main):005:1> }
+  #   => #<Object:0xb7d18adc>
+  #
+  # === 3. read an object
+  #
+  #   irb(main):006:0> st.transaction{|tx|
+  #   irb(main):007:1*   tx[:foo]
+  #   irb(main):008:1> }
+  #   => #<Object:0xb7a3b1e8>
+  #
+  # === 4. custom property
+  #
+  # custom property name shuold be a string.
+  #
+  #   irb(main):009:0> st.transaction{|tx|
+  #   irb(main):010:1*   tx.set_property(:foo, 'list', %w[ apple banana orange ])
+  #   irb(main):011:1> }
+  #   => nil
+  #   irb(main):012:0> st.transaction{|tx|
+  #   irb(main):013:1*   tx.property(:foo, 'list')
+  #   irb(main):014:1> }
+  #   => ["apple", "banana", "orange"]
+  #
+  # === 4. write a string with <tt>:string_only</tt> property
+  # 
+  # <tt>:string_only</tt> is system property.
+  # system property name should be a symbol.
+  #
+  #   irb(main):015:0> st.transaction{|tx|
+  #   irb(main):016:1*   tx['bar'] = "Hello world.\n"
+  #   irb(main):017:1>   tx.set_property('bar', :string_only, true)
+  #   irb(main):018:1> }
+  #   => nil
+  #
+  # === 5. read a string
+  #
+  #   irb(main):019:0> st.transaction{|tx|
+  #   irb(main):020:1*   tx['bar']
+  #   irb(main):021:1> }
+  #   => "Hello world.\n"
+  #
+  # === 6. <tt>:string_only</tt> property rejects an object
+  #
+  #   irb(main):022:0> st.transaction{|tx|
+  #   irb(main):023:1*   tx['bar'] = Object.new
+  #   irb(main):024:1> }
+  #   TypeError: can't convert Object (value) to String
+  #
+  # === 7. read-only transaction
+  #
+  #   irb(main):025:0> st.transaction{|tx|
+  #   irb(main):026:1*   p tx[:foo]
+  #   irb(main):027:1>   p tx.property(:foo, 'list')
+  #   irb(main):028:1>   p tx['bar']
+  #   irb(main):029:1>   p tx.property('bar', :string_only)
+  #   irb(main):030:1> }
+  #   #<Object:0xb79dd764>
+  #   ["apple", "banana", "orange"]
+  #   "Hello world.\n"
+  #   true
+  #   => nil
+  #
+  # === 8. shutdown
+  #
+  #   irb(main):031:0> st.shutdown
+  #   => nil
+  #   irb(main):032:0> exit
+  #
+  # === 9. unix TAR storage and some files
+  #
+  #   % ls -ln
+  #   total 20
+  #   -rw-r----- 1 1000 1000 1024 Sep 24 20:59 foo.idx
+  #   -rw-r----- 1 1000 1000 4096 Sep 24 20:59 foo.jlog
+  #   -rw-r----- 1 1000 1000    0 Sep 24 20:57 foo.lock
+  #   -rw-r--r-- 1 1000 1000   57 Sep 24 20:57 foo.log
+  #   -rw-r----- 1 1000 1000 5120 Sep 24 20:58 foo.tar
+  #
+  # === 10. unix TAR contents
+  #
+  #   % tar tvf foo.tar
+  #   -rw-r--r-- 1000/1000        12 2007-09-24 20:58 foo
+  #   -rw-r--r-- 1000/1000       316 2007-09-24 20:58 foo.p
+  #   -rw-r--r-- 1000/1000        13 2007-09-24 20:58 bar
+  #   -rw-r--r-- 1000/1000       250 2007-09-24 20:58 bar.p
+  #   % tar xvf foo.tar
+  #   foo
+  #   foo.p
+  #   bar
+  #   bar.p
+  #
+  # === 11. marshaled object
+  #
+  # `foo' is marshaled object
+  #
+  #   % ls -ln foo
+  #   -rw-r--r-- 1 1000 1000 12 Sep 24 20:58 foo
+  #   % od -c foo
+  #   0000000 004  \b   o   :  \v   O   b   j   e   c   t  \0
+  #   0000014
+  #
+  # `foo.p' is properties.
+  # <tt>"list"</tt> of custom property is defined.
+  #
+  #   % ls -ln foo.p
+  #   -rw-r--r-- 1 1000 1000 316 Sep 24 20:58 foo.p
+  #   % cat foo.p
+  #   # SUM16 21993
+  #   ---
+  #   custom_properties:
+  #     list:
+  #     - apple
+  #     - banana
+  #     - orange
+  #   system_properties:
+  #     hash_type: MD5
+  #     modified_time: &id001 2007-09-24 20:58:03.568701 +09:00
+  #     hash_value: 5490d172635c560daaff1de92779d605
+  #     created_time: *id001
+  #     changed_time: 2007-09-24 20:58:24.502465 +09:00
+  #     string_only: false
+  #
+  # === 12. string with <tt>:string_only</tt> property
+  #
+  # `bar' is a string
+  #
+  #   % ls -ln bar
+  #   -rw-r--r-- 1 1000 1000 13 Sep 24 20:58 bar
+  #   % od -c bar
+  #   0000000   H   e   l   l   o       w   o   r   l   d   .  \n
+  #   0000015
+  #   % cat bar
+  #   Hello world.
+  #
+  # `bar.p' is properties.
+  # <tt>:string_only</tt> of system property is <tt>true</tt>.
+  #
+  #   % ls -ln bar.p
+  #   -rw-r--r-- 1 1000 1000 250 Sep 24 20:58 bar.p
+  #   % cat bar.p
+  #   # SUM16 18034
+  #   ---
+  #   custom_properties: {}
+  #   
+  #   system_properties:
+  #     hash_type: MD5
+  #     modified_time: &id001 2007-09-24 20:58:45.420319 +09:00
+  #     created_time: *id001
+  #     hash_value: fa093de5fc603823f08524f9801f0546
+  #     changed_time: *id001
+  #     string_only: true
+  #
+  # === 13. other operations
+  #
+  # <tt>tx</tt> of block argument is an instance of
+  # Higgs::TransactionContext. see Higgs::TransactionContext for all
+  # operations of transaction.
+  #
+  # == sample script
+  #
+  # sample/count.rb
+  #   :include: sample/count.rb
+  #
+  class Store
+    # for ident(1)
+    CVS_ID = '$Id: store.rb 559 2007-09-25 15:20:20Z toki $'
+
+    include Storage::Export
+    include TransactionManager::Export
+
+    DECODE = proc{|r| Marshal.load(r) }
+    ENCODE = proc{|w| Marshal.dump(w) }
+
+    # <tt>name</tt> is a storage name and see Higgs::Storage.new for
+    # detail. see Higgs::Storage::InitOptions and
+    # Higgs::TransactionManager::InitOptions for <tt>options</tt>.
+    def initialize(name, options={})
+      options[:decode] = DECODE
+      options[:encode] = ENCODE
+      @storage = Storage.new(name, options)
+      @tman = TransactionManager.new(@storage, options)
+    end
+
+    def self.open(*args)
+      store = new(*args)
+      begin
+        r = yield(store)
+      ensure
+        store.shutdown
+      end
+      r
+    end
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/lib/higgs/tar.rb

@@ -0,0 +1,390 @@
+# = unix TAR utilities
+#
+# Author:: $Author: toki $
+# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
+# Revision:: $Revision: 559 $
+#
+# == license
+#   :include:LICENSE
+#
+
+require 'forwardable'
+require 'higgs/exceptions'
+
+module Higgs
+  module Tar
+    # for ident(1)
+    CVS_ID = '$Id: tar.rb 559 2007-09-25 15:20:20Z toki $'
+
+    include Exceptions
+
+    class Error < HiggsError
+    end
+
+    class FormatError < Error
+    end
+
+    class MagicError < FormatError
+    end
+
+    class VersionError < FormatError
+    end
+
+    class CheckSumError < FormatError
+    end
+
+    class TooLongPathError < FormatError
+    end
+
+    class IOError < Error
+    end
+
+    # tar header format
+    # 
+    #  name     : Z100 : null terminated string, primary hard link name
+    #  mode     : A8   : octet number format ascii string
+    #  uid      : A8   : octet number format ascii string
+    #  gid      : A8   : octet number format ascii string
+    #  size     : A12  : octet number format ascii string
+    #  mtime    : A12  : octet number format ascii string, seconds since epoch date-time (UTC 1970-01-01 00:00:00)
+    #  cksum    : A8   : octet number format ascii string
+    #  typeflag : a1   : ascii number character (null character is old tar format)
+    #  linkname : Z100 : null terminated string, secondly hard link name
+    #  magic    : A6   : white space terminated string
+    #  version  : a2   : 2 ascii characters
+    #  uname    : Z32  : null terminated string
+    #  gname    : Z32  : null terminated string
+    #  devmajor : Z8   : octet number format ascii string
+    #  devminor : Z8   : octet number format ascii string
+    #  prefix   : Z155 : null terminated string
+    #
+    module Block
+      # block size
+      BLKSIZ = 512
+
+      # limit name length
+      MAX_LEN = 100
+
+      # unix tar format parameters
+      MAGIC    = 'ustar'
+      VERSION  = '00'
+      REGTYPE  = '0'
+      AREGTYPE = "\0"
+      LNKTYPE  = '1'
+      SYMTYPE  = '2'
+      CHRTYPE  = '3'
+      BLKTYPE  = '4'
+      DIRTYPE  = '5'
+      FIFOTYPE = '6'
+      CONTTYPE = '7'            # reserved
+
+      # for pack/unpack
+      HEAD_FMT = 'Z100 A8 A8 A8 A12 A12 A8 a1 Z100 A6 a2 Z32 Z32 Z8 Z8 Z155'
+
+      # end of archive
+      EOA = "\0" * BLKSIZ
+
+      def padding_size(bytes)
+        r = bytes % BLKSIZ
+        (r > 0) ? BLKSIZ - r : 0
+      end
+      module_function :padding_size
+
+      def blocked_size(bytes)
+        bytes + padding_size(bytes)
+      end
+      module_function :blocked_size
+
+      def tar?(path)
+        if (File.file? path) then
+          head = File.open(path, 'rb') {|r_io|
+            r_io.binmode
+            r_io.read(BLKSIZ)
+          }
+          if (head && head.length == BLKSIZ) then
+            if (head.unpack(HEAD_FMT)[9] == MAGIC) then
+              return true
+            end
+          end
+        end
+        false
+      end
+      module_function :tar?
+    end
+
+    class RawIO
+      extend Forwardable
+
+      def initialize(io)
+        @io = io
+      end
+
+      def read(*args)
+        begin
+          @io.sysread(*args)
+        rescue EOFError
+          nil
+        end
+      end
+
+      def write(*args)
+        @io.syswrite(*args)
+      end
+
+      def seek(*args)
+        @io.sysseek(*args)
+      end
+
+      def tell
+        @io.sysseek(0, IO::SEEK_CUR)
+      end
+
+      alias pos tell
+
+      def pos=(pos)
+        @io.sysseek(pos, IO::SEEK_SET)
+      end
+
+      def_delegator :@io, :flush
+      def_delegator :@io, :fsync
+      def_delegator :@io, :truncate
+      def_delegator :@io, :close
+      def_delegator :@io, :closed?
+    end
+
+    class IOHandler
+      extend Forwardable
+      include Block
+
+      def initialize(io)
+        @io = io
+      end
+
+      def to_io
+        @io
+      end
+
+      def_delegator :@io, :seek
+      def_delegator :@io, :tell
+      def_delegator :@io, :pos
+      def_delegator :@io, :pos=
+      def_delegator :@io, :flush
+      def_delegator :@io, :fsync
+      def_delegator :@io, :truncate
+      def_delegator :@io, :close
+      def_delegator :@io, :closed?
+    end
+
+    class ArchiveReader < IOHandler
+      include Enumerable
+
+      def read_header(skip_body=false)
+        head_data = @io.read(BLKSIZ)
+        unless (head_data) then
+          raise FormatError, 'unexpected EOF'
+        end
+        if (head_data.length != BLKSIZ) then
+          raise FormatError, 'too short header'
+        end
+        if (head_data == EOA) then
+          next_head_data = @io.read(BLKSIZ)
+          if (next_head_data && next_head_data == EOA) then
+            return nil
+          else
+            raise FormatError, "not of EOA: #{head_data.inspect}, #{next_head_data.inspect}"
+          end
+        end
+        chksum = (head_data[0...148] + ' ' * 8 + head_data[156...BLKSIZ]).sum(20)
+        head_list = head_data.unpack(HEAD_FMT)
+        head = {}
+        [ :name, :mode, :uid, :gid, :size, :mtime,
+          :chksum, :typeflag, :linkname, :magic, :version,
+          :uname, :gname, :devmajor, :devminor, :prefix
+        ].each_with_index do |k, i|
+          head[k] = head_list[i]
+        end
+        if (head[:typeflag] == AREGTYPE) then
+          head[:typeflag] = REGTYPE
+        end
+        if (head[:magic] != MAGIC) then
+          raise MagicError, "unknown format: #{head[:magic].inspect}"
+        end
+        # why?
+        #if (head[:version] != VERSION) then
+        #  raise VersionError, "unknown version: #{head[:version].inspect}"
+        #end
+        [ :mode,
+          :uid,
+          :gid,
+          :size,
+          :mtime,
+          :chksum
+        ].each do |sym|
+          head[sym] = head[sym].oct
+        end
+        if (head[:chksum] != chksum) then
+          raise CheckSumError, 'broken tar'
+        end
+        head[:mtime] = Time.at(head[:mtime])
+        if (skip_body) then
+          skip_size = head[:size] + padding_size(head[:size])
+          @io.read(skip_size)
+        end
+        head
+      end
+
+      def fetch
+        head_and_body = read_header or return
+        if (head_and_body[:size] > 0) then
+          blocked_size = blocked_size(head_and_body[:size])
+          head_and_body[:body] = @io.read(blocked_size)
+          unless (head_and_body) then
+            raise FormatError, 'unexpected EOF'
+          end
+          if (head_and_body[:body].size != blocked_size) then
+            raise FormatError, 'mismatch body size'
+          end
+          head_and_body[:body][head_and_body[:size]...blocked_size] = ''
+        else
+          case (head_and_body[:typeflag])
+          when REGTYPE, CONTTYPE
+            head_and_body[:body] = ''
+          else
+            head_and_body[:body] = nil
+          end
+        end
+        head_and_body
+      end
+
+      def each(skip_body=false)
+        if (skip_body) then
+          while (head = read_header(true))
+            yield(head)
+          end
+        else
+          while (head_and_body = fetch)
+            yield(head_and_body)
+          end
+        end
+        self
+      end
+    end
+
+    class ArchiveWriter < IOHandler
+      include Block
+
+      def write_header(head)
+        name = head[:name] or raise FormatError, "required name: #{head.inspect}"
+        if (name.length > MAX_LEN) then
+          raise TooLongPathError, "too long path: #{name}"
+        end
+        mode     = format('%-8o', head[:mode] || 0100644) # 0100644 => -rw-r--r--
+        uid      = format('%-8o', head[:uid] || Process.euid)
+        gid      = format('%-8o', head[:gid] || Process.egid)
+        size     = format('%-12o', head[:size])
+        mtime    = format('%-12o', (head[:mtime] || Time.now).to_i)
+        dummy_chksum = ' ' * 8
+        typeflag = head[:typeflag] || REGTYPE
+        linkname = head[:linkname] || ''
+        magic    = head[:magic] || MAGIC
+        version  = head[:version] || VERSION
+        uname    = head[:uname] || ''
+        gname    = head[:gname] || ''
+        devmajor = head[:devmajor] || ''
+        devminor = head[:devminor] || ''
+        prefix   = ''
+        header = [
+          name, mode, uid, gid, size, mtime,
+          dummy_chksum, typeflag, linkname, magic, version,
+          uname, gname, devmajor, devminor, prefix
+        ].pack(HEAD_FMT)
+        header += "\0" * 12
+        if (head.key? :chksum) then
+          chksum = head[:chksum]
+        else
+          chksum = header.sum(20)
+        end
+        header[148, 8] = format('%-8o', chksum)
+        @io.write(header)
+        nil
+      end
+
+      def write_EOA
+        @io.write(EOA * 2)
+        nil
+      end
+
+      FTYPE_TO_TAR = {
+        'file' => REGTYPE,
+        'directory' => DIRTYPE,
+        'characterSpecial' => CHRTYPE,
+        'blockSpecial' => BLKTYPE,
+        'fifo' => FIFOTYPE,
+        'link' => SYMTYPE,
+        'socket' => FIFOTYPE
+      }
+
+      def add(name, body, options=nil)
+        head = {
+          :name => name,
+          :size => body.length
+        }
+        if (options) then
+          head.update(options) 
+        end
+        if (block_given?) then
+          yield(head)
+        end
+        write_header(head)
+        body += "\0" * padding_size(body.length)
+        @io.write(body)
+        nil
+      end
+
+      def add_file(path)
+        stat = File.stat(path)
+        unless (FTYPE_TO_TAR.key? stat.ftype) then
+          raise FormatError, "unknown file type: #{stat.ftype}"
+        end
+        head = {
+          :name => path,
+          :mode => stat.mode,
+          :uid => stat.uid,
+          :gid => stat.gid,
+          :size => (stat.file?) ? stat.size : 0,
+          :mtime => stat.mtime,
+          :typeflag => FTYPE_TO_TAR[stat.ftype]
+        }
+        if (block_given?) then
+          yield(head)
+        end
+        write_header(head)
+        if (stat.ftype == 'file') then
+          File.open(path, 'rb') {|r_io|
+            chunk_size = BLKSIZ * 128
+            remaining_size = stat.size
+            while (remaining_size > chunk_size)
+              s = r_io.read(chunk_size) or raise IOError, 'unexpected EOF'
+              @io.write(s)
+              remaining_size -= chunk_size
+            end
+            s = r_io.read(chunk_size) or raise IOError, 'unexpected EOF'
+            s += "\0" * padding_size(stat.size)
+            @io.write(s)
+          }
+        end
+        nil
+      end
+
+      def close(eoa=true)
+        write_EOA if eoa
+        super()
+      end
+    end
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End: