higgs 0.1.0

data/ChangeLog

@@ -0,0 +1,208 @@
+2007-09-29  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/version.rb: version 0.1.0.
+
+	* check ruby-1.8.6-p110
+	ruby 1.8.6 (2007-09-23 patchlevel 110) [i686-linux], OK.
+	ruby 1.8.6 (2007-09-23 patchlevel 110) [i386-cygwin], OK.
+
+2007-09-28  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* started subversion repository on rubyforge.org.
+
+	* add document utilities for package.
+
+2007-09-26  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* Source Code Management: cvs -> svn.
+
+2007-09-24  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/version.rb: version 0.0.6.
+
+	* LICENSE: BSD style license.
+
+	* lib/higgs/storage.rb (Higgs::Storage class): fix a bug of
+	auto-recovery at the read-write open.
+
+2007-09-18  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb (Higgs::Storage class): strict check for
+	recovery.
+
+2007-09-17  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* sample/count.rb: sample of Higgs::Store
+
+2007-09-09  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/tman.rb (Higgs::TransactionContext class): update
+	secondary cache on writing.
+
+	* lib/higgs/cache.rb (Higgs::SharedWorkCache class): update for
+	cache entry.
+
+	* lib/higgs/thread.rb (Higgs::SharedWork class): update for
+	result.
+
+2007-08-29  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/tman.rb: update method.
+
+2007-08-26  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/tman.rb: current transaction access in a thread.
+
+2007-08-07  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/tman.rb: secondary cache.
+
+2007-08-05  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/index.rb: identities for key of secondary cache.
+
+	* cvs tag of higgs_snapshot_20070805. snapshot before secondary
+	cache implementation.
+
+2007-07-28  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/version.rb: version 0.0.5.
+
+	* bin/higgs_backup: backup utility.
+
+2007-07-24  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/utils/bman.rb: online backup scenario.
+
+2007-07-23  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* test/test_utils_bman.rb: unit test for backup manager.
+
+	* lib/higgs/utils/bman.rb: backup manager.
+
+	* lib/higgs/utils.rb: namespace for higgs utilities.
+
+2007-07-16  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb: load storage options from YAML
+	configuration file.
+
+	* lib/higgs/tman.rb: new Higgs::TransactionManager::Export
+	module. export transaction manager methods from @tman instance
+	variable.
+
+2007-07-15  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb: new Higgs::Storage::Export module. export
+	storage methods from @storage instance variable.
+
+	* lib/cgi/session/higgs.rb, lib/higgs.rb, lib/higgs/block.rb,
+	lib/higgs/cache.rb, lib/higgs/dbm.rb, lib/higgs/exceptions.rb,
+	lib/higgs/flock.rb, lib/higgs/index.rb, lib/higgs/jlog.rb,
+	lib/higgs/lock.rb, lib/higgs/storage.rb, lib/higgs/store.rb,
+	lib/higgs/tar.rb, lib/higgs/thread.rb, lib/higgs/tman.rb: some
+	rdoc comments.
+
+	* lib/higgs/storage.rb: change confusing error name:
+	Higgs::Storage::BrokenError (alike Higgs::Block::BrokenError) ->
+	Higgs::Storage::PanicError.
+
+	* bin/higgs_verify: command line options.
+
+2007-07-12  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb: change journal log format.
+
+	* lib/higgs/version.rb: version 0.0.4.
+
+2007-07-11  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* bin/higgs_dump_index, bin/higgs_dump_jlog: command line options.
+
+2007-07-10  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/tman.rb: decode or freeze in each transaction for the rollback.
+
+2007-07-08  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* bin/higgs_dump_index, bin/higgs_dump_jlog, bin/higgs_verify:
+	tools.
+
+	* lib/higgs/version.rb: version 0.0.3.
+
+2007-07-07  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb: new `string_only' system property.
+
+2007-07-05  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/version.rb:  version 0.0.2.
+
+	* lib/higgs/tman.rb: clear local properties cache on new write.
+
+2007-07-04  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/version.rb: version description file.
+
+	* lib/higgs/tman.rb: clear local properties cache on new write.
+
+2007-06-23  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb: unique free block name.
+
+2007-06-07  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* Rakefile: gem package task.
+
+2007-05-21  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb: various data cksum type.
+
+	* lib/higgs/block.rb, lib/higgs/jlog.rb: various block body cksum
+	type.
+
+2007-05-19  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/cgi/session/higgs.rb: for CGI session.
+
+2007-05-16  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb (Higgs::Storage): cancel of initialization on fail.
+
+2007-05-14  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb (Higgs::Storage): fix a bug of
+	raw_write_and_commit.
+
+2007-05-07  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/storage.rb (Higgs::Storage): check index change number
+	in journal log.
+
+2007-05-06  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* test/optional/test_online_backup.rb: online backup test.
+
+	* indentation tab -> white spaces.
+
+	* lib/higgs/jlog.rb (Higgs::JournalLogger): optional eof flag to
+	close method.
+
+	* lib/higgs/tar.rb (Higgs::Tar::ArchiveWriter): reverse optional
+	flag of close method.
+
+	* lib/higgs/tman.rb (Higgs::TransactionContext): allow to store
+	nil value to local data cache.
+
+2007-05-04  TOKI Yoshinori  <toki@freedom.ne.jp>
+
+	* lib/higgs/thread.rb: obsolete name space: Higgs::Thread
+
+	* lib/higgs/lock.rb: obsolete name space: Higgs::Lock
+
+	* lib/higgs/cache.rb: obsolete name space: Higgs::Cache
+
+	* merge from development branch:
+	cvs update -j higgs_snapshot_20070420 -j higgs_snapshot_20070420_branch_20070420_snapshot_20070504
+
+

data/LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2007
+	TOKI Yoshinori <toki@freedom.ne.jp>. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+  1.  Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+
+  2.  Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+
+  3.  The name of the author may not be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+OF SUCH DAMAGE.

data/README

@@ -0,0 +1,2 @@
+please run `ruby mkrdoc.rb' or `rake rdoc', and see api/index.html by
+your browser.

data/Rakefile

@@ -0,0 +1,75 @@
+# for ident(1)
+CVS_ID = '$Id: Rakefile 576 2007-09-29 08:35:19Z toki $'
+
+require 'lib/higgs/version'
+require 'rake/gempackagetask'
+require 'yaml'
+
+LIB_DIR = 'lib'
+TEST_DIR = 'test'
+RDOC_DIR = 'api'
+RDOC_MAIN = 'Higgs'
+
+def cd_v(dir)
+  cd(dir, :verbose => true) {
+    yield
+  }
+end
+
+def load_rdoc_opts
+  YAML.load(IO.read('rdoc.yml'))
+end
+
+task :default
+
+task :test do
+  cd_v TEST_DIR do
+    sh 'rake'
+  end
+end
+
+task :rdoc do
+  rdoc_opts = load_rdoc_opts
+  sh 'rdoc', *(rdoc_opts['CommonOptions'] + rdoc_opts['CommandLineOptions']).flatten
+end
+
+task :rdoc_clean do
+  rm_rf RDOC_DIR
+end
+
+task :rdoc_upload => [ :rdoc_clean, :rdoc ] do
+  sh 'scp', '-pr', RDOC_DIR, 'rubyforge.org:/var/www/gforge-projects/higgs/.'
+end
+
+spec = Gem::Specification.new{|s|
+  s.name = 'higgs'
+  s.version = Higgs::VERSION
+  s.summary = 'pure ruby transactional storage compatible with unix TAR format'
+  s.author = 'TOKI Yoshinori'
+  s.email = 'toki@freedom.ne.jp'
+  s.executables  = %w[ higgs_backup higgs_dump_index higgs_dump_jlog higgs_verify ]
+  s.files = Dir['**/{Rakefile,.strc,*.{rb,yml}}']
+  s.files += %w[ ChangeLog LICENSE README ]
+  s.test_files = %w[ test/run.rb ]
+  s.has_rdoc = true
+  s.rdoc_options = load_rdoc_opts['CommonOptions'].flatten
+}
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.need_zip = true
+  pkg.need_tar_gz = true
+end
+
+task :gem_install => [ :gem ] do
+  sh 'gem', 'install', "pkg/higgs-#{Higgs::VERSION}.gem"
+end
+
+task :clean => [ :rdoc_clean, :clobber_package ] do
+  cd_v 'misc/dbm_bench' do
+    sh 'rake', 'clean'
+  end
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/bin/higgs_backup

@@ -0,0 +1,67 @@
+#!/usr/local/bin/ruby
+
+require 'higgs/utils/bman'
+require 'optparse'
+
+# for ident(1)
+CVS_ID = '$Id: higgs_backup 546 2007-09-23 14:20:23Z toki $'
+
+STDOUT.sync = true
+
+opts = OptionParser.new
+opts.banner = "Usage: #{opts.program_name} [options]"
+
+command = :online_backup
+options = { :verbose => 0 }
+
+opts.on('--command=BACKUP_COMMAND',
+	[ :index, :data, :rotate, :jlog, :recover, :verify, :clean, :online_backup ]) {|value|
+  command = value
+}
+
+opts.on('-f', '--from=BACKUP_TARGET_STORAGE', String) {|value|
+  options[:from] = value
+}
+opts.on('-t', '--to-dir=DIR_TO_BACKUP', String) {|value|
+  options[:to_dir] = value
+}
+opts.on('-n', '--to-name=NAME_TO_BACKUP', String) {|value|
+  options[:to_name] = value
+}
+opts.on('-u', '--jlog-rotate-service-uri=URI', String) {|value|
+  options[:jlog_rotate_service_uri] = value
+}
+opts.on('-v', '--verbose', '--[no-]verbose') {|value|
+  if (value) then
+    options[:verbose] += 1
+  else
+    options[:verbose] -= 1
+  end
+}
+opts.on('--verbose-level=LEVEL', Integer) {|value|
+  options[:verbose] = value
+}
+
+opts.parse!
+bman = Higgs::Utils::BackupManager.new(options)
+
+case (command)
+when :online_backup
+  bman.online_backup
+when :index
+  bman.backup_index
+when :data
+  bman.backup_data
+when :rotate
+  bman.rotate_jlog
+when :jlog
+  bman.backup_jlog
+when :recover
+  bman.recover
+when :verify
+  bman.verify
+when :clean
+  bman.clean_jlog
+else
+  raise "unknown backup command: #{command}"
+end

data/bin/higgs_dump_index

@@ -0,0 +1,43 @@
+#!/usr/local/bin/ruby
+
+require 'higgs/index'
+require 'optparse'
+require 'pp'
+require 'yaml'
+
+# for ident(1)
+CVS_ID = '$Id: higgs_dump_index 462 2007-07-15 09:25:07Z toki $'
+
+opts = OptionParser.new
+opts.banner = "Usage: #{opts.program_name} [options] INDEX_PATH"
+
+output_format = :pretty
+opts.on('-o', '--output-format=FORMAT', [ :pretty, :yaml ]) {|value|
+  output_format = value
+}
+
+opts.on_tail('-h', '--help') {
+  puts opts
+  puts 'Output format:'
+  print opts.summary_indent, :pretty, "\n"
+  print opts.summary_indent, :yaml, "\n"
+  exit(1)
+}
+opts.parse!
+
+index = Higgs::Index.new
+index.load(ARGV[0])
+
+case (output_format)
+when :pretty
+  pp index
+when :yaml
+  puts index.to_yaml
+else
+  raise "unknown output format: #{output_format}"
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/bin/higgs_dump_jlog

@@ -0,0 +1,42 @@
+#!/usr/local/bin/ruby
+
+require 'higgs/jlog'
+require 'optparse'
+require 'pp'
+require 'yaml'
+
+# for ident(1)
+CVS_ID = '$Id: higgs_dump_jlog 462 2007-07-15 09:25:07Z toki $'
+
+opts = OptionParser.new
+opts.banner = "Usage: #{opts.program_name} [options] [JOURNAL_LOG_PATHs]"
+
+output_format = :pretty
+opts.on('-o', '--output-format=FORMAT', [ :pretty, :yaml ]) {|value|
+  output_format = value
+}
+
+opts.on_tail('-h', '--help') {
+  puts opts
+  puts 'Output format:'
+  print opts.summary_indent, :pretty, "\n"
+  print opts.summary_indent, :yaml, "\n"
+  exit(1)
+}
+opts.parse!
+
+Higgs::JournalLogger.scan_log(ARGF) {|log|
+  case (output_format)
+  when :pretty
+    pp log
+  when :yaml
+    puts log.to_yaml
+  else
+    raise "unknown output format: #{output_format}"
+  end
+}
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/bin/higgs_verify

@@ -0,0 +1,37 @@
+#!/usr/local/bin/ruby
+
+require 'higgs/storage'
+require 'optparse'
+
+# for ident(1)
+CVS_ID = '$Id: higgs_verify 463 2007-07-15 09:52:12Z toki $'
+
+opts = OptionParser.new
+opts.banner = "Usage: #{opts.program_name} [options] HIGGS_NAME"
+
+verbose_level = 0
+opts.on('-v', '--[no-]verbose') {|v|
+  if (v) then
+    verbose_level += 1
+  else
+    verbose_level -= 1
+  end
+}
+opts.on('--verbose-level=LEVEL', Integer) {|level|
+  verbose_level = level
+}
+
+opts.parse!
+higgs_name = ARGV.shift or raise 'requried HIGGS_PATH'
+
+st = Higgs::Storage.new(higgs_name, :read_only => true)
+begin
+  st.verify(STDOUT, verbose_level)
+ensure
+  st.shutdown
+end
+
+# Local Variables:
+# mode: Ruby
+# indent-tabs-mode: nil
+# End:

data/lib/cgi/session/higgs.rb

@@ -0,0 +1,72 @@
+# like cgi/session/pstore.rb
+#
+# Author:: $Author: toki $
+# Date:: $Date: 2007-09-26 00:20:20 +0900 (Wed, 26 Sep 2007) $
+# Revision:: $Revision: 559 $
+#
+
+require 'cgi/session'
+require 'digest/md5'
+require 'fileutils'
+require 'higgs/store'
+
+class CGI
+  class Session
+    # like cgi/session/pstore.rb
+    class HiggsStore
+      # for ident(1)
+      CVS_ID = '$Id: higgs.rb 559 2007-09-25 15:20:20Z toki $'
+
+      def initialize(session, options={})
+        dir = options['tmpdir'] || Dir.tmpdir
+        prefix = options['prefix'] || ''
+        id = session.session_id
+        name = options['name'] || 'session'
+        md5 = Digest::MD5.hexdigest(id)
+        @store_dir = File.join(dir, prefix + md5)
+        FileUtils.mkdir_p(@store_dir)
+        @store_path = File.join(@store_dir, name)
+        @store_path.untaint
+        if (File.exist? "#{@store_path}.lock") then
+          @hash = nil
+        else
+          unless (session.new_session) then
+            raise CGI::Session::NoSession, 'uninitialized session'
+          end
+          @hash = {}
+        end
+        @store = Higgs::Store.new(@store_path, options)
+      end
+
+      def restore
+        unless (@hash) then
+          @store.transaction{|tx|
+            @hash = tx[:hash] || {}
+          }
+        end
+        @hash
+      end
+
+      def update
+        @store.transaction{|tx|
+          tx[:hash] = @hash
+        }
+      end
+
+      def close
+        r = nil
+        unless (@store.shutdown?) then
+          r = update
+          @store.shutdown
+        end
+        r
+      end
+
+      def delete
+        @store.shutdown unless @store.shutdown?
+        FileUtils.rm_rf(@store_dir)
+        nil
+      end
+    end
+  end
+end