sldb 0.1.0

data/lib/sldb.rb

@@ -0,0 +1,1302 @@
+module SLDB 
+#--{{{
+  require 'logger'
+  require 'socket'
+  require 'sync'
+
+  require 'sqlite'
+
+  require 'posixlock'
+  require 'arrayfields'
+  require 'lockfile'
+  require 'traits'
+
+  VERSION = '0.1.0'
+
+  module Util
+#--{{{
+    class << self
+      def export(*syms)
+#--{{{
+        syms.each do |sym|
+          sym = "#{ sym }".intern
+          module_function sym 
+          public sym
+        end
+#--}}}
+      end
+      def append_features c
+#--{{{
+        super
+        c.extend self 
+#--}}}  
+      end
+    end
+
+    def fork(*args, &block)
+#--{{{
+      begin
+        verbose = $VERBOSE
+        $VERBOSE = nil
+        ::Process::fork(*args, &block)
+      ensure
+        $VERBOSE = verbose
+      end
+#--}}}
+    end
+    export 'fork'
+
+    def getopt opt, hash, default = nil
+#--{{{
+      key = opt
+      return hash[key] if hash.has_key? key
+      key = "#{ key }"
+      return hash[key] if hash.has_key? key
+      key = key.intern
+      return hash[key] if hash.has_key? key
+      return default
+#--}}}
+    end
+    export 'getopt'
+
+    def optfilter(*list)
+#--{{{
+      args, opts = [ list ].flatten.partition{|item| not Hash === item}
+      [args, Util::hashify(*opts)]
+#--}}}
+    end
+    export 'optfilter'
+
+    def hashify(*hashes)
+#--{{{
+      hashes.inject(accum={}){|accum,hash| accum.update hash}
+#--}}}
+    end
+    export 'hashify'
+
+    def hostname
+#--{{{
+      @__hostname__ ||= Socket::gethostname
+#--}}}
+    end
+    export 'hostname'
+
+    def host
+#--{{{
+      @__host__ ||= Util::hostname.gsub(%r/\..*$/o,'')
+#--}}}
+    end
+    export 'host'
+
+    def uncache file 
+#--{{{
+      refresh = nil
+      begin
+        is_a_file = File === file
+        path = (is_a_file ? file.path : file.to_s) 
+        stat = (is_a_file ? file.stat : File::stat(file.to_s)) 
+        refresh = Util::tmpnam(File::dirname(path))
+        File::link path, refresh rescue File::symlink path, refresh
+        File::chmod stat.mode, path
+        File::utime stat.atime, stat.mtime, path
+        open(File::dirname(path)){|d| d.fsync rescue nil}
+      ensure 
+        begin
+          File::unlink refresh if refresh
+        rescue Errno::ENOENT
+        end
+      end
+#--}}}
+    end
+    export 'uncache'
+
+    def tmpnam opts = {} 
+#--{{{
+      dir = Util::getopt 'dir', opts, Dir::tmpdir
+      seed = Util::getopt 'seed', opts, Util::prognam
+      path = 
+        "%s_%s_%s_%s_%d" % [
+          Util::hostname, 
+          seed,
+          Process::pid, 
+          Util::timestamp('nospace' => true),
+          rand(101010)
+        ] 
+      dirname, basename = File::dirname(path), File::basename(path)
+      tn = File::join(dir, dirname, basename.gsub(%r/[^0-9a-zA-Z]/,'_')).gsub(%r/\s+/, '_')
+      File::expand_path tn
+#--}}}
+    end
+    export 'tmpnam'
+
+    def timestamp opts = {}
+#--{{{
+      time = Util::getopt 'time', opts, Time::now
+      local = Util::getopt 'local', opts, false 
+      nospace = Util::getopt 'nospace', opts, false
+      time = time.utc unless local
+      usec = "#{ time.usec }"
+      usec << ('0' * (6 - usec.size)) if usec.size < 6 
+      stamp = time.strftime('%Y-%m-%d %H:%M:%S.') << usec
+      stamp.gsub!(%r/\s+/,'_') if nospace
+      stamp
+#--}}}
+    end
+    export 'timestamp'
+
+    def stamptime string, opts = {} 
+#--{{{
+      local = Util::getopt 'local', opts, false 
+      string = "#{ string }"
+      pat = %r/^\s*(\d\d\d\d)-(\d\d)-(\d\d)[\s_]+(\d\d):(\d\d):(\d\d)(?:.(\d+))?\s*$/o
+      match = pat.match string
+      raise ArgumentError, "<#{ string.inspect }>" unless match
+      yyyy,mm,dd,h,m,s,u = match.to_a[1..-1].map{|m| (m || 0).to_i}
+      if local
+        Time::local yyyy,mm,dd,h,m,s,u
+      else
+        Time::gm yyyy,mm,dd,h,m,s,u
+      end
+#--}}}
+    end
+    export 'stamptime'
+
+    def system(*args, &block)
+#--{{{
+      begin
+        verbose = $VERBOSE
+        $VERBOSE = nil
+        ::Kernel::system(*args, &block)
+      ensure
+        $VERBOSE = verbose
+      end
+#--}}}
+    end
+    export 'system'
+
+    def env_intlist arg, opts = {}
+#--{{{
+      default = Util::getopt 'default', opts
+      return default unless arg
+      list =
+        case arg
+          when Array
+            arg
+          else
+            "#{ arg }".split %r/,+/
+        end
+      list.map{|item| env_int item}
+#--}}}
+    end
+    export 'env_intlist'
+
+    def env_floatlist arg, opts = {}
+#--{{{
+      default = Util::getopt 'default', opts
+      return default unless arg
+      list =
+        case arg
+          when Array
+            arg
+          else
+            "#{ arg }".split %r/,+/
+        end
+      list.map{|item| env_float item}
+#--}}}
+    end
+    export 'env_floatlist'
+
+    def env_boolean arg, opts = {}
+#--{{{
+      default = Util::getopt 'default', opts
+      return default unless arg
+      case arg
+        when String
+          arg =~ %r/true/io ? true : false
+        when Symbol
+          arg =~ :true ? true : false
+        when Numeric
+          arg == 1 ? true : false
+        else
+          arg ? true : false
+      end
+#--}}}
+    end
+    export 'env_boolean'
+
+    def env_int arg, opts = {} 
+#--{{{
+      default = Util::getopt 'default', opts
+      return default unless arg
+      Integer "#{ arg }"
+#--}}}
+    end
+    export 'env_int'
+
+    def env_float arg, opts = {} 
+#--{{{
+      default = Util::getopt 'default', opts
+      return default unless arg
+      Float "#{ arg }"
+#--}}}
+    end
+    export 'env_float'
+
+    def erreq a, b
+#--{{{
+      a.class == b.class and
+      a.message == b.message and
+      a.backtrace == b.backtrace
+#--}}}
+    end
+    export 'erreq'
+
+    def escape! s, char, esc
+#--{{{
+      #re = %r/([#{ 0x5c.chr << esc }]*)#{ char }/
+      re = %r/([#{ Regexp::quote esc }]*)#{ Regexp::quote char }/
+      s.gsub!(re) do
+        (($1.size % 2 == 0) ? ($1 << esc) : $1) + char 
+      end
+#--}}}
+    end
+    export 'escape!'
+
+    def escape s, char, esc
+#--{{{
+      s = "#{ s }"
+      escape! s, char, esc
+      s
+#--}}}
+    end
+    export 'escape'
+
+    def quote list, qm = "'" 
+#--{{{
+      [ list ].flatten.map do |f| 
+        if f
+          qm + Util::escape(f,qm,qm) + qm 
+        else
+          'NULL'
+        end
+      end
+#--}}}
+    end
+    export 'quote'
+    alias q quote
+    export 'q'
+#--}}}
+  end # module Util
+
+  class Logger < ::Logger
+#--{{{
+    def initialize(*a, &b)
+#--{{{
+      ret = super
+      self.level = SLDB::default_logger_level
+      ret
+#--}}}
+    end
+#--}}}
+  end # class Logger
+
+  class ProcessRefresher
+#--{{{
+    SIGNALS = %w( SIGTERM SIGINT SIGKILL )
+    attr :path
+    attr :pid
+    attr :refresh_rate
+    def initialize path, refresh_rate = 8
+#--{{{
+      @path = path
+      File::stat path
+      @refresh_rate = Float refresh_rate
+      @pipe = IO::pipe
+      if((@pid = Util::fork))
+        @pipe.last.close
+        @pipe = @pipe.first
+        @thread = Thread::new{loop{@pipe.gets}}
+        Process::detach @pid
+      else
+        begin
+          SIGNALS.each{|sig| trap(sig){ exit! }}
+          pid = Process::pid
+          ppid = Process::ppid
+          $0 = "#{ path }.refresher.#{ pid }"
+          @pipe.first.close
+          @pipe = @pipe.last
+          loop do
+            FileUtils::touch @path
+            sleep @refresh_rate
+            Process::kill 0, ppid
+            @pipe.puts pid
+          end
+        rescue Exception => e
+          exit!
+        end
+      end
+#--}}}
+    end
+    def kill
+#--{{{
+      #Thread::new(Thread::current, @thread, @pipe) do |c, t, p|
+        dead = false
+        begin
+          t.kill rescue nil
+          p.close rescue nil
+          SIGNALS.each do |sig| 
+            begin
+              Process::kill sig, @pid
+              sleep 0.01
+              Process::kill sig, @pid
+            rescue Errno::ESRCH => e
+              dead = true
+              break
+            rescue => e
+              break
+            end
+          end
+        ensure
+          unless dead
+            n = 42
+            begin
+              n.times do |i|
+                sleep 0.2
+                Process::kill 0, @pid
+                sleep 1
+              end
+            rescue Errno::ESRCH
+              dead = true
+            end
+            current.raise "runaway refresher <#{ @pid }> must be killed!" unless dead
+          end
+        end
+      #end
+#--}}}
+    end
+#--}}}
+  end # class ProcessRefresher
+
+  class ThreadedRefresher
+#--{{{
+    attr :path
+    attr :refresh_rate
+    def initialize path, refresh_rate = 8
+#--{{{
+      @path = path
+      @refresh_rate = Float refresh_rate
+      @thread = 
+        Thread::new(@path, @refresh_rate, Thread::current) do |path, refresh_rate, current|
+          begin
+            loop do
+              FileUtils::touch path
+              sleep refresh_rate
+            end
+          rescue Exception => e
+            # current.raise e
+            exit!
+          end
+        end
+#--}}}
+    end
+    def kill
+#--{{{
+      @thread.kill
+#--}}}
+    end
+#--}}}
+  end # class ThreadedRefresher
+
+  class SleepCycle
+#--{{{
+    attr :min
+    attr :max
+    attr :range
+    attr :inc
+    def initialize(*args)
+#--{{{
+      min, max, inc = args.flatten
+      raise ArgumentError, 'no min' unless min
+      raise ArgumentError, 'no max' unless max
+      raise ArgumentError, 'no inc' unless inc
+      @min, @max, @inc = Float(min), Float(max), Float(inc)
+      @range = @max - @min
+      raise RangeError, "max < min" if @max < @min
+      raise RangeError, "inc > range" if @inc > @range
+      @list = []
+      s = @min
+      @list.push(s) and s += @inc while(s <= @max)
+      @list[-1] = @max if @list[-1] < @max
+      reset
+#--}}}
+    end   
+    def next
+#--{{{
+      ret = @list[@idx]
+      @idx = ((@idx + 1) % @list.size)
+      ret
+#--}}}
+    end
+    def reset
+#--{{{
+      @idx = 0
+#--}}}
+    end
+#--}}}
+  end # class SleepCycle
+
+  #
+  # module defaults - configurable via ENV
+  #
+  class << self
+#--{{{
+    trait :default_logger_level =>
+      (ENV['SLDB_DEBUG'] ? ::Logger::DEBUG : ::Logger::INFO)
+
+    trait :default_logger => 
+      Logger::new(ENV['SLDB_LOG'] || STDERR)
+
+    trait :default_sql_debug =>
+      Util::env_boolean(ENV['SLDB_SQL_DEBUG'], 'default' => false)
+
+    trait :default_busy_sc =>
+      SleepCycle::new(Util::env_floatlist(ENV['SLDB_BUSY_SC'], 'default' => [3, 16, 2]))
+
+    trait :default_transaction_retries =>
+      Util::env_int(ENV['SLDB_TRANSACTION_RETRIES'], 'default' => 4)
+
+    trait :default_aquire_lock_sc =>
+      SleepCycle::new(Util::env_floatlist(ENV['SLDB_AQUIRE_LOCK_SC'], 'default' => [3, 16, 2]))
+
+    trait :default_transaction_retries_sc =>
+      SleepCycle::new(Util::env_floatlist(ENV['SLDB_TRANSACTION_RETRIES_SC'], 'default' => [8, 24, 8]))
+
+    trait :default_attempt_lockd_recovery =>
+      Util::env_boolean(ENV['SLDB_TTEMPT_LOCKD_RECOVERY'], 'default' => true)
+
+    trait :default_lockd_recover_wait =>
+      Util::env_int(ENV['SLDB_LOCKD_RECOVER_WAIT'], 'default' => 1800)
+
+    trait :default_aquire_lock_lockfile_stale_age =>
+      Util::env_int(ENV['SLDB_AQUIRE_LOCK_LOCKFILE_STALE_AGE'], 'default' => 1800)
+
+    trait :default_refresher =>
+      case ENV['SLDB_REFRESHER']
+        when %r/thread/i
+          ThreadedRefresher
+        when %r/process/i
+          ProcessRefresher
+        else
+          ThreadedRefresher
+        end
+
+    trait :default_aquire_lock_refresh_rate =>
+      Util::env_int(ENV['SLDB_AQUIRE_LOCK_REFRESH_RATE'], 'default' => 8)
+
+    trait :default_administrator =>
+      ENV['SLDB_ADMINISTRATOR'] || ENV['USER'] || ENV['LOGNAME'] || 'ara.t.howard@noaa.gov'
+#--}}}
+  end
+
+  class AbstractSLDB 
+#--{{{
+    include Util
+  #
+  # abstract class traits
+  #
+    class_trait :fields
+    class_trait :pragmas
+    class_trait :schema
+    class_trait :indexes
+    class_trait :path
+  #
+  # concrete class traits and defaults 
+  #
+    class_trait :logger                         => SLDB::default_logger
+    class_trait :sql_debug                      => SLDB::default_sql_debug
+    class_trait :busy_sc                        => SLDB::default_busy_sc
+    class_trait :transaction_retries            => SLDB::default_transaction_retries
+    class_trait :aquire_lock_sc                 => SLDB::default_aquire_lock_sc
+    class_trait :transaction_retries_sc         => SLDB::default_transaction_retries_sc
+    class_trait :attempt_lockd_recovery         => SLDB::default_attempt_lockd_recovery
+    class_trait :lockd_recover_wait             => SLDB::default_lockd_recover_wait
+    class_trait :aquire_lock_lockfile_stale_age => SLDB::default_aquire_lock_lockfile_stale_age
+    class_trait :refresher                      => SLDB::default_refresher
+    class_trait :aquire_lock_refresh_rate       => SLDB::default_aquire_lock_refresh_rate
+    class_trait :administrator                  => SLDB::default_administrator
+  #
+  # concrete instance traits
+  #
+    trait :path
+    trait :dbdir
+    trait :dbpath
+    trait :opts
+    trait :dirname
+    trait :schema
+    trait :fields
+    trait :mutex
+    trait :lockfile
+    trait :sql_debug
+    trait :busy_sc
+    trait :transaction_retries
+    trait :aquire_lock_sc
+    trait :transaction_retries_sc
+    trait :attempt_lockd_recovery
+    trait :lockd_recover_wait
+    trait :aquire_lock_lockfile_stale_age
+    trait :refresher
+    trait :aquire_lock_refresh_rate
+    trait :administrator
+
+    def initialize(*args)
+#--{{{
+      @args, @opts = Util::optfilter args
+      @path = @args.first || __getopt('path')
+      raise ArgumentError, 'no path' unless @path
+
+      @dirname             = @path
+      @dbpath              = File::join @dirname, 'db'
+      @schema              = File::join @dirname, '.schema' 
+      @waiting_w           = File::join @dirname, ".#{ Util::hostname }.#{ $$ }.waiting.w"
+      @waiting_r           = File::join @dirname, ".#{ Util::hostname }.#{ $$ }.waiting.r"
+      @lock_w              = File::join @dirname, ".#{ Util::hostname }.#{ $$ }.lock.w"
+      @lock_r              = File::join @dirname, ".#{ Util::hostname }.#{ $$ }.lock.r"
+      @lockfile            = File::join @dirname, '.lock'
+      @lockf               = Lockfile::new(File::join(@dirname, '.lockf'))
+      @in_transaction      = false
+      @in_ro_transaction   = false
+      @db                  = nil
+      @synchronizing       = false
+      @lockd_recover       = File::join(File::dirname(@dirname), ".#{ File::basename(@dirname) }.lockd_recover")
+      @lockd_recover_lockf = Lockfile::new "#{ @lockd_recover }.lock"
+      @lockd_recovered     = false
+
+      @sync = Sync::new
+
+      @logger                         = __getopt 'logger', SLDB::default_logger
+      @sql_debug                      = __getopt 'sql_debug'
+      @busy_sc                        = __getopt 'busy_sc'
+      @transaction_retries            = __getopt 'transaction_retries'
+      @aquire_lock_sc                 = __getopt 'aquire_lock_sc'
+      @transaction_retries_sc         = __getopt 'transaction_retries_sc'
+      @attempt_lockd_recovery         = __getopt 'attempt_lockd_recovery'
+      @lockd_recover_wait             = __getopt 'lockd_recover_wait'
+      @aquire_lock_lockfile_stale_age = __getopt 'aquire_lock_lockfile_stale_age'
+      @aquire_lock_refresh_rate       = __getopt 'aquire_lock_refresh_rate'
+      @refresher                      = __getopt 'refresher'
+      @administrator                  = __getopt 'administrator'
+
+      __bootstrap
+#--}}}
+    end
+
+    def ro_transaction(opts = {}, &block)
+#--{{{
+      ret = nil
+      opts['read_only'] = true
+      __synchronizing do
+        if @in_ro_transaction or @in_transaction
+          ret = yield 
+        else
+          __busy_catch do
+            __ro_transaction do
+              __lockd_recover_wrap(opts) do
+                __transaction_wrap(opts) do
+                  __aquire_lock(opts) do
+                    __connect(opts) do
+                      ret = yield 
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+      ret
+#--}}}
+    end
+    alias read_only_transaction ro_transaction
+
+    def transaction opts = {} 
+#--{{{
+      ret = nil
+      __synchronizing do
+        if @in_transaction
+          ret = yield 
+        else
+          __busy_catch do
+            __transaction do
+              __lockd_recover_wrap(opts) do
+                __transaction_wrap(opts) do
+                  __aquire_lock(opts) do
+                    __connect(opts) do
+                      execute 'begin'
+                      ret = yield 
+                      execute 'commit'
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+      ret
+#--}}}
+    end
+
+    def execute sql, &block
+#--{{{
+      raise 'not in transaction' unless @in_transaction or @in_ro_transaction
+      raise 'not connected' unless @connected
+      __logger << "sql_debug:\n#{ sql }\n" if @sql_debug
+      begin
+        @db.execute sql, &block
+      rescue SQLite::SQLException => e
+        error{ "bad sql!\n#{ sql }" }
+        raise
+      end
+#--}}}
+    end
+
+    def vacuum
+#--{{{
+      ret = false 
+      __synchronizing do
+        if @in_transaction
+          ret = yield 
+        else
+          __busy_catch do
+            __transaction do
+              __lockd_recover_wrap(opts) do
+                __transaction_wrap(opts) do
+                  __aquire_lock(opts) do
+                    __connect(opts) do
+                      ret = execute 'vacuum'
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+      ret
+#--}}}
+    end
+
+    def integrity_check
+#--{{{
+      debug{ "running integrity_check on <#{ @dbpath }>" }
+      ret = nil
+      __synchronizing do
+        if @in_transaction
+          ret = yield 
+        else
+          __busy_catch do
+            __transaction do
+              __lockd_recover_wrap(opts) do
+                __transaction_wrap(opts) do
+                  __aquire_lock(opts) do
+                    __connect(opts) do
+                      tuple = execute 'PRAGMA integrity_check;'
+                      ret = (tuple and tuple.first and (tuple.first["integrity_check"] =~ /^\s*ok\s*$/io))
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+      ret
+#--}}}
+    end
+
+    def lock opts = {} 
+#--{{{
+      __lockd_recover_wrap(opts){ __aquire_lock(opts){ yield }}
+#--}}}
+    end
+    alias write_lock lock
+    alias wlock write_lock
+
+    def read_lock(opts = {}, &block)
+#--{{{
+      opts['read_only'] = true
+      lock opts, &block
+#--}}}
+    end
+    alias rlock read_lock
+
+    def table_names
+#--{{{
+      ro_transaction do
+        tuples = execute "select name from sqlite_master where type = 'table' and name notnull"
+        tuples.map{|t| t['name']}
+      end
+#--}}}
+    end
+    alias tablenames table_names
+
+    def field_names table_name = nil
+#--{{{
+      fields = Hash::new{|h,k| h[k] = []}
+      ro_transaction do
+        tnames = (table_name ? [table_name] : table_names)
+        tnames.each do |tname|
+          tuples = execute "pragma table_info('#{ tname }')"
+          tuples.each{|t| fields[tname] << t['name']}
+        end
+      end
+      table_name ? fields[table_name] : fields
+#--}}}
+    end
+    alias fieldnames field_names
+    alias fields field_names
+    alias fields_for field_names
+
+    def t2h tuple
+#--{{{
+      h = {}
+      fields = tuple.fields
+      fields.each{|f| h[f] = tuple[f]}
+      h
+#--}}}
+    end
+
+    def h2t h, table_name = nil
+#--{{{
+      table_name = table_names.first
+      raise ArgumentError, "no table_name" unless table_name
+      fields = field_names table_name
+      t = tuple
+      fields.each{|f| t[f] = h[f]}
+      t
+#--}}}
+    end
+
+    def tuple table_name = nil
+#--{{{
+      table_name = table_names.first
+      raise ArgumentError, "no table_name" unless table_name
+      fields = field_names table_name
+      t = Array::new fields.size
+      t.fields = fields 
+      t
+#--}}}
+    end
+    alias tuple_for tuple
+
+
+# private
+
+    def __klass
+#--{{{
+      @klass ||= self.class
+#--}}}
+    end
+    private '__klass'
+
+    def __dbopen
+#--{{{
+      ret = nil
+      $db = @db =
+        begin
+          SQLite::Database::new @dbpath
+        rescue
+          SQLite::Database::new @dbpath, 0
+        end
+      @db.use_array = true
+      if block_given?
+        begin
+          ret = yield @db
+        ensure
+          @db.close rescue nil
+          $db = @db = nil
+        end
+      else
+        ret = @db
+      end
+      ret
+#--}}}
+    end
+    private '__dbopen'
+
+    def __bootstrap
+#--{{{
+      return if test ?e, @schema
+
+      __synchronizing do
+        FileUtils::mkdir_p @path rescue nil unless test ?d, path
+
+        raise "<#{ @path }> already exists as a file!" if test ?f, path
+
+        setup = File::join @dirname, '.setup'
+
+        open(setup, 'a+') do |f|
+          f.posixlock File::LOCK_EX
+          unless test ?e, @schema
+            tr = @transaction_retries
+            begin
+              sql = %w( pragmas schema indexes )
+              @transaction_retries = 0
+              transaction('__ignore_missing_schema' => true) do 
+                sql.each{|s| execute __klass.send(s) if __klass.send(s)}
+              end
+              FileUtils::touch @lockfile
+              tmp = "#{ @schema }.tmp"
+              open(tmp, 'w') do |f| 
+                sql.each{|s| f.puts __klass.send(s) if __klass.send(s)}
+              end
+              FileUtils::mv tmp, @schema 
+            ensure
+              @transaction_retries = tr
+            end
+            FileUtils::rm_f setup
+          end
+        end
+      end
+#--}}}
+    end
+    private '__bootstrap'
+
+    def __getopt opt, default = nil
+#--{{{
+      class_default = __klass.send opt #rescue nil
+      Util::getopt(opt, @opts, default || class_default) 
+#--}}}
+    end
+    private '__getopt'
+
+    def __synchronizing(*a, &b)
+#--{{{
+      begin
+        @sync.lock(Sync::EX)
+        yield
+      ensure
+        @sync.lock(Sync::UN)
+      end
+#--}}}
+    end
+    private '__synchronizing'
+
+    def __ro_transaction
+#--{{{
+      ret = nil
+      __synchronizing do
+        if @in_ro_transaction or @in_transaction
+          ret = yield 
+        else
+          state = @in_ro_transaction
+          begin 
+            @in_ro_transaction = true
+            ret = yield 
+          ensure
+            @in_ro_transaction = state 
+          end
+        end
+      end
+      ret
+#--}}}
+    end
+    private '__ro_transaction'
+
+    def __transaction
+#--{{{
+      __synchronizing do
+        raise 'cannot upgrade read-only transaction' if @in_ro_transaction
+        ret = nil
+        if @in_transaction
+          ret = yield 
+        else
+          state = @in_transaction
+          begin 
+            @in_transaction = true
+            ret = yield 
+          ensure
+            @in_transaction = state 
+          end
+        end
+        ret
+      end
+#--}}}
+    end
+    private '__transaction'
+
+    def __busy_catch
+#--{{{
+      @busy_sc.reset
+      begin
+        yield
+      rescue SQLite::BusyException => e
+        warn{ "<#{ @dbpath }> seems to be locked!" }
+        warn{ "this should never happen unless - another process is locking the db outside the sldb protocol" }
+        timeout = @busy_sc.next
+        warn{ "waiting <#{ timeout }>..." }
+        sleep timeout 
+        warn{ "retrying..." }
+        retry
+      end
+#--}}}
+    end
+    private '__busy_catch'
+
+    def __lockd_recover_wrap opts = {}
+#--{{{
+      ret = nil
+      try_again = false
+      begin
+        begin
+          @lockd_recovered = false
+          old_mtime = 
+            begin
+              Util::uncache @lockd_recover rescue nil
+              File::stat(@lockd_recover).mtime
+            rescue
+              Time::now 
+            end
+          ret = yield
+        ensure
+          new_mtime =
+            begin
+              Util::uncache @lockd_recover rescue nil
+              File::stat(@lockd_recover).mtime
+            rescue
+              old_mtime
+            end
+
+          if new_mtime and old_mtime and new_mtime > old_mtime and not @lockd_recovered
+            try_again = true
+          end
+        end
+      rescue
+        if try_again
+          warn{ "a remote lockd recovery has invalidated this transaction!" }
+          warn{ "retrying..."}
+          sleep 120
+          retry
+        else
+          raise
+        end
+      end
+      ret
+#--}}}
+    end
+    private '__lockd_recover_wrap'
+
+#
+# TODO - perhaps should not retry on SQLException??  yet errors seem to map to
+# this exception even when the sql is fine... safest (and most anoying) is to
+# simply retry on any error.  arggh.
+#
+    def __transaction_wrap opts = {} 
+#--{{{
+      ro = Util::getopt 'read_only', opts
+      ret = nil
+      if ro
+        ret = yield 
+      else
+        errors = []
+        @transaction_retries_sc.reset
+        begin
+          ret = yield 
+        rescue => e
+        #rescue SQLite::DatabaseException, SQLite::SQLException, SystemCallError => e
+          if @transaction_retries == 0
+            raise
+          elsif errors.size >= @transaction_retries
+            error{ "MAXIMUM TRANSACTION RETRIES SURPASSED" }
+            raise
+          else
+            warn{ e } if(errors.empty? or not Util::erreq(errors[-1], e))
+            errors << e
+            warn{ "retry <#{ errors.size }>..." }
+          end
+          sleep @transaction_retries_sc.next
+          retry
+        end
+      end
+      ret
+#--}}}
+    end
+    private '__transaction_wrap'
+
+    def __aquire_lock opts = {} 
+#--{{{
+      ro = Util::getopt 'read_only', opts
+      refresh =
+        if Class === refresher
+          Util::getopt 'refresh', opts, true
+        else
+          false
+        end
+      ret = nil
+
+      @aquire_lock_sc.reset
+  
+      waiting, ltype, lfile =
+        if ro
+          [@waiting_r, File::LOCK_SH | File::LOCK_NB, @lock_r]
+        else
+          [@waiting_w, File::LOCK_EX | File::LOCK_NB, @lock_w]
+        end
+  
+      ltype_s = (ltype == File::LOCK_EX ? 'write' : 'read')
+      ltype ||= File::LOCK_NB
+
+      aquired = false
+
+      until aquired
+        begin
+          debug{ "aquiring lock" }
+          #@lockf.lock unless ro
+    
+          open(@lockfile, 'a+') do |lf|
+            locked = false
+            lock_refresher = nil
+            sc = nil
+
+            begin
+              FileUtils::touch waiting
+              # poll
+              42.times do |i|
+                debug{ "applying posixlock to <#{ @lockfile }>" }
+                locked = lf.posixlock(ltype | File::LOCK_NB)
+                break if locked
+                sleep rand
+              end
+
+              if locked
+                aquired = true
+                if refresh
+                  lock_refresher = refresher::new @lockfile, @aquire_lock_refresh_rate
+                  debug{ "refresher pid <#{ refresher.pid }> refresh_rate <#{ @aquire_lock_refresh_rate }>" }
+                end
+                FileUtils::rm_f waiting rescue nil
+                FileUtils::touch lfile rescue nil
+                debug{ "aquired lock" }
+                ret = yield
+                debug{ "released lock" }
+              else
+                aquired = false 
+                stat = File::stat @lockfile
+                mtime = stat.mtime
+                stale = mtime < (Time::now - @aquire_lock_lockfile_stale_age)
+                if stale
+                  warn{ "detected stale lockfile of mtime <#{ mtime }>" }
+                  __lockd_recover if @attempt_ockd_recovery
+                end
+                sc = @aquire_lock_sc.next
+                debug{ "failed to aquire lock - sleep(#{ sc })" }
+                sleep sc 
+              end
+
+            ensure
+              if locked
+                unlocked = false
+                begin
+                  42.times do
+                    unlocked = lf.posixlock(File::LOCK_UN | File::LOCK_NB)
+                    break if unlocked
+                    sleep rand
+                  end
+                ensure
+                  lf.posixlock File::LOCK_UN unless unlocked
+                end
+              end
+              lock_refresher.kill if refresh and lock_refresher
+              FileUtils::rm_f waiting rescue nil
+              FileUtils::rm_f lfile rescue nil 
+            end
+          end
+        ensure
+          #@lockf.unlock rescue nil unless read_only
+        end
+      end
+      ret
+#--}}}
+    end
+    private '__aquire_lock'
+
+    def __connect opts = {} 
+#--{{{
+      yield @db if @connected
+      unless(Util::getopt('__ignore_missing_schema', opts))
+        raise "missing db schema file <#{ @schema }>" unless test ?e, @schema
+      end
+      __dbopen do
+        begin
+          @connected = true
+          yield @db
+        ensure
+          @connected = false
+        end
+      end
+#--}}}
+    end
+    private '__connect'
+
+    def __lockd_recover
+#--{{{
+      return nil unless @attempt_lockd_recovery
+      warn{ "attempting lockd recovery" }
+      time = Time::now
+      ret = nil
+
+      @lockd_recover_lockf.lock do
+        Util::uncache @dirname rescue nil
+        Util::uncache @dbpath rescue nil
+        Util::uncache @lockfile rescue nil
+        Util::uncache @lockd_recover rescue nil
+        mtime = File::stat(@lockd_recover).mtime rescue time
+
+        if mtime > time 
+          warn{ "skipping lockd recovery (another node has already recovered)" }
+          ret = true
+        else
+          moved = false
+          begin
+            FileUtils::touch @lockd_recover 
+            @lockd_recovered = false 
+
+            begin
+              report = <<-msg
+                the sldb library has detected a locking failure on your system.
+                this means that fcntl(2) based locks are failing.  common causes
+                include bad nfs setup.  you are advised to read the nfs faq and
+                consider things such as
+                  - having proper nfs daemons running
+                  - having firewalls configured properly for bidirectional
+                    communication between nfs client and servers
+                  - mount options that can prevent locking
+
+                if your system is not on nfs this is most likely the cause of a
+                kernel bug.  in either case this library can perform auto-matic
+                recover in 99% of cases and has done so now.  you should not see
+                this message again unless you system is very sick.
+
+                the following information may be useful in your debugging.
+
+                  hostname : #{ Util::hostname }
+                  pid      : #{ Process.pid }
+                  time     : #{ Time::now }
+                  db       : 
+                    dir  : #{ @dirname }
+                    path : #{ @dbpath }
+                    stat : #{ File::stat(@dbpath).inspect }
+                  lockfile : 
+                    path : #{ @lockfile }
+                    stat : #{ File::stat(@lockfile).inspect }
+              msg
+              info{ "SLDB LOCKD RECOVERY REPORT" }
+              __logger << report
+              cmd = "mail -s SLDB_LOCKD_RECOVERY #{ @administrator } <<eof\n#{ report }\neof"
+              Util::system cmd
+            rescue Exception
+              nil
+            end
+
+            warn{ "sleeping #{ @lockd_recover_wait }s before continuing..." }
+            sleep @lockd_recover_wait 
+
+            tmp = "#{ @dirname }.#{ $$ }.#{ Util::hostname }.tmp"
+
+            warn{ "rm_rf <#{ tmp }>" }
+            FileUtils::rm_rf tmp
+
+            warn{ "mv <#{ @dirname }> <#{ tmp }>" }
+            FileUtils::mv @dirname, tmp
+
+            moved = true
+
+            rfiles = [@dbpath, @lockfile].map{|f| File::join(tmp,File::basename(f))}
+            rfiles.each do |f|
+              ftmp = "#{ f }.tmp"
+              warn{ "rm_rf <#{ ftmp }>" }
+              FileUtils::rm_rf ftmp
+              warn{ "cp <#{ f } <#{ ftmp }>" }
+              FileUtils::cp f, ftmp 
+              warn{ "rm_f <#{ f }>" }
+              FileUtils::rm f 
+              warn{ "mv <#{ ftmp } <#{ f }>" }
+              FileUtils::mv ftmp, f 
+            end
+
+            dbtmp = File::join(tmp, File::basename(@dbpath))
+            Util::uncache dbtmp
+
+            if integrity_check(dbtmp)
+              FileUtils::mv tmp, @dirname
+              FileUtils::cp @lockd_recover_lockf.path, @lockd_recover 
+              @lockd_recovered = true 
+              Util::uncache @dirname rescue nil
+              Util::uncache @dbpath rescue nil
+              Util::uncache @lockfile rescue nil
+              Util::uncache @lockd_recover rescue nil
+              warn{ "lockd recovery complete" }
+            else
+              FileUtils::mv tmp, @dirname
+              @lockd_recovered = false 
+              error{ "lockd recovery failed" }
+            end
+
+            ret = @lockd_recovered 
+          ensure
+            if moved and not @lockd_recovered and tmp and test(?d, tmp)
+              FileUtils::mv tmp, @dirname rescue nil
+            end
+          end
+        end
+      end
+      ret
+#--}}}
+    end
+    private '__lockd_recover'
+    
+    def __logger
+#--{{{
+      __synchronizing{ @logger }
+#--}}}
+    end
+    private '__logger'
+
+    %w( debug info warn error fatal ).each do |meth|
+      module_eval <<-code
+        def #{ meth }(*a, &b)
+          __logger.#{ meth }(*a, &b)
+        end
+      code
+    end
+#--}}}
+  end # class AbstractSLDB 
+
+#
+# class factories
+#
+  class << self
+#--{{{
+    def new argv = [], &b
+#--{{{
+      args, opts = Util::optfilter argv
+      path = args.first
+      opts['path'] = path if path
+      klass = Class::new AbstractSLDB
+      klass.instance_eval &b if b
+      opts.each{|k,v| klass.send k, v}
+      # raise ArgumentError, "no schema given" unless klass.schema
+      # raise ArgumentError, "no fields given" unless klass.fields
+      #puts "klass <#{ klass.inspect }>"
+      #puts "klass::new <#{ klass::new.inspect }>"
+      (path ? klass::new : klass)
+#--}}}
+    end
+    alias klass new
+    alias db_klass new
+    alias class new
+    alias db_class new
+#--}}}
+  end
+#--}}}
+end # module SLDB