fsdb 0.5 → 0.6.0

data/lib/fsdb/faster-mutex.rb

@@ -0,0 +1,138 @@
+#!/usr/bin/env ruby
+
+### only use this if you can ensure Thread.critical is not already set!
+### it mught be worth implementing a granular atomic-add-based mutex in
+### place of Thread.critical
+
+# Make sure we use the fast definition, not the thread.rb one!
+class Thread # :nodoc:
+  def self.exclusive
+    old = critical
+    self.critical = true
+    yield
+  ensure
+    self.critical = old
+  end
+end
+
+module FSDB
+
+# Mutex class based on standard thread.rb Mutex, which has some problems:
+#
+# - waiters are not a strict queue (try_lock can jump the queue, after
+#   which the queue gets *rotated*). Race condition.
+#
+# - doesn't use Thread.exclusive in enough places
+#
+# - no way to make dead threads give up the mutex, which is crucial in a fork
+#
+# Note: neither this Mutex nor the one in thread.rb is nestable.
+#
+class Mutex
+  def initialize
+    @waiting = []
+    @locked = nil
+  end
+
+  def locked?
+    @locked
+  end
+
+  def try_lock
+    Thread.critical = true
+      if not @locked
+        @locked = Thread.current
+        rslt = true
+      end
+    Thread.critical = false
+    rslt
+  end
+  
+  def lock
+    thread = Thread.current
+    Thread.critical = true
+      if @locked
+        @waiting.push thread
+        Thread.stop
+        unless @locked == thread
+          raise ThreadError, "queue was jumped"
+        end
+      else
+        @locked = thread
+      end
+    Thread.critical = false
+      self
+  end
+
+  def unlock
+    return unless @locked
+    
+    Thread.critical = true; t = wake_next_waiter; Thread.critical = false
+    
+    begin
+      t.run if t
+    rescue ThreadError
+    end
+    self
+  end
+
+  def synchronize
+    lock
+    yield
+  ensure
+    unlock
+  end
+
+  def remove_dead # :nodoc:
+    Thread.critical = true
+      @waiting = @waiting.select {|t| t.alive?}
+      wake_next_waiter if @locked and not @locked.alive?
+    Thread.critical = false
+  end
+
+  private
+  def wake_next_waiter
+    t = @waiting.shift
+    t.wakeup if t
+    @locked = t
+  rescue ThreadError
+    retry
+  end
+
+  module ForkSafely
+    def fork # :nodoc:
+      super do
+        ObjectSpace.each_object(Mutex) { |m| m.remove_dead }
+        yield
+      end
+    end
+  end
+end
+
+# FSDB users who fork should include ForkSafely or FSDB itself. If you use
+# mutexes (outside of those in FSDB), they should be FSDB::Mutexes.
+module ForkSafely
+  include Mutex::ForkSafely
+end
+include ForkSafely
+
+end # module FSDB
+
+
+if __FILE__ == $0
+  # Stress test is in fsdb/test/test-mutex.rb. This is just to show fork usage.
+  
+  include FSDB::ForkSafely
+  
+  m = FSDB::Mutex.new
+
+  Thread.new { m.synchronize { sleep 1 } }
+
+  fork do
+    m.synchronize { puts "Didn't get here if you used standard mutex or fork." }
+  end
+
+  m.synchronize { puts "Got here." }
+
+  Process.wait
+end

data/lib/fsdb/mutex.rb

@@ -29,6 +29,8 @@ class Mutex
   def initialize
     @waiting = []
     @locked = nil
+    @waiting.taint  # enable tainted comunication
+    self.taint
   end
 
   def locked?
@@ -37,7 +39,7 @@ class Mutex
 
   def try_lock
     Thread.exclusive do
-      if not @locked
+      if not @locked # and @waiting.empty? # not needed
         @locked = Thread.current
         true
       end
@@ -83,6 +85,7 @@ class Mutex
     Thread.exclusive do
       @waiting = @waiting.select {|t| t.alive?}
       wake_next_waiter if @locked and not @locked.alive?
+      ## what if @locked thread left the resource in an inconsistent state?
     end
   end
 

data/lib/fsdb/persistent.rb

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+
+require 'thread'
+require 'ftools'
+require 'fsdb/file-lock'
+
+# Mixin for an object that persists in a file by itself.
+#
+# References from the object to objects which need to persist separately
+# should be through nonpersistent_attr_accessors. Otherwise, objects
+# referred to in attrs will persist in the same file.
+
+module Persistent
+
+  def persistent_mutex
+    @persistent_mutex ||
+      Thread.exclusive do
+        @persistent_mutex ||= Mutex.new  # ||= to prevent race condition
+      end
+  end
+
+  # Save the persistent object (and all of its persistent references) to
+  # its file. If a block is given, call it (with self) in the context of
+  # locks that protect the file from other threads and processes. This can
+  # be used to atomically save related objects in separate files.
+  def save
+    persistent_mutex.synchronize do
+      File.makedirs(File.dirname(persistent_file))
+      File.open(persistent_file, "wb") do |f|
+        f.lock_exclusive do
+          dump(f)
+          yield self if block_given?
+        end
+      end
+    end
+  end
+  
+  def dump(*args)
+    Marshal.dump(self, *args)
+  end
+
+  def persistent_file
+    raise "#{self.class} must define the #persistent_file method to" +
+          " return the path of the file in which object persists."
+  end
+
+  class << self
+
+    # Need to take care that only one thread in the process is restoring
+    # a particular object, or there will be multiple copies.
+    def restore file
+      object = File.open(file, "rb") do |f|
+        f.lock_shared do
+          load(f)
+        end
+      end
+      object.restore file
+      object
+    end
+    
+    def load(*args)
+      Marshal.load(*args)
+    end
+    
+  end
+  
+  # Called when the object is loaded, allowing the object to restore some
+  # state from the path at which it was saved.
+  def restore file; end
+
+end
+
+if __FILE__ == $0
+
+  class Foo
+    include Persistent
+    attr_accessor :x
+    attr_accessor :dir
+    def persistent_file; "/tmp/foo"; end
+    def restore file; @dir = File.dirname(file); end
+  end
+  
+  foo = Foo.new
+  foo.x = 1
+  foo.save
+  foo.x = 2
+  foo = Persistent.restore("/tmp/foo")
+  p foo.x
+  p foo.dir
+
+end

data/lib/fsdb/read-write-object.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/ruby
+
+# based on drb.rb
+
+class IO
+  def write_object(obj)
+    str = Marshal.dump(obj)
+    packet = [str.size].pack('N') + str
+    write(packet)
+  end
+  
+  # returns nil at eof
+  def read_object
+    sz = read(4)	# sizeof (N)
+    return nil if sz.nil?
+    raise TypeError, "incomplete header, size == #{sz.size}" if sz.size < 4
+    sz = sz.unpack('N')[0]
+    str = read(sz)
+    raise TypeError, 'incomplete object' if str.nil? || str.size < sz
+    Marshal::load(str)
+  end
+end
+
+if __FILE__ == $0
+
+  f = File.open('/tmp/foo', 'w')
+
+  f.write_object([1,2,3])
+
+  f.close
+
+  f = File.open('/tmp/foo', 'r')
+
+  p f.read_object
+
+end

data/lib/fsdb/server.rb

@@ -0,0 +1,44 @@
+require 'drb'
+require 'fsdb'
+
+## can we use ssl?
+
+module FSDB
+  class Server
+
+  private
+    attr_reader :name, :dir, :transactions
+
+    def initialize name = nil, dir = nil, db_class = nil, opts = {}
+      @name = name || "fsdb-server"
+      @dir  = dir || name
+      @db   = (db_class || Database).new(@dir, opts)
+
+      @listening    = true
+      @transactions = 0
+    end
+    
+    # counts transactions running in the server process
+    def transaction
+      Thread.exclusive {@transactions += 1}
+      yield
+    ensure
+      Thread.exclusive {@transactions -= 1}
+    end
+    
+    def enter_admin_runlevel
+      @listening = false
+    end
+    
+    def leave_admin_runlevel
+      @listening = true
+    end
+  
+    def check_listening
+      unless @listening or user.name == 'admin'
+        raise "Server not listening."
+      end
+    end
+    
+  end
+end

data/misc/fsdb-blorubu.txt

@@ -0,0 +1,47 @@
+Like pstore, fsdb is:
+
+- pure ruby, cross-platform
+
+- ruby license
+
+- simple API
+
+- simple implementation (though less so)
+
+- transaction based, with abort capability
+
+However, pstore has some drawbacks:
+
+- not thread-safe (though it is process-safe)
+
+- coarse granularity (transaction applies to whole pstore), does not scale well
+
+- uses marshalling, even if your data is is some other format that doesn't require the overhead of marshalling (e.g., strings, binary data)
+
+Other advantages of fsdb:
+
+- can use YAML (or other formats) where you want to (e.g., in user settings, but perhaps not for large ruby objects or binary data). It's easy to specify a mapping from path patterns to data formats.
+
+- thread-safe, so you can easily wire it up to a multithreaded server, like drb (see examples/server.rb)
+
+- objects in the database are just files, and so can be manipulated with standard command line tools (including rsync, versioning tools, etc.).
+
+- granularity is up to you: you can have lots of little objects, if you want, (there are tradeoffs, of course)
+
+- scales as well as the fs does
+
+- thoroughly stress-tested, partially unit-tested. (But still not quite stable on Windows.)
+
+- Can select fcntl-based locking (if platform supports it), which should allow safe use with NFS mounts on linux
+
+Disadvantages of fsdb:
+
+- If you want indexes, you need to implement them on top of fsdb. You could do:
+
+  db['name-index'] = RBTree.new
+
+and define your own methods for adding/removing objects that also updates the name-index. The name-index maps names (or other keys) to paths in the db.
+
+- Likewise, no SQL, out of the box.
+
+- Efficiency and reliability depends on the fs. (My benchmarks get on the order of 1000 transactions per sec on a 3 yr old linux box.)

data/misc/mtime-and-file-id.txt

@@ -0,0 +1,23 @@
+mtime granularity on win?
+
+see:
+http://msdn.microsoft.com/library/default.asp?url=/library/en-us/fileio/base/by_handle_file_information_str.asp
+http://msdn.microsoft.com/library/default.asp?url=/library/en-us/sysinfo/base/file_times.asp
+
+http://www.google.com/search?hl=en&ie=ISO-8859-1&q=unique+file+id+on+windows&btnG=Google+Search
+
+http://discuss.fogcreek.com/joelonsoftware/default.asp?cmd=show&ixPost=42340
+
+Windows can give a unique file indentifier using the function GetFileInformationByHandle() (See http://msdn.microsoft.com/library/default.asp?url=/library/en-us/fileio/base/getfileinformationbyhandle.asp)
+ 
+ Example: (hFile is a handle to a file)
+ 
+ BY_HANDLE_FILE_INFORMATION FileInfo;
+ 
+ GetFileInformationByHandle(hFile, &FileInfo);
+ inode= FileInfo.nFileIndexLow | (FileInfo.nFileIndexHigh << 32);
+ volume=FileInfo.dwVolumeSerialNumber;
+ 
+ According to Micorsoft : "The identifier (low and high parts) and the volume serial number uniquely identify a file on a single computer. To determine whether two open handles represent the same file, combine this identifier and the volume serial number for each file and compare them." 
+ 
+ 

data/misc/posixlock.txt

@@ -0,0 +1,148 @@
+Subject:
+[ANN] posixlock
+From:
+"Ara.T.Howard" <ahoward@ngdc.noaa.gov>
+Date:
+Sat, 6 Dec 2003 05:22:07 +0900
+To:
+ruby-talk@ruby-lang.org (ruby-talk ML)
+Newsgroups:
+comp.lang.ruby
+
+
+SYNOPSIS:
+  posixlock
+    use fcntl (vs. flock) based locking for File#flock
+
+USAGE
+  require 'posixlock' # replaces File#flock!
+
+
+i currently do not have a place to post software - so here's a little
+extension that replaces flock with an fcntl based impl for posix compliant
+locking (safe on nfs) - posted to c.l.r for posterity:
+
+extconf.rb
+================
+require 'mkmf'
+create_makefile 'posixlock'
+================
+
+posixlock.c
+================
+#ifdef _WIN32
+#include "missing/file.h"
+#endif
+
+#include "ruby.h"
+#include "rubyio.h"
+#include "rubysig.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#include <errno.h>
+
+extern VALUE rb_cFile;
+
+# ifndef LOCK_SH
+#  define LOCK_SH 1
+# endif
+# ifndef LOCK_EX
+#  define LOCK_EX 2
+# endif
+# ifndef LOCK_NB
+#  define LOCK_NB 4
+# endif
+# ifndef LOCK_UN
+#  define LOCK_UN 8
+# endif
+
+static int
+posixlock (fd, operation)
+     int fd;
+     int operation;
+{
+  struct flock lock;
+
+  switch (operation & ~LOCK_NB)
+    {
+    case LOCK_SH:
+      lock.l_type = F_RDLCK;
+      break;
+    case LOCK_EX:
+      lock.l_type = F_WRLCK;
+      break;
+    case LOCK_UN:
+      lock.l_type = F_UNLCK;
+      break;
+    default:
+      errno = EINVAL;
+      return -1;
+    }
+  lock.l_whence = SEEK_SET;
+  lock.l_start = lock.l_len = 0L;
+  return fcntl (fd, (operation & LOCK_NB) ? F_SETLK : F_SETLKW, &lock);
+}
+
+
+static VALUE
+rb_file_posixlock (obj, operation)
+     VALUE obj;
+     VALUE operation;
+{
+#ifndef __CHECKER__
+  OpenFile *fptr;
+  int ret;
+
+  rb_secure (2);
+  GetOpenFile (obj, fptr);
+
+  if (fptr->mode & FMODE_WRITABLE)
+    {
+      fflush (GetWriteFile (fptr));
+    }
+retry:
+  TRAP_BEG;
+  ret = posixlock (fileno (fptr->f), NUM2INT (operation));
+  TRAP_END;
+  if (ret < 0)
+    {
+      switch (errno)
+	{
+	case EAGAIN:
+	case EACCES:
+#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
+	case EWOULDBLOCK:
+#endif
+	  return Qfalse;
+	case EINTR:
+#if defined(ERESTART)
+	case ERESTART:
+#endif
+	  goto retry;
+	}
+      rb_sys_fail (fptr->path);
+    }
+#endif
+
+  return INT2FIX (0);
+}
+
+
+void
+Init_posixlock()
+{
+    rb_define_method(rb_cFile, "flock", rb_file_posixlock, 1);
+}
+================
+
+
+
+-a
+-- ATTN: please update your address books with address below! =============================================================================== | EMAIL :: Ara [dot] T [dot] Howard [at] noaa [dot] gov | PHONE :: 303.497.6469 | ADDRESS :: E/GC2 325 Broadway, Boulder, CO 80305-3328 | STP :: http://www.ngdc.noaa.gov/stp/ | NGDC :: http://www.ngdc.noaa.gov/ | NESDIS :: http://www.nesdis.noaa.gov/ | NOAA :: http://www.noaa.gov/ | US DOC :: http://www.commerce.gov/ | | The difference between art and science is that science is what we | understand well enough to explain to a computer. | Art is everything else. | -- Donald Knuth, "Discover" | | /bin/sh -c 'for l in ruby perl;do $l -e "print \"\x3a\x2d\x29\x0a\"";done' ===============================================================================