fsdb 0.6.1 → 0.7.0

data/bench/bench.rb

@@ -117,7 +117,7 @@ end
 # this is like in test-concurrency.rb -- generalize?
 def cleanup dir
   @db.browse_dir dir do |child_path|
-    if child_path[-1] == ?/ ## ugh!
+    if child_path =~ /\/$/ ## ugh!
       cleanup(child_path)
     else
       @db.delete(child_path)

data/examples/forks.rb

@@ -0,0 +1,43 @@
+require 'fsdb'
+
+db = FSDB::Database.new '/tmp/fsdb-example'
+
+db['foo.txt'] = "hello, world"
+puts db['foo.txt']
+puts
+
+p1 = fork do
+  puts "*** starting fork #1 at #{Time.now}"
+  db.edit 'foo.txt' do |str|
+    str << "\n edited by fork #1 at #{Time.now}"
+    sleep 2 # give fork #2 a chance to try editing
+    str << "\n done editing in fork #1 at #{Time.now}"
+  end
+end
+
+sleep 1
+
+p2 = fork do
+  puts "*** starting fork #2 at #{Time.now}"
+  db.edit 'foo.txt' do |str|
+    str << "\n edited by fork #2 at #{Time.now}"
+    str << "\n done editing in fork #2 at #{Time.now}"
+  end
+end
+
+Process.waitpid p1
+Process.waitpid p2
+
+puts db['foo.txt']
+
+__END__
+
+hello, world
+
+*** starting fork #1 at Tue Nov 29 11:03:03 -0800 2011
+*** starting fork #2 at Tue Nov 29 11:03:04 -0800 2011
+hello, world
+ edited by fork #1 at Tue Nov 29 11:03:03 -0800 2011
+ done editing in fork #1 at Tue Nov 29 11:03:05 -0800 2011
+ edited by fork #2 at Tue Nov 29 11:03:05 -0800 2011
+ done editing in fork #2 at Tue Nov 29 11:03:05 -0800 2011

data/lib/fsdb/database.rb

@@ -1,7 +1,5 @@
 require 'fileutils'
 require 'fsdb/platform'
-require 'fsdb/compat' if RUBY_VERSION.to_f < 1.7
-require 'fsdb/mutex'
 require 'fsdb/modex'
 require 'fsdb/file-lock'
 require 'fsdb/formats'
@@ -9,7 +7,7 @@ require 'fsdb/formats'
 module FSDB
 include Formats
 
-FSDB::VERSION = "0.6.1"
+FSDB::VERSION = "0.7.0"
 
 # A thread-safe, process-safe object database class which uses the
 # native file system as its back end and allows multiple file formats.
@@ -108,14 +106,10 @@ class Database
   
   # Subclasses can change the defaults.
   DEFAULT_META_PREFIX       = '..fsdb.meta.'
-###  if RUBY_PLATFORM =~ /darwin/
-###    DEFAULT_LOCK_TYPE       = :fcntl_lock
-###  else
-    DEFAULT_LOCK_TYPE       = :flock
-###  end
+  DEFAULT_LOCK_TYPE         = :flock
     
   # These must be methods of File.
-  LOCK_TYPES                = [:flock, :fcntl_lock]
+  LOCK_TYPES                = [:flock] # obsolete: :fcntl_lock
   
   @cache = {}                   # maps <file id> to <CacheEntry>
   @cache_mutex = Mutex.new      # protects access to @cache hash
@@ -130,15 +124,14 @@ class Database
   # The root directory of the db, to which paths are relative.
   attr_reader :dir
   
-  # The lock type of the db, by default <tt>:flock</tt>, optionally
-  # <tt>:fcntl_lock</tt>.
+  # The lock type of the db, by default <tt>:flock</tt>.
   attr_reader :lock_type
   
   # Create a new database object that accesses +dir+. Makes sure that the
   # directory exists on disk, but doesn't create or open any other files.
   # The +opts+ hash can include:
   #
-  # <tt>:lock_type</tt>::   <tt>:flock</tt> by default, or <tt>:fcntl_lock</tt>
+  # <tt>:lock_type</tt>::   <tt>:flock</tt> by default
   #
   # <tt>:meta_prefix</tt>:: <tt>'..fsdb.meta.'</tt> by default
   #
@@ -152,10 +145,6 @@ class Database
       raise "Unknown lock type: #{lock_type}"
     end
     
-    if @lock_type == :fcntl_lock
-      require 'fcntl_lock' ## hack.
-    end
-
     @meta_prefix = opts[:meta_prefix] || DEFAULT_META_PREFIX
 
     @formats = opts[:formats]

data/lib/fsdb/file-lock.rb

@@ -7,10 +7,6 @@ class File
   CAN_DELETE_OPEN_FILE  = !FSDB::PLATFORM_IS_WINDOWS
   CAN_OPEN_DIR          = !FSDB::PLATFORM_IS_WINDOWS
   
-  LOCK_BLOCK_FIXED_VER  = "1.8.2" # Hurray!
-  LOCK_DOESNT_BLOCK     = [RUBY_VERSION, LOCK_BLOCK_FIXED_VER].
-    map {|s| s.split('.')}.sort[0].join('.') ==  LOCK_BLOCK_FIXED_VER
-
   if FSDB::PLATFORM_IS_WINDOWS_ME
     # no flock() on WinME
 
@@ -21,61 +17,22 @@ class File
     end
   
   else
-    if LOCK_DOESNT_BLOCK
-      # Get an exclusive (i.e., write) lock on the file.
-      # If the lock is not available, wait for it without blocking other ruby
-      # threads.
-      def lock_exclusive lock_type
-        send(lock_type, LOCK_EX)
-      rescue Errno::EINTR
-        retry
-      end
-
-      # Get a shared (i.e., read) lock on the file.
-      # If the lock is not available, wait for it without blocking other ruby
-      # threads.
-      def lock_shared lock_type
-        send(lock_type, LOCK_SH)
-      rescue Errno::EINTR
-        retry
-      end
-
-    else
-      def lock_exclusive lock_type
-        if Thread.list.size == 1
-          begin
-            send(lock_type, LOCK_EX)
-          rescue Errno::EINTR
-            retry
-          end
-        else
-          # ugly hack because waiting for a lock in a Ruby thread blocks the
-          # entire process
-          period = 0.001
-          until send(lock_type, LOCK_EX|LOCK_NB)
-            sleep period
-            period *= 2 if period < 1
-          end
-        end
-      end
+    # Get an exclusive (i.e., write) lock on the file.
+    # If the lock is not available, wait for it without blocking other ruby
+    # threads.
+    def lock_exclusive lock_type
+      send(lock_type, LOCK_EX)
+    rescue Errno::EINTR
+      retry
+    end
 
-      def lock_shared lock_type
-        if Thread.list.size == 1
-          begin
-            send(lock_type, LOCK_SH)
-          rescue Errno::EINTR
-            retry
-          end
-        else
-          # ugly hack because waiting for a lock in a Ruby thread blocks the
-          # entire process
-          period = 0.001
-          until send(lock_type, LOCK_SH|LOCK_NB)
-            sleep period
-            period *= 2 if period < 1
-          end
-        end
-      end
+    # Get a shared (i.e., read) lock on the file.
+    # If the lock is not available, wait for it without blocking other ruby
+    # threads.
+    def lock_shared lock_type
+      send(lock_type, LOCK_SH)
+    rescue Errno::EINTR
+      retry
     end
   end
   

data/lib/fsdb/modex.rb

@@ -1,30 +1,6 @@
-#!/usr/bin/env ruby
-
-unless defined? Thread.exclusive
-  class Thread # :nodoc:
-    def self.exclusive
-      old = critical
-      self.critical = true
-      yield
-    ensure
-      self.critical = old
-    end
-  end
-end
-
-class Thread
-  def self.nonexclusive
-    old = critical
-    self.critical = false
-    yield
-  ensure
-    self.critical = old
-  end
-end
-
 module FSDB
 
-# Modex is a modal exclusion semaphore, like in syncronizer.rb.
+# Modex is a modal exclusion semaphore.
 # The two modes are shared (SH) and exclusive (EX).
 # Modex is not nestable.
 #
@@ -37,12 +13,13 @@ class Modex
     @locked   = []
     @mode     = nil
     @first    = true
+    @m        = Mutex.new
   end
 
   def try_lock mode
-    Thread.exclusive do
+    @m.synchronize do
       thread = Thread.current
-      raise ThreadError if @locked.include?(thread)
+      raise ThreadError, "nesting not allowed" if @locked.include?(thread)
 
       if @mode == mode and mode == SH and @waiting.empty? # strict queue
         @locked << thread
@@ -51,15 +28,17 @@ class Modex
         @mode = mode
         @locked << thread
         true
+      else
+        false
       end
     end
   end
   
   # the block is executed in the exclusive context
   def lock mode
-    Thread.exclusive do
+    @m.synchronize do
       thread = Thread.current
-      raise ThreadError if @locked.include?(thread)
+      raise ThreadError, "nesting not allowed" if @locked.include?(thread)
 
       if @mode == mode and mode == SH and @waiting.empty? # strict queue
         @locked << thread
@@ -68,8 +47,9 @@ class Modex
         @locked << thread
       else
         @waiting << thread << mode
+        @m.unlock
         Thread.stop
-        Thread.critical = true
+        @m.lock
       end
       
       yield if block_given?
@@ -88,11 +68,20 @@ class Modex
 
   # the block is executed in the exclusive context
   def unlock
-    raise ThreadError unless @mode
+    raise ThreadError, "already unlocked" unless @mode
     
-    Thread.exclusive do
-      yield if block_given?
-      @locked.delete Thread.current
+    @m.synchronize do
+      if block_given?
+        begin
+          yield
+        ensure
+          @locked.delete Thread.current
+        end
+      
+      else
+        @locked.delete Thread.current
+      end
+      
       wake_next_waiter if @locked.empty?
     end
 
@@ -109,7 +98,7 @@ class Modex
             @mode = EX
           end
 
-          Thread.nonexclusive { do_when_first[arg] }
+          nonexclusive { do_when_first[arg] }
 
           if mode == SH
             @mode = SH
@@ -126,7 +115,7 @@ class Modex
       if @locked.size == 1
         if do_when_last
           @mode = EX
-          Thread.nonexclusive { do_when_last[arg] }
+          nonexclusive { do_when_last[arg] }
         end
         @first = true
       end
@@ -134,7 +123,7 @@ class Modex
   end
   
   def remove_dead # :nodoc:
-    Thread.exclusive do
+    @m.synchronize do
       waiting = @waiting; @waiting = []
       until waiting.empty?
       
@@ -148,13 +137,21 @@ class Modex
   end
 
   private
+  def nonexclusive
+    raise ThreadError unless @m.locked?
+    @m.unlock
+    yield
+  ensure
+    @m.lock
+  end
+  
   def wake_next_waiter
-    first = @waiting.shift; @mode = @waiting.shift && EX
-    if first
-      first.wakeup
-      @locked << first
+    first_waiter = @waiting.shift; @mode = @waiting.shift && EX
+    if first_waiter
+      first_waiter.wakeup
+      @locked << first_waiter
     end
-    first
+    first_waiter
   rescue ThreadError
     retry
   end

data/test/test-concurrency.rb

@@ -295,7 +295,7 @@ class ConcurrencyTest
     
     @total_iters = process_count * thread_count * rep_count
     
-    unless quiet or not $defout.isatty
+    unless quiet or not $stdout.isatty
       monitor_thread = Thread.new do
         loop do
           print "\r#{status_meter}"
@@ -356,7 +356,7 @@ class ConcurrencyTest
   
   def cleanup dir
     @db.browse_dir dir do |child_path|
-      if child_path[-1] == ?/ ## ugh!
+      if  child_path =~ /\/$/ ## ugh!
         cleanup(child_path)
       else
         @db.delete(child_path)

data/test/test-concurrency/init.rb

@@ -68,10 +68,6 @@ class ConcurrencyTest
         require 'profile'
       end
 
-      opts.on("--fcntl-lock", "use fcntl version of flock") do
-        # actually, this arg is already picked out by test.rb
-      end
-
       opts.on("--nofork", "do not use fork, even if available") do
         params["nofork"] = true
       end

data/test/test-formats.rb

@@ -32,7 +32,7 @@ class Test_Formats < Test::Unit::TestCase
   # this is like in test-concurrency.rb -- generalize?
   def cleanup dir
     @db.browse_dir dir do |child_path|
-      if child_path[-1] == ?/ ## ugh!
+      if  child_path =~ /\/$/ ## ugh!
         cleanup(child_path)
       else
         @db.delete(child_path)

data/test/test-fsdb.rb

@@ -23,13 +23,15 @@ class Test_FSDB < Test::Unit::TestCase
   end
 
   def test_zzz_cleanup # Argh! Test::Unit is missing a few features....
+    @db['foo.txt'] = "abc" # something to clean up
+      ## really, cleanup should work when the dir is not there
     cleanup '/'
   end
 
   # this is like in test-concurrency.rb -- generalize?
   def cleanup dir
     @db.browse_dir dir do |child_path|
-      if child_path[-1] == ?/ ## ugh!
+      if child_path =~ /\/$/ ## ugh!
         cleanup(child_path)
       else
         @db.delete(child_path)

data/test/test-modex.rb

@@ -4,28 +4,70 @@ require 'fsdb/modex'
 
 include FSDB
 
-modex = Modex.new
-counter = 0
+DEBUG_MODEX = ARGV.delete '-d'
+VERBOSE_MODEX = ARGV.delete '-v'
+
 thread_count = (ARGV.shift || 10).to_i
 rep_count = (ARGV.shift || 1000).to_i
 
+def tabbed a
+  a.map {|s| s.sub(/^(\d+)/) {|n| " " * n.to_i * 20 + n}}
+end
+
+def thread_pass
+  if rand < 0.1
+    sleep rand/1000
+  else
+    Thread.pass
+  end
+end
+
+modex = Modex.new
+counter = 0
+
 out = []
 sharers = 0
 excluders = 0
 dumper = proc do |str|
   puts "*** #{str} called when there were #{sharers} threads sharing modex ***"
-  puts out[-20..-1].join("\n")
+  puts tabbed(out[-20..-1])
   exit!
 end
 
+class TestThread < Thread
+  def initialize n, outdev
+    self[:id] = n
+    self[:writes] = 0
+    @outdev = outdev
+    
+    super do
+      begin
+        yield
+      rescue => ex
+        puts "#{ex}\n  #{ex.backtrace.join("\n  ")}"
+        # let thread stop, but let process continue
+      end
+    end
+  end
+  
+  def inspect
+    "#<test thread #{self[:id]}>"
+  end
+  
+  if DEBUG_MODEX
+    def out s
+      @outdev << "#{self[:id]}: #{s}"
+    end
+  else
+    def out(*); end
+  end
+end
+
 threads = (0...thread_count).map do |n|
-  Thread.new do
+  TestThread.new(n, out) do
     thread = Thread.current
-    thread[:id] = n
-    thread[:writes] = 0
-
+    
     do_when_first = proc do
-      out << "#{thread[:id]}: do_when_first"
       sharers += 1
       if sharers > 1
         dumper["do_when_first"]
@@ -33,7 +75,6 @@ threads = (0...thread_count).map do |n|
     end
 
     do_when_last = proc do
-      out << "#{thread[:id]}: do_when_last"
       if sharers > 1
         dumper["do_when_last"]
       end
@@ -44,37 +85,40 @@ threads = (0...thread_count).map do |n|
       x = rand(100)
       case
       when x < 50
-        out << "#{thread[:id]}: trying SH"
+        thread.out "trying SH"
         modex.synchronize(Modex::SH, do_when_first, do_when_last) do
-          out << "#{thread[:id]}: locked SH"
+          thread.out "begin SH"
           c_old = counter
-          Thread.pass
+          thread_pass
           raise if excluders > 0
           raise unless counter == c_old
-          out << "#{thread[:id]}: unlocked SH"
+          thread.out "end SH"
         end
+        thread_pass
+
       else
-        out << "#{thread[:id]}: trying EX"
+        thread.out "trying EX"
         modex.synchronize(Modex::EX) do
-          out << "#{thread[:id]}: locked EX"
+          thread.out "begin EX"
           excluders += 1
           counter += 1
-          Thread.pass
+          thread_pass
           raise unless excluders == 1
+          raise unless sharers == 0
           excluders -= 1
-          out << "#{thread[:id]}: unlocked EX"
+          thread.out "end EX"
         end
+        thread_pass
         thread[:writes] += 1
       end
     end
   end
 end
 
-expected = 0
-threads.each {|t| t.join; expected += t[:writes]}
+expected = threads.inject(0) {|s, t| t.join; s + t[:writes]}
+puts tabbed(out) if VERBOSE_MODEX
 
 actual = counter
-
 if expected == actual
   puts "Test passed: #{actual} write operations"
 else