fsdb 0.5 → 0.6.0

data/junk/restore-fsdb.rb

@@ -0,0 +1,37 @@
+class FSDB
+
+  class CacheEntry
+  
+    def object(mtime)
+      @mutex.synchronize do
+        unless @object and mtime == @mtime
+          @mtime = mtime
+          @object = yield
+#          @should_restore = true
+        end
+        @object
+      end
+    end
+    
+    # Returns true for the first thread that calls it (test and set).
+#    def should_restore?
+#      @mutex.synchronize do
+#        result = @should_restore
+#        @should_restore = false
+#        result
+#      end
+#    end
+
+
+  def cache_object(f, cache_entry)
+    object = cache_entry.object(f.mtime) do
+      load(f)
+    end
+#    if cache_entry.should_restore? and object.respond_to?(:restore)
+#      object.restore(self) ### this doesn't really do the trick
+#      # we do this after updating the entry to allow for loading circular
+#      # references (even if it is not possible to correctly lock them)
+#      # for the non-concurrent version
+#    end
+    object
+  end

data/junk/rf.txt

@@ -0,0 +1,5 @@
+
+scp -r ruby/prj/fsdb/doc/api/ vjoel@rubyforge.org:/var/www/gforge-projects/fsdb/
+
+
+project news

data/junk/solaris-bug-fixed.rb

@@ -0,0 +1,184 @@
+require 'thread'
+require 'sync'
+
+# Extensions to the File class for exception-safe file locking in a
+# environment with multiple user threads.
+
+# This is here because closing a file on solaris unlocks any locks that
+# other threads might have. So we have to make sure that only the last
+# reader thread closes the file.
+#
+# The hash maps inode number to a count of reader threads
+$reader_count = Hash.new(0)
+
+class File
+
+  # Get an exclusive (i.e., write) lock on the file, and yield to the block.
+  # If the lock is not available, wait for it without blocking other ruby
+  # threads.
+  def lock_exclusive
+    if Thread.list.size == 1
+      flock(LOCK_EX)
+    else
+      # ugly hack because waiting for a lock in a Ruby thread blocks the process
+      period = 0.001
+      until flock(LOCK_EX|LOCK_NB)
+        sleep period
+        period *= 2 if period < 1
+      end
+    end
+    
+    yield self
+    
+  ensure
+    flush
+    flock(LOCK_UN)
+  end
+  
+  # Get a shared (i.e., read) lock on the file, and yield to the block.
+  # If the lock is not available, wait for it without blocking other ruby
+  # threads.
+  def lock_shared
+    if Thread.exclusive {$reader_count[self.stat.ino] == 1}
+      if Thread.list.size == 1
+        flock(LOCK_SH)
+      else
+        # ugly hack because waiting for a lock in a Ruby thread blocks the process
+        period = 0.001
+        until flock(LOCK_SH|LOCK_NB)
+          sleep period
+          period *= 2 if period < 1
+        end
+      end
+
+      yield self
+    end
+
+  ensure
+    Thread.exclusive {flock(LOCK_UN) if $reader_count[self.stat.ino] == 1}
+    ## for solaris, no need to unlock here--closing does it
+    ## but this has no effect on the bug
+  end
+
+end
+
+# Provides instance methods to open files in mode "r" or "r+"
+
+module OpenLock
+
+  # Opens path for reading ("r") with a shared lock for the
+  # duration of the block.
+  def open_read_lock(path)
+    f = nil
+    f = File.open(path, "r")
+    Thread.exclusive {$reader_count[f] += 1}
+    f.lock_shared do
+      yield f
+    end
+  ensure
+    if f
+      Thread.exclusive do
+        cache = @openlock_file_cache ||= Hash.new {|h,v| h[v] = []}
+          # maps <inode number> => [<open File instance>, ...]
+        ino = f.stat.ino
+        if $reader_count[f] == 1
+          f.close
+          files = cache.delete(ino) and files.each {|g| g.close}
+          $reader_count.delete(f)
+        else
+          cache[ino] << f
+          $reader_count[f] -= 1
+        end
+      end
+    end
+  end
+
+  # Opens path for writing and reading ("r+") with an exclusive lock for
+  # the duration of the block.
+  def open_write_lock(path)
+    File.open(path, "r+") do |f|
+      f.lock_exclusive {yield f}
+    end
+  end
+
+end
+
+include OpenLock
+
+process_count = 2
+thread_count  = 2
+rep_count     = (ARGV.shift || 10000).to_i
+sync = Sync.new
+
+test_file = '/tmp/test-file-lock.dat'
+
+File.open(test_file, "w") {|f| Marshal.dump(0, f)}
+
+(0...process_count).each do
+  fork do
+    
+    increments = 0
+    
+    threads =
+      (0...thread_count).map do
+        Thread.new do
+          (0...rep_count).each do
+            if rand(100) < 50
+              sync.synchronize(Sync::SH) do
+                open_read_lock(test_file) do |f|
+                  str = f.read
+                  data = Marshal.load(str)
+                end
+              end
+            else
+              sync.synchronize(Sync::EX) do
+                open_write_lock(test_file) do |f|
+                  str = f.read
+                  data = Marshal.load(str)
+                  data += 1
+                  f.rewind; f.truncate(0)
+                  Marshal.dump(data, f)
+                  f.flush
+                  Thread.exclusive {increments += 1}
+                end
+              end
+            end
+          end
+        end
+      end
+    
+    threads.each {|thread| thread.join}
+    
+    File.open("#{test_file}#{Process.pid}", "w") do |f|
+      Marshal.dump(increments, f)
+    end
+  
+  end
+end
+
+Thread.new do
+  count = 0
+  loop do
+    puts count
+    sleep 1
+    count += 1
+  end  
+end
+
+increments = 0
+(0...process_count).each do
+  pid = Process.wait
+  File.open("#{test_file}#{pid}", "r") do |f|
+    increments += Marshal.load(f)
+  end
+end
+
+data = File.open(test_file, "r") {|f| Marshal.load(f)}
+
+if data == increments
+  puts "Equal counts: #{data}"
+else
+  puts "Not equal:"
+  puts "  increments: #{increments}"
+  puts "  data      : #{data}"
+end

data/junk/solaris-bug.rb

@@ -0,0 +1,182 @@
+require 'thread'
+require 'sync'
+
+# Extensions to the File class for exception-safe file locking in a
+# environment with multiple user threads.
+
+# This is here because closing a file on solaris unlocks any locks that
+# other threads might have. So we have to make sure that only the last
+# reader thread closes the file.
+#
+# The hash maps inode number to a count of reader threads
+$reader_count = Hash.new(0)
+
+class File
+
+  # Get an exclusive (i.e., write) lock on the file, and yield to the block.
+  # If the lock is not available, wait for it without blocking other ruby
+  # threads.
+  def lock_exclusive
+    if Thread.list.size == 1
+      flock(LOCK_EX)
+    else
+      # ugly hack because waiting for a lock in a Ruby thread blocks the process
+      period = 0.001
+      until flock(LOCK_EX|LOCK_NB)
+        sleep period
+        period *= 2 if period < 1
+      end
+    end
+    
+    yield self
+    
+  ensure
+    flush
+    flock(LOCK_UN)
+  end
+  
+  # Get a shared (i.e., read) lock on the file, and yield to the block.
+  # If the lock is not available, wait for it without blocking other ruby
+  # threads.
+  def lock_shared
+    if Thread.list.size == 1
+      flock(LOCK_SH)
+    else
+      # ugly hack because waiting for a lock in a Ruby thread blocks the process
+      period = 0.001
+      until flock(LOCK_SH|LOCK_NB)
+        sleep period
+        period *= 2 if period < 1
+      end
+    end
+    
+    yield self
+    
+  ensure
+    Thread.exclusive {flock(LOCK_UN) if $reader_count[self.stat.ino] == 1}
+    ## for solaris, no need to unlock here--closing does it
+    ## but this has no effect on the bug
+  end
+
+end
+
+# Provides instance methods to open files in mode "r" or "r+"
+
+module OpenLock
+
+  # Opens path for reading ("r") with a shared lock for the
+  # duration of the block.
+  def open_read_lock(path)
+    f = nil
+    f = File.open(path, "r")
+    Thread.exclusive {$reader_count[f] += 1}
+    f.lock_shared do
+      yield f
+    end
+  ensure
+    if f
+      Thread.exclusive do
+        cache = @openlock_file_cache ||= Hash.new {|h,v| h[v] = []}
+          # maps <inode number> => [<open File instance>, ...]
+        ino = f.stat.ino
+        if $reader_count[f] == 1
+          f.close
+          files = cache.delete(ino) and files.each {|g| g.close}
+          $reader_count.delete(f)
+        else
+          cache[ino] << f
+          $reader_count[f] -= 1
+        end
+      end
+    end
+  end
+
+  # Opens path for writing and reading ("r+") with an exclusive lock for
+  # the duration of the block.
+  def open_write_lock(path)
+    File.open(path, "r+") do |f|
+      f.lock_exclusive {yield f}
+    end
+  end
+
+end
+
+include OpenLock
+
+process_count = 2
+thread_count  = 2
+rep_count     = (ARGV.shift || 10000).to_i
+sync = Sync.new
+
+test_file = '/tmp/test-file-lock.dat'
+
+File.open(test_file, "w") {|f| Marshal.dump(0, f)}
+
+(0...process_count).each do
+  fork do
+    
+    increments = 0
+    
+    threads =
+      (0...thread_count).map do
+        Thread.new do
+          (0...rep_count).each do
+            if rand(100) < 50
+              sync.synchronize(Sync::SH) do
+                open_read_lock(test_file) do |f|
+                  str = f.read
+                  data = Marshal.load(str)
+                end
+              end
+            else
+              sync.synchronize(Sync::EX) do
+                open_write_lock(test_file) do |f|
+                  str = f.read
+                  data = Marshal.load(str)
+                  data += 1
+                  f.rewind; f.truncate(0)
+                  Marshal.dump(data, f)
+                  f.flush
+                  Thread.exclusive {increments += 1}
+                end
+              end
+            end
+          end
+        end
+      end
+    
+    threads.each {|thread| thread.join}
+    
+    File.open("#{test_file}#{Process.pid}", "w") do |f|
+      Marshal.dump(increments, f)
+    end
+  
+  end
+end
+
+Thread.new do
+  count = 0
+  loop do
+    puts count
+    sleep 1
+    count += 1
+  end  
+end
+
+increments = 0
+(0...process_count).each do
+  pid = Process.wait
+  File.open("#{test_file}#{pid}", "r") do |f|
+    increments += Marshal.load(f)
+  end
+end
+
+data = File.open(test_file, "r") {|f| Marshal.load(f)}
+
+if data == increments
+  puts "Equal counts: #{data}"
+else
+  puts "Not equal:"
+  puts "  increments: #{increments}"
+  puts "  data      : #{data}"
+end

data/junk/solaris-bug.txt

@@ -0,0 +1,43 @@
+Any solaris gurus out there?
+
+I'm having trouble porting some multi-thread, multi-process code from linux to solaris. I've already dealt with (or tried to deal with) some differences in flock (solaris flock is based on fcntl locks), like the fact that closing a file releases locks on the file held by other threads.
+
+I've managed to isolate the problem in a fairly simple test program. It's at
+
+  http://path.berkeley.edu/~vjoel/ruby/solaris-bug.rb
+
+The program creates /tmp/test-file-lock.dat, which holds a marshalled fixnum starting at 0. Then it creates Np processes each with Nt threads which do a random sequence of reads and writes using some locking methods. The writes just increment the counter.
+
+When a process is done, it writes the number of times it incremented the counter to the file /tmp/test-file-lock.dat#{pid}. Then the main process adds these up and compares with the contents of the counter file. The point of this is to test for colliding writers.
+
+But the program fails before that final test--it seems to be having a collision between a reader and a writer that causes the reader to see a corrupt file.
+
+A typical run fails like this. The counter 0..3 is a seconds clock:
+
+  $ ruby solaris-bug.rb
+  0
+  1
+  2
+  3
+  solaris-bug.rb:128:in `load': marshal data too short (ArgumentError)
+
+It looks like there are a reader and a writer accessing the file at the same time, and the writer has just truncated the file (line 137) when the reader tries to read it.
+
+This happens:
+
+  - on solaris, quad cpu
+    - ruby 1.7.3 (2002-10-30) [sparc-solaris2.7]
+ 
+  - *not* on single processor linux
+    - ruby 1.7.3 (2002-12-12) [i686-linux]
+
+  - *not* on dual SMP linux
+    - ruby 1.6.7 (2002-03-01) [i686-linux]
+
+Also, the bug requires *both* of:
+
+  - thread_count >= 2
+  
+  - process_count >= 2
+ 
+Also, the bug requires that there be both reader and writer operations (i.e., that the random number lead to each branch often enough, say 50/50).