fsdb 0.5 → 0.6.0

data/junk/test-processes.rb

@@ -0,0 +1,131 @@
+require './test.rb'
+
+## should also test nested transactions, replace, insert, delete, etc...
+module TestStressProcesses
+
+  class ProcessTester
+    attr_accessor :x
+    def initialize
+      @x = 0
+    end
+    def inspect
+      "<ProcessTester:@x=#{@x}>"
+    end
+    def to_s
+      inspect
+    end
+  end
+
+  def self.go
+
+    db = $db
+
+    object_count  =     5
+    process_count =    10
+    rep_count     =  1000
+    max_sleep     =     0.01
+
+    paths = (0...object_count).map {|i| "thread-test/t#{i}"}
+
+    paths.each do |path|
+      db[path] = ProcessTester.new
+    end
+
+    processes = (0...process_count).map do
+      fork do
+        srand(Process.pid)
+
+        rep_count.times do |iter|
+          path = paths[rand(paths.size)]
+
+          case rand(100)
+
+          when 0..49
+            db.browse path do |tester|
+              __x__ = tester.x
+
+              data = File.read(File.join(db.dir, path))
+              uncached_x = Marshal.load(data).x
+              unless uncached_x == __x__
+                fail "browse test: cache is stale:\n" +
+                      "  #{uncached_x} on disk, #{__x__} in cache"
+              end
+
+              sleep(rand*max_sleep)
+
+              unless __x__ == tester.x
+                fail "browse test: x == #{__x__} but tester.x == #{tester.x}"
+              end
+            end
+
+          when 50..69
+            db.edit path do |tester|
+              __x__ = tester.x
+
+              data = File.read(File.join(db.dir, path))
+              uncached_x = Marshal.load(data).x
+              unless uncached_x == __x__
+                fail "edit test: cache is stale:\n" +
+                      "  #{uncached_x} on disk, #{__x__} in cache"
+              end
+
+              tester.x += 1
+              __x__ = tester.x
+              
+              sleep(rand*max_sleep)
+              
+              unless __x__ == tester.x
+                fail "edit test: x == #{__x__} but tester.x == #{tester.x}"
+              end
+            end
+
+          when 70..90
+            db.replace path do |tester|
+              __x__ = tester.x
+
+              data = File.read(File.join(db.dir, path))
+              uncached_x = Marshal.load(data).x
+              unless uncached_x == __x__
+                fail "replace test: cache is stale:\n" +
+                      "  #{uncached_x} on disk, #{__x__} in cache"
+              end
+
+              tester.x += 1
+              __x__ = tester.x
+              
+              sleep(rand*max_sleep)
+              
+              unless __x__ == tester.x
+                fail "replace test: x == #{__x__} but tester.x == #{tester.x}"
+              end
+              
+              tester
+            end
+
+          else
+            sleep(rand*max_sleep)
+            db.clear_cache
+
+          end
+          
+        end
+      end
+    end
+    
+    thread = Thread.new do
+      i = 0
+      loop do
+        puts i
+        sleep 1
+        i += 1
+      end
+    end
+
+    process_count.times {Process.wait}
+    thread.kill
+
+  end
+
+end
+
+TestStressProcesses.go if __FILE__ == $0

data/junk/test-threads.rb

@@ -0,0 +1,113 @@
+require './test.rb'
+
+## should also test nested transactions, replace, insert, delete, etc...
+module TestStressThreads
+
+  class ThreadTester
+    attr_accessor :x
+    def initialize
+      @x = 0
+    end
+  end
+
+  def self.go
+
+    db = $db
+
+    srand(3765)
+
+    object_count  =    100
+    thread_count  =    100
+    rep_count     = 1000
+    max_sleep     =    nil
+
+    paths = (0...object_count).map {|i| "thread-test/t#{i}"}
+
+    paths.each do |path|
+      db[path] = ThreadTester.new
+    end
+
+    threads = (0...thread_count).map do |thread_index|
+      Thread.new(thread_index) do |ti|
+        Thread.current[:number] = ti
+        Thread.current[:transactions] = 0
+        rep_count.times do |iter|
+          Thread.current[:iter] = iter
+          path = paths[rand(paths.size)]
+
+          case rand(3)
+
+          when 0
+            db.browse path do |tester|
+              __x__ = tester.x
+              sleep(rand*max_sleep) if max_sleep
+              unless __x__ == tester.x
+                fail "browse: x == #{__x__} but tester.x == #{tester.x}"
+              end
+            end
+            Thread.current[:transactions] += 1
+
+          when 1
+            db.edit path do |tester|
+              tester.x += 1
+              __x__ = tester.x
+              sleep(rand*max_sleep) if max_sleep
+              unless __x__ == tester.x
+                fail "edit: x == #{__x__} but tester.x == #{tester.x}"
+              end
+            end
+            Thread.current[:transactions] += 1
+
+          else
+            sleep(rand*max_sleep) if max_sleep
+            if rand(20) == 0
+              db.clear_cache
+            end
+
+          end
+
+        end
+      end
+    end
+    
+    start_time = Process.times
+
+#    threads.each{|t|printf "%6d", t[:number]}
+#    puts
+#    puts "-----|" * threads.size
+    sleep 1
+
+#    stats = old_stats = nil
+    loop do
+#      old_stats = stats
+#      stats = threads.map{|t|t[:iter]}
+#      stats.each{|s|printf "%6d", s}
+      puts "."
+      break unless threads.any?{|t|t.alive?}
+#      if old_stats == stats
+#        fail "Deadlock!"
+#      end
+      sleep 1
+    end
+    
+    end_time = Process.times
+    
+    utime = end_time.utime - start_time.utime
+    stime = end_time.stime - start_time.stime
+    rate = threads.inject(0) {|sum, t| sum += t[:transactions]} /
+           (utime + stime)
+
+    puts "All db worker threads dead."
+    puts "time: %12s%12s"     % %w{user system}
+    puts "      %12.2f%12.2f" % [utime, stime]
+    puts "\nExecuted %d transactions per second" % rate
+
+    threads.each do |thread|
+      thread.join
+    end
+
+  end
+
+end
+
+TestStressThreads.go

data/junk/wiki-mutex.rb

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+
+require 'thread'
+
+# Mutex class based on standard thread.rb Mutex, which has some problems:
+# - waiters are not a strict queue (try_lock can jump the queue, after
+#   which the queue gets *rotated*). Race condition.
+# - doesn't use Thread.exclusive in enough places
+# - no way to make dead threads give up the mutex, which is crucial in a fork
+class ForkableMutex
+  def initialize
+    @waiting = []               # queue of waiting threads
+    @locked = nil;              # the thread that has the lock
+    @waiting.taint  # enable tainted comunication
+    self.taint
+  end
+
+  def locked?
+    @locked
+  end
+
+  def try_lock
+    Thread.exclusive do
+      if not @locked and @waiting.empty?
+        @locked = Thread.current
+        true
+      end
+    end
+  end
+  
+  def lock
+    thread = Thread.current
+    Thread.exclusive do
+      if @locked
+        @waiting.push thread
+        Thread.stop
+        unless @locked == thread
+          raise ThreadError, "queue was jumped"
+        end
+      else
+        @locked = thread
+      end
+      self
+    end
+  end
+
+  def unlock
+    return unless @locked
+    
+    t = Thread.exclusive { wake_next_waiter }
+    
+    begin
+      t.run if t
+    rescue ThreadError
+    end
+    self
+  end
+
+  def synchronize
+    lock
+    yield
+  ensure
+    unlock
+  end
+
+  def remove_dead
+    Thread.exclusive do
+      @waiting = @waiting.select {|t| t.alive?}
+      wake_next_waiter if @locked and not @locked.alive?
+    end
+  end
+
+  private
+  def wake_next_waiter
+    t = @waiting.shift
+    t.wakeup if t
+    @locked = t
+  rescue ThreadError
+    retry
+  end
+end
+
+module ForkSafely
+  def fork
+    super do
+      ObjectSpace.each_object(ForkableMutex) { |m| m.remove_dead }
+      yield
+    end
+  end
+end
+include ForkSafely
+
+
+if __FILE__ == $0
+  include ForkSafely
+  
+  m = ForkableMutex.new
+
+  Thread.new { m.synchronize { sleep 2 } }
+
+  fork do
+    m.synchronize { puts "Didn't get here if you used standard mutex or fork." }
+  end
+
+  m.synchronize { puts "Got here." }
+
+  Process.wait
+end

data/lib/fsdb/database.rb

@@ -1,4 +1,4 @@
-require 'ftools'
+require 'fileutils'
 require 'fsdb/platform'
 require 'fsdb/compat' if RUBY_VERSION.to_f < 1.7
 require 'fsdb/mutex'
@@ -9,6 +9,8 @@ require 'fsdb/formats'
 module FSDB
 include Formats
 
+FSDB::VERSION = "0.6.0"
+
 # A thread-safe, process-safe object database class which uses the
 # native file system as its back end and allows multiple file formats.
 
@@ -158,7 +160,7 @@ class Database
 
     @formats = opts[:formats]
     
-    File.makedirs(@dir)
+    FileUtils.makedirs(@dir)
   end
   
   # Shortcut to create a new database at +path+.
@@ -236,7 +238,7 @@ class Database
   def make_file_id(abs_path)
     dirname = File.dirname(abs_path)
     begin
-      File.makedirs(dirname)
+      FileUtils.makedirs(dirname)
     rescue Errno::EEXIST
       raise PathComponentError
     end

data/lib/fsdb/delegatable.rb

@@ -0,0 +1,21 @@
+module Delegatable
+  def delegate h
+    h.each do |meth, reader|
+      module_eval %{
+        def #{meth}(*args, &block); #{reader}.#{meth}(*args, &block); end
+      }
+    end
+  end
+end
+  
+    def browse(&block);  @db.browse(path, &block);  end
+    def edit(&block);    @db.edit(path, &block);    end
+    def replace(&block); @db.replace(path, &block); end
+    def insert(&block);  @db.insert(path, &block);  end
+    def delete(&block);  @db.delete(path, &block);  end
+    def fetch(&block);   @db.fetch(path, &block);   end
+
+    extend Delegatable
+    delegate :browse => "@db", :edit => "@db", :replace => "@db",
+             :insert => "@db", :delegate => "@db", :fetch => "@db"
+    

data/lib/fsdb/faster-modex.rb

@@ -0,0 +1,223 @@
+#!/usr/bin/env ruby
+
+### only use this if you can ensure Thread.critical is not already set!
+
+# Make sure we use the fast definition, not the thread.rb one!
+class Thread # :nodoc:
+  def self.exclusive
+    old = critical
+    self.critical = true
+    yield
+  ensure
+    self.critical = old
+  end
+end
+
+class Thread
+  def self.nonexclusive
+    old = critical
+    self.critical = false
+    yield
+  ensure
+    self.critical = old
+  end
+end
+
+module FSDB
+
+# Modex is a modal exclusion semaphore, like in syncronizer.rb.
+# The two modes are shared (SH) and exclusive (EX).
+# Modex is not nestable.
+#
+class Modex
+  SH = :SH
+  EX = :EX
+  
+  def initialize
+    @waiting  = []
+    @locked   = []
+    @mode     = nil
+    @first    = true
+  end
+
+  def try_lock mode
+    Thread.critical = true
+      thread = Thread.current
+      if @locked.include?(thread)
+        Thread.critical = false
+        raise ThreadError
+      end
+
+      if @mode == mode and mode == SH and @waiting.empty? # strict queue
+        @locked << thread
+        rslt = true
+      elsif not @mode
+        @mode = mode
+        @locked << thread
+        rslt = true
+      end
+    Thread.critical = false
+    rslt
+  end
+  
+  # the block is executed in the exclusive context
+  def lock mode
+    Thread.critical = true
+      thread = Thread.current
+      if @locked.include?(thread)
+        Thread.critical = false
+        raise ThreadError
+      end
+
+      if @mode == mode and mode == SH and @waiting.empty? # strict queue
+        @locked << thread
+      elsif not @mode
+        @mode = mode
+        @locked << thread
+      else
+        @waiting << thread << mode
+        Thread.stop
+        Thread.critical = true
+      end
+      
+      yield if block_given?
+
+#      if @mode != mode
+#        raise "@mode == #{@mode} but mode == #{mode}"
+#      end
+#
+#      if @mode == EX and @locked.size > 1
+#        raise "@mode == EX but @locked.size == #{@locked.size}"
+#      end
+
+    Thread.critical = false
+      self
+  end
+
+  # the block is executed in the exclusive context
+  def unlock
+    raise ThreadError unless @mode
+    
+    Thread.critical = true
+      yield if block_given?
+      @locked.delete Thread.current
+      wake_next_waiter if @locked.empty?
+    Thread.critical = false
+
+    self
+  end
+  
+  def synchronize mode, do_when_first = nil, do_when_last = nil, arg = nil
+    lock mode do
+      if @first
+        @first = false
+        
+        if do_when_first
+          if mode == SH
+            @mode = EX
+          end
+
+          Thread.critical = false; do_when_first[arg]; Thread.critical = true
+
+          if mode == SH
+            @mode = SH
+            wake_waiting_sharers
+          end
+        end
+      end
+    end
+    
+    yield
+    
+  ensure
+    unlock do
+      if @locked.size == 1
+        if do_when_last
+          @mode = EX
+          Thread.critical = false; do_when_last[arg]; Thread.critical = true
+        end
+        @first = true
+      end
+    end
+  end
+  
+  def remove_dead # :nodoc:
+    Thread.exclusive do
+      waiting = @waiting; @waiting = []
+      until waiting.empty?
+      
+        t = waiting.shift; m = waiting.shift
+        @waiting << t << m if t.alive?
+      end
+      
+      @locked = @locked.select {|t| t.alive?}
+      wake_next_waiter if @locked.empty?
+    end
+  end
+
+  private
+  def wake_next_waiter
+    first = @waiting.shift; @mode = @waiting.shift && EX
+    if first
+      first.wakeup
+      @locked << first
+    end
+    first
+  rescue ThreadError
+    retry
+  end
+  
+  def wake_waiting_sharers
+    while @waiting[1] == SH  # note strict queue order
+      t = @waiting.shift; @waiting.shift
+      @locked << t
+      t.wakeup
+    end
+  rescue ThreadError
+    retry
+  end
+
+  module ForkSafely
+    def fork # :nodoc:
+      super do
+        ObjectSpace.each_object(Modex) { |m| m.remove_dead }
+        yield
+      end
+    end
+  end
+end
+
+# FSDB users who fork should include ForkSafely or FSDB itself. (The reason for
+# this is that the fork may inherit some dead threads from the parent, and if
+# they hold any locks, you may get a deadlock. ForkSafely modifies fork so that
+# these dead threads are cleared. If you use modexes (outside of those in FSDB),
+# they should be FSDB::Modexes.
+module ForkSafely
+  include Modex::ForkSafely
+end
+include ForkSafely
+
+end # module FSDB
+
+
+if __FILE__ == $0
+  # Stress test is in fsdb/test/test-modex.rb. This is just to show fork usage.
+  
+  include FSDB::ForkSafely
+  
+  m = FSDB::Modex.new
+  
+  SH = FSDB::Modex::SH
+
+  Thread.new { m.synchronize(SH) { sleep 1 } }
+
+  fork do
+    m.synchronize(SH) do
+     puts "Didn't get here if you used standard mutex or fork."
+    end
+  end
+
+  m.synchronize(SH) { puts "Got here." }
+
+  Process.wait
+end