fsdb 0.6.0 → 0.6.1

data/junk/test-processes.rb

@@ -1,131 +0,0 @@
-require './test.rb'
-
-## should also test nested transactions, replace, insert, delete, etc...
-module TestStressProcesses
-
-  class ProcessTester
-    attr_accessor :x
-    def initialize
-      @x = 0
-    end
-    def inspect
-      "<ProcessTester:@x=#{@x}>"
-    end
-    def to_s
-      inspect
-    end
-  end
-
-  def self.go
-
-    db = $db
-
-    object_count  =     5
-    process_count =    10
-    rep_count     =  1000
-    max_sleep     =     0.01
-
-    paths = (0...object_count).map {|i| "thread-test/t#{i}"}
-
-    paths.each do |path|
-      db[path] = ProcessTester.new
-    end
-
-    processes = (0...process_count).map do
-      fork do
-        srand(Process.pid)
-
-        rep_count.times do |iter|
-          path = paths[rand(paths.size)]
-
-          case rand(100)
-
-          when 0..49
-            db.browse path do |tester|
-              __x__ = tester.x
-
-              data = File.read(File.join(db.dir, path))
-              uncached_x = Marshal.load(data).x
-              unless uncached_x == __x__
-                fail "browse test: cache is stale:\n" +
-                      "  #{uncached_x} on disk, #{__x__} in cache"
-              end
-
-              sleep(rand*max_sleep)
-
-              unless __x__ == tester.x
-                fail "browse test: x == #{__x__} but tester.x == #{tester.x}"
-              end
-            end
-
-          when 50..69
-            db.edit path do |tester|
-              __x__ = tester.x
-
-              data = File.read(File.join(db.dir, path))
-              uncached_x = Marshal.load(data).x
-              unless uncached_x == __x__
-                fail "edit test: cache is stale:\n" +
-                      "  #{uncached_x} on disk, #{__x__} in cache"
-              end
-
-              tester.x += 1
-              __x__ = tester.x
-              
-              sleep(rand*max_sleep)
-              
-              unless __x__ == tester.x
-                fail "edit test: x == #{__x__} but tester.x == #{tester.x}"
-              end
-            end
-
-          when 70..90
-            db.replace path do |tester|
-              __x__ = tester.x
-
-              data = File.read(File.join(db.dir, path))
-              uncached_x = Marshal.load(data).x
-              unless uncached_x == __x__
-                fail "replace test: cache is stale:\n" +
-                      "  #{uncached_x} on disk, #{__x__} in cache"
-              end
-
-              tester.x += 1
-              __x__ = tester.x
-              
-              sleep(rand*max_sleep)
-              
-              unless __x__ == tester.x
-                fail "replace test: x == #{__x__} but tester.x == #{tester.x}"
-              end
-              
-              tester
-            end
-
-          else
-            sleep(rand*max_sleep)
-            db.clear_cache
-
-          end
-          
-        end
-      end
-    end
-    
-    thread = Thread.new do
-      i = 0
-      loop do
-        puts i
-        sleep 1
-        i += 1
-      end
-    end
-
-    process_count.times {Process.wait}
-    thread.kill
-
-  end
-
-end
-
-TestStressProcesses.go if __FILE__ == $0

data/junk/test-threads.rb

@@ -1,113 +0,0 @@
-require './test.rb'
-
-## should also test nested transactions, replace, insert, delete, etc...
-module TestStressThreads
-
-  class ThreadTester
-    attr_accessor :x
-    def initialize
-      @x = 0
-    end
-  end
-
-  def self.go
-
-    db = $db
-
-    srand(3765)
-
-    object_count  =    100
-    thread_count  =    100
-    rep_count     = 1000
-    max_sleep     =    nil
-
-    paths = (0...object_count).map {|i| "thread-test/t#{i}"}
-
-    paths.each do |path|
-      db[path] = ThreadTester.new
-    end
-
-    threads = (0...thread_count).map do |thread_index|
-      Thread.new(thread_index) do |ti|
-        Thread.current[:number] = ti
-        Thread.current[:transactions] = 0
-        rep_count.times do |iter|
-          Thread.current[:iter] = iter
-          path = paths[rand(paths.size)]
-
-          case rand(3)
-
-          when 0
-            db.browse path do |tester|
-              __x__ = tester.x
-              sleep(rand*max_sleep) if max_sleep
-              unless __x__ == tester.x
-                fail "browse: x == #{__x__} but tester.x == #{tester.x}"
-              end
-            end
-            Thread.current[:transactions] += 1
-
-          when 1
-            db.edit path do |tester|
-              tester.x += 1
-              __x__ = tester.x
-              sleep(rand*max_sleep) if max_sleep
-              unless __x__ == tester.x
-                fail "edit: x == #{__x__} but tester.x == #{tester.x}"
-              end
-            end
-            Thread.current[:transactions] += 1
-
-          else
-            sleep(rand*max_sleep) if max_sleep
-            if rand(20) == 0
-              db.clear_cache
-            end
-
-          end
-
-        end
-      end
-    end
-    
-    start_time = Process.times
-
-#    threads.each{|t|printf "%6d", t[:number]}
-#    puts
-#    puts "-----|" * threads.size
-    sleep 1
-
-#    stats = old_stats = nil
-    loop do
-#      old_stats = stats
-#      stats = threads.map{|t|t[:iter]}
-#      stats.each{|s|printf "%6d", s}
-      puts "."
-      break unless threads.any?{|t|t.alive?}
-#      if old_stats == stats
-#        fail "Deadlock!"
-#      end
-      sleep 1
-    end
-    
-    end_time = Process.times
-    
-    utime = end_time.utime - start_time.utime
-    stime = end_time.stime - start_time.stime
-    rate = threads.inject(0) {|sum, t| sum += t[:transactions]} /
-           (utime + stime)
-
-    puts "All db worker threads dead."
-    puts "time: %12s%12s"     % %w{user system}
-    puts "      %12.2f%12.2f" % [utime, stime]
-    puts "\nExecuted %d transactions per second" % rate
-
-    threads.each do |thread|
-      thread.join
-    end
-
-  end
-
-end
-
-TestStressThreads.go

data/junk/wiki-mutex.rb

@@ -1,108 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'thread'
-
-# Mutex class based on standard thread.rb Mutex, which has some problems:
-# - waiters are not a strict queue (try_lock can jump the queue, after
-#   which the queue gets *rotated*). Race condition.
-# - doesn't use Thread.exclusive in enough places
-# - no way to make dead threads give up the mutex, which is crucial in a fork
-class ForkableMutex
-  def initialize
-    @waiting = []               # queue of waiting threads
-    @locked = nil;              # the thread that has the lock
-    @waiting.taint  # enable tainted comunication
-    self.taint
-  end
-
-  def locked?
-    @locked
-  end
-
-  def try_lock
-    Thread.exclusive do
-      if not @locked and @waiting.empty?
-        @locked = Thread.current
-        true
-      end
-    end
-  end
-  
-  def lock
-    thread = Thread.current
-    Thread.exclusive do
-      if @locked
-        @waiting.push thread
-        Thread.stop
-        unless @locked == thread
-          raise ThreadError, "queue was jumped"
-        end
-      else
-        @locked = thread
-      end
-      self
-    end
-  end
-
-  def unlock
-    return unless @locked
-    
-    t = Thread.exclusive { wake_next_waiter }
-    
-    begin
-      t.run if t
-    rescue ThreadError
-    end
-    self
-  end
-
-  def synchronize
-    lock
-    yield
-  ensure
-    unlock
-  end
-
-  def remove_dead
-    Thread.exclusive do
-      @waiting = @waiting.select {|t| t.alive?}
-      wake_next_waiter if @locked and not @locked.alive?
-    end
-  end
-
-  private
-  def wake_next_waiter
-    t = @waiting.shift
-    t.wakeup if t
-    @locked = t
-  rescue ThreadError
-    retry
-  end
-end
-
-module ForkSafely
-  def fork
-    super do
-      ObjectSpace.each_object(ForkableMutex) { |m| m.remove_dead }
-      yield
-    end
-  end
-end
-include ForkSafely
-
-
-if __FILE__ == $0
-  include ForkSafely
-  
-  m = ForkableMutex.new
-
-  Thread.new { m.synchronize { sleep 2 } }
-
-  fork do
-    m.synchronize { puts "Didn't get here if you used standard mutex or fork." }
-  end
-
-  m.synchronize { puts "Got here." }
-
-  Process.wait
-end

data/misc/fsdb-blorubu.txt

@@ -1,47 +0,0 @@
-Like pstore, fsdb is:
-
-- pure ruby, cross-platform
-
-- ruby license
-
-- simple API
-
-- simple implementation (though less so)
-
-- transaction based, with abort capability
-
-However, pstore has some drawbacks:
-
-- not thread-safe (though it is process-safe)
-
-- coarse granularity (transaction applies to whole pstore), does not scale well
-
-- uses marshalling, even if your data is is some other format that doesn't require the overhead of marshalling (e.g., strings, binary data)
-
-Other advantages of fsdb:
-
-- can use YAML (or other formats) where you want to (e.g., in user settings, but perhaps not for large ruby objects or binary data). It's easy to specify a mapping from path patterns to data formats.
-
-- thread-safe, so you can easily wire it up to a multithreaded server, like drb (see examples/server.rb)
-
-- objects in the database are just files, and so can be manipulated with standard command line tools (including rsync, versioning tools, etc.).
-
-- granularity is up to you: you can have lots of little objects, if you want, (there are tradeoffs, of course)
-
-- scales as well as the fs does
-
-- thoroughly stress-tested, partially unit-tested. (But still not quite stable on Windows.)
-
-- Can select fcntl-based locking (if platform supports it), which should allow safe use with NFS mounts on linux
-
-Disadvantages of fsdb:
-
-- If you want indexes, you need to implement them on top of fsdb. You could do:
-
-  db['name-index'] = RBTree.new
-
-and define your own methods for adding/removing objects that also updates the name-index. The name-index maps names (or other keys) to paths in the db.
-
-- Likewise, no SQL, out of the box.
-
-- Efficiency and reliability depends on the fs. (My benchmarks get on the order of 1000 transactions per sec on a 3 yr old linux box.)

data/misc/mtime-and-file-id.txt

@@ -1,23 +0,0 @@
-mtime granularity on win?
-
-see:
-http://msdn.microsoft.com/library/default.asp?url=/library/en-us/fileio/base/by_handle_file_information_str.asp
-http://msdn.microsoft.com/library/default.asp?url=/library/en-us/sysinfo/base/file_times.asp
-
-http://www.google.com/search?hl=en&ie=ISO-8859-1&q=unique+file+id+on+windows&btnG=Google+Search
-
-http://discuss.fogcreek.com/joelonsoftware/default.asp?cmd=show&ixPost=42340
-
-Windows can give a unique file indentifier using the function GetFileInformationByHandle() (See http://msdn.microsoft.com/library/default.asp?url=/library/en-us/fileio/base/getfileinformationbyhandle.asp)
- 
- Example: (hFile is a handle to a file)
- 
- BY_HANDLE_FILE_INFORMATION FileInfo;
- 
- GetFileInformationByHandle(hFile, &FileInfo);
- inode= FileInfo.nFileIndexLow | (FileInfo.nFileIndexHigh << 32);
- volume=FileInfo.dwVolumeSerialNumber;
- 
- According to Micorsoft : "The identifier (low and high parts) and the volume serial number uniquely identify a file on a single computer. To determine whether two open handles represent the same file, combine this identifier and the volume serial number for each file and compare them." 
- 
- 

data/misc/posixlock.txt

@@ -1,148 +0,0 @@
-Subject:
-[ANN] posixlock
-From:
-"Ara.T.Howard" <ahoward@ngdc.noaa.gov>
-Date:
-Sat, 6 Dec 2003 05:22:07 +0900
-To:
-ruby-talk@ruby-lang.org (ruby-talk ML)
-Newsgroups:
-comp.lang.ruby
-
-
-SYNOPSIS:
-  posixlock
-    use fcntl (vs. flock) based locking for File#flock
-
-USAGE
-  require 'posixlock' # replaces File#flock!
-
-
-i currently do not have a place to post software - so here's a little
-extension that replaces flock with an fcntl based impl for posix compliant
-locking (safe on nfs) - posted to c.l.r for posterity:
-
-extconf.rb
-================
-require 'mkmf'
-create_makefile 'posixlock'
-================
-
-posixlock.c
-================
-#ifdef _WIN32
-#include "missing/file.h"
-#endif
-
-#include "ruby.h"
-#include "rubyio.h"
-#include "rubysig.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-
-#include <errno.h>
-
-extern VALUE rb_cFile;
-
-# ifndef LOCK_SH
-#  define LOCK_SH 1
-# endif
-# ifndef LOCK_EX
-#  define LOCK_EX 2
-# endif
-# ifndef LOCK_NB
-#  define LOCK_NB 4
-# endif
-# ifndef LOCK_UN
-#  define LOCK_UN 8
-# endif
-
-static int
-posixlock (fd, operation)
-     int fd;
-     int operation;
-{
-  struct flock lock;
-
-  switch (operation & ~LOCK_NB)
-    {
-    case LOCK_SH:
-      lock.l_type = F_RDLCK;
-      break;
-    case LOCK_EX:
-      lock.l_type = F_WRLCK;
-      break;
-    case LOCK_UN:
-      lock.l_type = F_UNLCK;
-      break;
-    default:
-      errno = EINVAL;
-      return -1;
-    }
-  lock.l_whence = SEEK_SET;
-  lock.l_start = lock.l_len = 0L;
-  return fcntl (fd, (operation & LOCK_NB) ? F_SETLK : F_SETLKW, &lock);
-}
-
-
-static VALUE
-rb_file_posixlock (obj, operation)
-     VALUE obj;
-     VALUE operation;
-{
-#ifndef __CHECKER__
-  OpenFile *fptr;
-  int ret;
-
-  rb_secure (2);
-  GetOpenFile (obj, fptr);
-
-  if (fptr->mode & FMODE_WRITABLE)
-    {
-      fflush (GetWriteFile (fptr));
-    }
-retry:
-  TRAP_BEG;
-  ret = posixlock (fileno (fptr->f), NUM2INT (operation));
-  TRAP_END;
-  if (ret < 0)
-    {
-      switch (errno)
-	{
-	case EAGAIN:
-	case EACCES:
-#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
-	case EWOULDBLOCK:
-#endif
-	  return Qfalse;
-	case EINTR:
-#if defined(ERESTART)
-	case ERESTART:
-#endif
-	  goto retry;
-	}
-      rb_sys_fail (fptr->path);
-    }
-#endif
-
-  return INT2FIX (0);
-}
-
-
-void
-Init_posixlock()
-{
-    rb_define_method(rb_cFile, "flock", rb_file_posixlock, 1);
-}
-================
-
-
-
--a
--- ATTN: please update your address books with address below! =============================================================================== | EMAIL :: Ara [dot] T [dot] Howard [at] noaa [dot] gov | PHONE :: 303.497.6469 | ADDRESS :: E/GC2 325 Broadway, Boulder, CO 80305-3328 | STP :: http://www.ngdc.noaa.gov/stp/ | NGDC :: http://www.ngdc.noaa.gov/ | NESDIS :: http://www.nesdis.noaa.gov/ | NOAA :: http://www.noaa.gov/ | US DOC :: http://www.commerce.gov/ | | The difference between art and science is that science is what we | understand well enough to explain to a computer. | Art is everything else. | -- Donald Knuth, "Discover" | | /bin/sh -c 'for l in ruby perl;do $l -e "print \"\x3a\x2d\x29\x0a\"";done' ===============================================================================